Merge branch 'develop' into logExpandCollapse

tmobile · Jan 29, 2019 · 7731548 · 7731548
2 parents 48de68b + 80ba8c9
commit 7731548
Show file tree

Hide file tree

Showing 50 changed files with 970 additions and 298 deletions.
diff --git a/builds/jazz-build-module/utility-loader.groovy b/builds/jazz-build-module/utility-loader.groovy
@@ -166,4 +166,9 @@ def getApiToken(){
 	}
 }
 
+def isReplayedBuild() {
+  def replayClassName = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
+  currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClassName) }
+}
+
 return this
diff --git a/builds/jenkins-build-pack-api/Jenkinsfile b/builds/jenkins-build-pack-api/Jenkinsfile
@@ -40,6 +40,7 @@ node() {
 
 	def branch = params.scm_branch
 	def repo_name = params.service_name
+	def requestId = params.request_id
 	def gitCommitOwner
 	def gitCommitHash
 	def context_map
@@ -135,15 +136,18 @@ node() {
 				}
 			}
 		}
-
-		def requestId = utilModule.generateRequestId()
-		if (requestId != null) {
+		echo "requestId: $requestId"
+		if (requestId != "none" && requestId != null && !utilModule.isReplayedBuild()) {
+			echo "Setting up requestId"
+			requestId = requestId
 			events.setRequestId(requestId)
 			environmentDeploymentMetadata.setRequestId(requestId)
 		} else {
-			error "Request Id Generation failed"
+			requestId = utilModule.generateRequestId()
+			events.setRequestId(requestId)
+			environmentDeploymentMetadata.setRequestId(requestId)
+			events.sendStartedEvent('CREATE_DEPLOYMENT', 'Service deployment workflow started', environmentDeploymentMetadata.generateDeploymentMap("started", environment_logical_id, gitCommitHash), environment_logical_id)
 		}
-		events.sendStartedEvent('CREATE_DEPLOYMENT', 'Service deployment workflow started', environmentDeploymentMetadata.generateDeploymentMap("started", environment_logical_id, gitCommitHash), environment_logical_id)
 
 		stage('Pre-Build Validation') {
 			try {
@@ -505,16 +509,16 @@ def buildLambda(String runtime, String repo_name) {
 		  	pip install pytest
 	  		"""
 		}
-	} else if (runtime.indexOf("go") > -1 ) {
-		// Installing dependencies using dep ensure
-		// golang build scripts
-		withEnv(["GOPATH=${env.WORKSPACE}"]) {
-			sh "mkdir -p $GOPATH/src"
-			sh "rsync -a --exclude='.*' $GOPATH/" + repo_name + " $GOPATH/src"
-			sh "cd $GOPATH/src/" + repo_name + " && dep ensure"
-			sh "env GOOS=linux GOARCH=amd64 go build -o main $GOPATH/src/" + repo_name + "/main.go"
-		}
-	}
+	} else if (runtime.indexOf("go") > -1 ){
+    //Installing dependencies using dep ensure
+    //golang build scripts
+   	withEnv(["GOPATH=${env.WORKSPACE}"]) {
+      sh "mkdir -p $GOPATH/src"
+      sh "rsync -a --exclude='.*' $GOPATH/"+ repo_name+"  $GOPATH/src"
+      sh "cd $GOPATH/src/"+ repo_name+" && dep ensure"
+      sh "cd $GOPATH/src/"+ repo_name+" && env GOOS=linux GOARCH=amd64 go build -o $GOPATH/"+ repo_name+"/main  *.go"
+    }
+  }
 }
 
 /**
@@ -678,7 +682,7 @@ def loadServerlessConfig(config) {
 	} else if (config['providerRuntime'].indexOf("go") > -1) {
 		sh "cp _config/serverless-go.yml ./serverless.yml"
 	}
-	
+
 	removeEventResources()
 	echoServerlessFile()
 }

diff --git a/builds/jenkins-build-pack-lambda/Jenkinsfile b/builds/jenkins-build-pack-lambda/Jenkinsfile
@@ -42,6 +42,7 @@ node() {
 
   def branch = params.scm_branch
   def repo_name = params.service_name
+  def requestId = params.request_id
   def gitCommitOwner
   def gitCommitHash
   def context_map
@@ -138,15 +139,18 @@ node() {
     isEventSchdld = true
   }
 
-  def requestId = utilModule.generateRequestId()
-  if (requestId != null) {
+  echo "requestId: $requestId"
+  if (requestId != "none" && requestId != null && !utilModule.isReplayedBuild()) {
+    requestId = requestId
     events.setRequestId(requestId)
     environmentDeploymentMetadata.setRequestId(requestId)
   } else {
-    error "Request Id Generation failed"
+    requestId = utilModule.generateRequestId()
+    events.setRequestId(requestId)
+    environmentDeploymentMetadata.setRequestId(requestId)
+    events.sendStartedEvent('CREATE_DEPLOYMENT', 'Function deployment started', environmentDeploymentMetadata.generateDeploymentMap("started", environment_logical_id, gitCommitHash), environment_logical_id)
   }
 
-  events.sendStartedEvent('CREATE_DEPLOYMENT', 'Function deployment started', environmentDeploymentMetadata.generateDeploymentMap("started", environment_logical_id, gitCommitHash), environment_logical_id)
 
   dir(repo_name) {
     loadServerlessConfig(runtime, isEventSchdld, isScheduleEnabled, isEc2EventEnabled, isS3EventEnabled, isSQSEventEnabled, isStreamEnabled, isDynamoDbEnabled)
@@ -563,7 +567,7 @@ def buildLambda(String runtime, String repo_name) {
       sh "mkdir -p $GOPATH/src"
       sh "rsync -a --exclude='.*' $GOPATH/" + repo_name + " $GOPATH/src"
       sh "cd $GOPATH/src/"+ repo_name+" && dep ensure"
-      sh "env GOOS=linux GOARCH=amd64 go build -o main $GOPATH/src/" + repo_name + "/main.go"
+      sh "cd $GOPATH/src/"+ repo_name+" && env GOOS=linux GOARCH=amd64 go build -o $GOPATH/"+ repo_name+"/main  *.go"
     }
   }
 }
@@ -712,7 +716,7 @@ def echoServiceInfo(String env) {
     echo "==============================================================================================="
     echo svc_response
     echo "==============================================================================================="
-    
+
     return svc_response
 
   } catch (Exception ex) {
@@ -945,7 +949,7 @@ def getEventsArn(config, env) {
 
 def getLambdaARN(stackName, credsId) {
   def ARN = "";
-  
+
   try {
     def cloudformation_resources = "";
     cloudformation_resources = sh(returnStdout: true, script: "aws cloudformation describe-stacks --output json --stack-name ${stackName} --profile ${credsId}")

diff --git a/builds/jenkins-build-pack-website/Jenkinsfile b/builds/jenkins-build-pack-website/Jenkinsfile
@@ -41,6 +41,7 @@ node()  {
 	def domain = params.domain
 	def service = params.service_name
 	def repo_name = params.service_name
+	def requestId = params.request_id
 	def gitCommitOwner
 	def gitCommitHash
 
@@ -75,21 +76,22 @@ node()  {
 
 	environmentDeploymentMetadata.initialize(service_config, configLoader, scmModule, branch, env.BUILD_URL, env.BUILD_ID, g_base_url + "/jazz/environments", auth_token)
 
-	def environment_logical_id = environmentDeploymentMetadata.getEnvironmentLogicalId();
+	def environment_logical_id
 	if (params.scm_branch == 'master') {
-		environment = 'prod'
+		environment_logical_id = 'prod'
 		current_environment = 'PROD'
 		s3Bucket = configLoader.JAZZ.S3.WEBSITE_PROD_BUCKET;
 	} else {
-		if (environment_logical_id) {
-			environment = environment_logical_id
-		} else {
+    environment_logical_id = environmentDeploymentMetadata.getEnvironmentLogicalId();
+		if (!environment_logical_id) {
 			error "The environment has not been created yet and its missing the logical Id"
 		}
 		current_environment = "DEV"
 		s3Bucket = configLoader.JAZZ.S3.WEBSITE_DEV_BUCKET;
 	}
 
+  environment = environment_logical_id
+
 	if (!events) { error "Can't load events module"	} //Fail here
 	events.initialize(configLoader, service_config, "SERVICE_DEPLOYMENT", branch, environment, g_base_url + "/jazz/events")
 
@@ -99,17 +101,21 @@ node()  {
 	gitCommitHash = scmModule.getRepoCommitHash()
 	gitCommitOwner = scmModule.getRepoCommitterInfo(gitCommitHash)
 
-	def requestId = utilModule.generateRequestId()
-	if (requestId != null) {
+	echo "requestId: $requestId"
+	if(requestId != "none" && requestId != null && !utilModule.isReplayedBuild()) {
+		echo "Setting up requestId"
 		events.setRequestId(requestId)
 		environmentDeploymentMetadata.setRequestId(requestId)
 	} else {
-		error "Request Id Generation failed"
+		requestId = utilModule.generateRequestId()
+		events.setRequestId(requestId)
+		environmentDeploymentMetadata.setRequestId(requestId)
+		events.sendStartedEvent('CREATE_DEPLOYMENT', 'Website deployment started', environmentDeploymentMetadata.generateDeploymentMap("started", environment_logical_id, gitCommitHash), environment_logical_id)
 	}
 
+
 	dir(repo_name)
 	{
-		events.sendStartedEvent('CREATE_DEPLOYMENT', 'Website deployment started', environmentDeploymentMetadata.generateDeploymentMap("started", environment, , gitCommitHash), environment)
 
 		stage('Pre-Build Validation'){
 
@@ -149,7 +155,7 @@ node()  {
 				sh "aws configure set profile.${credsId}.aws_access_key_id $AWS_ACCESS_KEY_ID"
 				sh "aws configure set profile.${credsId}.aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
 				sh "aws configure set preview.cloudfront true"
-				
+
 				generateAssetInfo()
 				def assetInfo = "S3 Folder - /${asset_info}"
 

diff --git a/builds/serverless-config-pack/serverless-go.yml b/builds/serverless-config-pack/serverless-go.yml
@@ -47,7 +47,20 @@ functions:
           enabled: ${file(deployment-env.yml):eventScheduleEnable}
       #End:isScheduleEnabled
       #Start:isDynamoDbEnabled
-      - stream: {event_source_dynamodb}
+      - stream:
+          type: dynamodb
+          #Start:dynamoDbstreamArn
+          streamArnDisabled: {event_dynamodb_stream_arn}
+          #End:dynamoDbstreamArn
+          #Start:dynamoDbstreamGetArn
+          arn:
+            Fn::GetAtt:
+              - sourceDynamoDbTable
+              - StreamArn
+          #End:dynamoDbstreamGetArn
+          batchSize: 1
+          startingPosition: LATEST
+          enabled: true
       #End:isDynamoDbEnabled
       #Start:isSQSEventEnabled
       - sqs:
@@ -62,7 +75,7 @@ functions:
           #Start:streamGetArn
           arn:
             Fn::GetAtt:
-              - {event_source_kinesis}
+              - sourceKinesisStreams
               - Arn
           #End:streamGetArn
           #Start:streamArn
@@ -96,7 +109,7 @@ resourcesDisabled:
     customEventRole: ${file(policyFile.yml):kinesisRole}
     #End:isStreamEnabled
     #Start:isDynamoDbEnabled
-    customEventRole: ${file(policyFile.yml):eventSubscribed}
+    customEventRole: ${file(policyFile.yml):dynamoDbRole}
     #End:isDynamoDbEnabled
     #Start:isSQSEventEnabled
     customEventRole: ${file(policyFile.yml):sqsRole}

diff --git a/core/jazz_deployments-event-handler/config/dev-config.json b/core/jazz_deployments-event-handler/config/dev-config.json
@@ -18,5 +18,12 @@
         "create_event_name": "CREATE_DEPLOYMENT",
         "update_event_name": "UPDATE_DEPLOYMENT",
         "delete_event_name": "DELETE_DEPLOYMENT"
-    }
+    },
+    "DEPLOYMENT_KEY_LIST": [
+        "status",
+        "provider_build_url",
+        "provider_build_id",
+        "scm_commit_hash",
+        "scm_url",
+        "scm_branch"]
 }
diff --git a/core/jazz_deployments-event-handler/config/prod-config.json b/core/jazz_deployments-event-handler/config/prod-config.json
@@ -18,5 +18,12 @@
         "create_event_name": "CREATE_DEPLOYMENT",
         "update_event_name": "UPDATE_DEPLOYMENT",
         "delete_event_name": "DELETE_DEPLOYMENT"
-    }
-}
+    },
+    "DEPLOYMENT_KEY_LIST": [
+        "status",
+        "provider_build_url",
+        "provider_build_id",
+        "scm_commit_hash",
+        "scm_url",
+        "scm_branch"]
+}
diff --git a/core/jazz_deployments-event-handler/config/stg-config.json b/core/jazz_deployments-event-handler/config/stg-config.json
@@ -18,5 +18,12 @@
         "create_event_name": "CREATE_DEPLOYMENT",
         "update_event_name": "UPDATE_DEPLOYMENT",
         "delete_event_name": "DELETE_DEPLOYMENT"
-    }
+    },
+    "DEPLOYMENT_KEY_LIST": [
+        "status",
+        "provider_build_url",
+        "provider_build_id",
+        "scm_commit_hash",
+        "scm_url",
+        "scm_branch"]
 }
diff --git a/core/jazz_deployments-event-handler/index.js b/core/jazz_deployments-event-handler/index.js
@@ -217,14 +217,13 @@ function updateDeployments(res, deploymentPayload, configData, authToken) {
 			var deploymentData;
 
 			for (var idx in deploymentsCollection) {
-				if (deploymentsCollection[idx].provider_build_url === deploymentPayload.provider_build_url &&
-					deploymentsCollection[idx].provider_build_id === deploymentPayload.provider_build_id) {
+				if (deploymentsCollection[idx].request_id === deploymentPayload.request_id) {
 					deploymentData = deploymentsCollection[idx];
 				}
 			}
 			if (deploymentData && deploymentData.deployment_id) {
 				Object.keys(deploymentPayload).forEach(function (key) {
-					if (key !== 'status') {
+					if (!(configData.DEPLOYMENT_KEY_LIST.includes(key))) {
 						deploymentPayload[key] = deploymentData[key];
 					}
 				});

diff --git a/core/jazz_deployments-event-handler/test/test.js b/core/jazz_deployments-event-handler/test/test.js
@@ -594,7 +594,8 @@ describe("updateDeployments", () => {
       service: 'test-02',
       environment_logical_id: 'temp_env_ID',
       provider_build_url: "http://temp_testing/dccdw.com",
-      provider_build_id: "temp_build_id"
+      provider_build_id: "temp_build_id",
+      request_id: "temp-reqid-0001"
     };
     res = {
       "data": {
@@ -624,6 +625,7 @@ describe("updateDeployments", () => {
       sinon.assert.calledOnce(processRequestStub);
       processRequestStub.restore();
     })
+
   })
   it("should call return error if  processRequest is unsucesfull", () => {
     var processRequestStub = sinon.stub(index, "processRequest").rejects({

diff --git a/core/jazz_deployments/config/dev-config.json b/core/jazz_deployments/config/dev-config.json
@@ -5,16 +5,11 @@
         "service",
         "domain",
         "environment_logical_id",
-        "provider_build_url",
-        "provider_build_id",
-        "scm_commit_hash",
-        "scm_url",
-        "scm_branch",
         "status",
         "request_id"
     ],
     "REQUIRED_PARAMS": ["service", "domain", "environment"],
-    "OPTIONAL_PARAMS": ["status", "offset", "limit"],
+    "OPTIONAL_PARAMS": ["status", "offset", "limit", "provider_build_url", "provider_build_id", "scm_commit_hash", "scm_url","scm_branch"],
     "ARCHIVED_DEPLOYMENT_STATUS": "archived",
     "DEPLOYMENTS_SORTING_KEY": "provider_build_id",
     "DEPLOYMENTS_SORTING_ORDER": "desc",
@@ -38,5 +33,6 @@
         "api": "build-pack-api",
         "function": "build-pack-lambda",
         "website": "build-pack-website"
-    }
+    },
+    "EVENT_API_URL": "/dev/jazz/events"
 }
diff --git a/core/jazz_deployments/config/prod-config.json b/core/jazz_deployments/config/prod-config.json
@@ -5,16 +5,11 @@
         "service",
         "domain",
         "environment_logical_id",
-        "provider_build_url",
-        "provider_build_id",
-        "scm_commit_hash",
-        "scm_url",
-        "scm_branch",
         "status",
         "request_id"
     ],
     "REQUIRED_PARAMS": ["service", "domain", "environment"],
-    "OPTIONAL_PARAMS": ["status", "offset", "limit"],
+    "OPTIONAL_PARAMS": ["status", "offset", "limit", "provider_build_url", "provider_build_id", "scm_commit_hash", "scm_url","scm_branch"],
     "ARCHIVED_DEPLOYMENT_STATUS": "archived",
     "DEPLOYMENTS_SORTING_KEY": "provider_build_id",
     "DEPLOYMENTS_SORTING_ORDER": "desc",
@@ -38,5 +33,6 @@
         "api": "build-pack-api",
         "function": "build-pack-lambda",
         "website": "build-pack-website"
-    }
+    },
+    "EVENT_API_URL": "/prod/jazz/events"
 }