diff --git a/build.sbt b/build.sbt index c653152..ea2af59 100644 --- a/build.sbt +++ b/build.sbt @@ -68,7 +68,7 @@ lazy val utils = (project in file(".")). site.addMappingsToSiteDir(mappings in(ScalaUnidoc, packageDoc), "latest/api"), git.remoteRepo := "git@github.com:indix/utils.git" ). - aggregate(coreUtils, storeUtils, sparkUtils) + aggregate(coreUtils, storeUtils, sparkUtils, gocdUtils) lazy val coreUtils = (project in file("util-core")). settings(commonSettings: _*). @@ -118,3 +118,22 @@ lazy val sparkUtils = (project in file("util-spark")). "org.bdgenomics.utils" %% "utils-misc" % "0.2.13" ) ) + +lazy val gocdUtils = (project in file("util-gocd")). + settings(commonSettings: _*). + settings(publishSettings: _*). + settings( + name := "util-gocd", + crossScalaVersions := Seq("2.10.6", "2.11.11"), + libraryDependencies ++= Seq( + "org.apache.commons" % "commons-lang3" % "3.1", + "commons-io" % "commons-io" % "1.3.2", + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.127", + "cd.go.plugin" % "go-plugin-api" % "17.2.0" % Provided, + "com.google.code.gson" % "gson" % "2.2.3", + "junit" % "junit" % "4.12" % Test, + "com.novocode" % "junit-interface" % "0.11" % Test, + "org.mockito" % "mockito-all" % "1.10.19" % Test, + "org.scalatest" %% "scalatest" % "3.0.3" % Test + ) + ) diff --git a/publish.sh b/publish.sh index de9f7b1..703c624 100755 --- a/publish.sh +++ b/publish.sh @@ -5,6 +5,7 @@ set -ex sbt "project coreUtils" +publishSigned sbt "project sparkUtils" +publishSigned sbt "project storeUtils" +publishSigned +sbt "project gocdUtils" +publishSigned sbt sonatypeReleaseAll echo "Released" \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala new file mode 100644 index 0000000..6b9494a --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala @@ -0,0 +1,25 @@ +package com.indix.gocd.models + +/** + * S3 Artifact + * + * A gocd pipeline can create an s3 artifact at a location that can + * be identified using the pipeline details like pipeline name, + * stage name, job name, etc + * + * @param pipelineName + * @param stageName + * @param jobName + * @param revision + */ +case class Artifact(pipelineName: String, stageName: String, jobName: String, revision: Option[Revision] = None) { + + def withRevision(revision: Revision): Artifact = Artifact(pipelineName, stageName, jobName, Some(revision)) + + def prefix: String = String.format("%s/%s/%s/", pipelineName, stageName, jobName) + + def prefixWithRevision: String = { + if (revision.isDefined) String.format("%s/%s/%s/%s/", pipelineName, stageName, jobName, revision.get.revision) + else prefix + } +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala b/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala new file mode 100644 index 0000000..f906a41 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala @@ -0,0 +1,9 @@ +package com.indix.gocd.models + +object ResponseMetadataConstants { + val TRACEBACK_URL = "traceback_url" + val USER = "user" + val REVISION_COMMENT = "revision_comment" + val COMPLETED = "completed" + val GO_PIPELINE_LABEL = "go_pipeline_label" +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala new file mode 100644 index 0000000..cd4e8c0 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala @@ -0,0 +1,33 @@ +package com.indix.gocd.models + +/** + * Represents revision + * Parses revision string to populate fields + * + * @param revision + */ +case class Revision(revision: String) extends Ordered[Revision] { + + val parts: Seq[String] = revision.split("\\.") + val major: Int = Integer.valueOf(parts.head) + val minor: Int = Integer.valueOf(parts(1)) + val patch: Int = { + if (parts.size == 3) Integer.valueOf(parts(2)) + else 0 + } + + override def compare(that: Revision): Int = { + val majorDiff = this.major.compareTo(that.major) + val minorDiff = this.minor.compareTo(that.minor) + val patchDiff = this.patch.compareTo(that.patch) + + if (majorDiff != 0) majorDiff + else if (minorDiff != 0) minorDiff + else if (patchDiff != 0) patchDiff + else 0 + } +} + +object Revision { + def base: Revision = Revision("0.0.0") +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala new file mode 100644 index 0000000..8e88578 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala @@ -0,0 +1,31 @@ +package com.indix.gocd.models + +import java.text.SimpleDateFormat +import java.util.Date + +import com.amazonaws.util.StringUtils + +/** + * Used to display the status of revisions in gocd + * + * @param revision + * @param lastModified + * @param trackbackUrl + * @param user + * @param revisionLabel + */ +case class RevisionStatus(revision: Revision, lastModified: Date, trackbackUrl: String, user: String, revisionLabel: String = "") { + + val DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" + + def toMap: Map[String, String] = { + Map( + "revision" -> revision.revision, + "timestamp" -> new SimpleDateFormat(DATE_PATTERN).format(lastModified), + "user" -> user, + "revisionComment" -> String.format("Original revision number: %s", if (StringUtils.isNullOrEmpty(revisionLabel)) "unavailable" + else revisionLabel), + "trackbackUrl" -> trackbackUrl + ) + } +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala new file mode 100644 index 0000000..26c3b16 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala @@ -0,0 +1,38 @@ +package com.indix.gocd.utils + +object Constants { + val METADATA_USER = "user" + val METADATA_TRACEBACK_URL = "traceback_url" + val COMPLETED = "completed" + + val GO_ARTIFACTS_S3_BUCKET = "GO_ARTIFACTS_S3_BUCKET" + val GO_SERVER_DASHBOARD_URL = "GO_SERVER_DASHBOARD_URL" + + val SOURCEDESTINATIONS = "sourceDestinations" + val DESTINATION_PREFIX = "destinationPrefix" + val ARTIFACTS_BUCKET = "artifactsBucket" + + val AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" + val AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" + val AWS_REGION = "AWS_REGION" + val AWS_USE_IAM_ROLE = "AWS_USE_IAM_ROLE" + val AWS_STORAGE_CLASS = "AWS_STORAGE_CLASS" + val STORAGE_CLASS_STANDARD = "standard" + val STORAGE_CLASS_STANDARD_IA = "standard-ia" + val STORAGE_CLASS_RRS = "rrs" + val STORAGE_CLASS_GLACIER = "glacier" + + val GO_PIPELINE_LABEL = "GO_PIPELINE_LABEL" + + val MATERIAL_TYPE = "MaterialType" + val REPO = "Repo" + val PACKAGE = "Package" + val MATERIAL = "Material" + val JOB = "Job" + val STAGE = "Stage" + val SOURCE = "Source" + val SOURCE_PREFIX = "SourcePrefix" + val DESTINATION = "Destination" + + val REQUIRED_FIELD_MESSAGE = "This field is required" +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala new file mode 100644 index 0000000..bf451b5 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala @@ -0,0 +1,20 @@ +package com.indix.gocd.utils + +import java.nio.file.Paths + +import com.thoughtworks.go.plugin.api.task.JobConsoleLogger + +import scala.collection.JavaConverters._ + +case class Context(environmentVariables: Map[String, String], workingDir: String, console: JobConsoleLogger) { + + def printMessage(message: String): Unit = console.printLine(message) + + def printEnvironment(): Unit = console.printEnvironment(environmentVariables.asJava) + + def getAbsoluteWorkingDir: String = Paths.get("").toAbsolutePath.resolve(workingDir).toString +} + +object Context { + def apply(context: Map[String, _]): Context = Context(context("environmentVariables").asInstanceOf[Map[String, String]], context("workingDirectory").asInstanceOf[String], JobConsoleLogger.getConsoleLogger) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala new file mode 100644 index 0000000..007d5e2 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala @@ -0,0 +1,85 @@ +package com.indix.gocd.utils + +import java.util.regex.Pattern + +import com.indix.gocd.utils.Constants.{AWS_USE_IAM_ROLE, GO_SERVER_DASHBOARD_URL} +import org.apache.commons.lang3.BooleanUtils +import org.apache.commons.lang3.StringUtils.isNotEmpty + +import scala.collection.JavaConversions._ + +class GoEnvironment(input: Option[Map[String, String]] = None) { + + val environment: Map[String, String] = { + if (input.isDefined) + System.getenv().toMap ++ input.get + else + System.getenv().toMap + } + val envPat: Pattern = Pattern.compile("\\$\\{(\\w+)\\}") + + def putAll(existing: Map[String, String]): GoEnvironment = new GoEnvironment(Some(existing ++ environment)) + + def asMap: Map[String, String] = environment + + def get(name: String): String = environment(name) + + def getOrElse(name: String, defaultValue: String): String = environment.getOrElse(name, defaultValue) + + def has(name: String): Boolean = environment.containsKey(name) && isNotEmpty(get(name)) + + def isAbsent(name: String): Boolean = !has(name) + + def traceBackUrl: String = { + val serverUrl = get(GO_SERVER_DASHBOARD_URL) + val pipelineName = get("GO_PIPELINE_NAME") + val pipelineCounter = get("GO_PIPELINE_COUNTER") + val stageName = get("GO_STAGE_NAME") + val stageCounter = get("GO_STAGE_COUNTER") + val jobName = get("GO_JOB_NAME") + String.format("%s/go/tab/build/detail/%s/%s/%s/%s/%s", serverUrl, pipelineName, pipelineCounter, stageName, stageCounter, jobName) + } + + def triggeredUser: String = get("GO_TRIGGER_USER") + + def replaceVariables(str: String): String = { + val m = envPat.matcher(str) + val sb = new StringBuffer + while (m.find()) { + val replacement = environment.get(m.group(1)) + if (replacement.isDefined) { + m.appendReplacement(sb, replacement.get) + } + } + m.appendTail(sb) + sb.toString + } + + /** + * Version Format on S3 is pipeline/stage/job/pipeline_counter.stage_counter + */ + def artifactsLocationTemplate: String = { + val pipeline = get("GO_PIPELINE_NAME") + val stageName = get("GO_STAGE_NAME") + val jobName = get("GO_JOB_NAME") + val pipelineCounter = get("GO_PIPELINE_COUNTER") + val stageCounter = get("GO_STAGE_COUNTER") + artifactsLocationTemplate(pipeline, stageName, jobName, pipelineCounter, stageCounter) + } + + def artifactsLocationTemplate(pipeline: String, stageName: String, jobName: String, pipelineCounter: String, stageCounter: String): String = String.format("%s/%s/%s/%s.%s", pipeline, stageName, jobName, pipelineCounter, stageCounter) + + private val validUseIamRoleValues: List[String] = List("true", "false", "yes", "no", "on", "off") + + def hasAWSUseIamRole: Boolean = { + if (!has(AWS_USE_IAM_ROLE)) false + val useIamRoleValue = get(AWS_USE_IAM_ROLE) + val result = BooleanUtils.toBooleanObject(useIamRoleValue) + if (result == null) throw new IllegalArgumentException(getEnvInvalidFormatMessage(AWS_USE_IAM_ROLE, useIamRoleValue, validUseIamRoleValues.mkString("[", ", ", "]"))) + else result.booleanValue + } + + private def getEnvInvalidFormatMessage(environmentVariable: String, value: String, expected: String) = String.format("Unexpected value in %s environment variable; was %s, but expected one of the following %s", environmentVariable, value, expected) + + def this() = this(None) +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala new file mode 100644 index 0000000..8e53fce --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala @@ -0,0 +1,22 @@ +package com.indix.gocd.utils + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse + +case class MaterialResult(success: Boolean, messages: Seq[String]) { + + def toMap = Map( + "status" -> { + if (success) "success" else "failure" + }, + "messages" -> messages + ) + + def responseCode: Int = DefaultGoApiResponse.SUCCESS_RESPONSE_CODE +} + +object MaterialResult { + + def apply(success: Boolean) : MaterialResult = MaterialResult(success, Seq()) + + def apply(success: Boolean, message: String) : MaterialResult = MaterialResult(success, Seq(message)) +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala new file mode 100644 index 0000000..0de507f --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala @@ -0,0 +1,10 @@ +package com.indix.gocd.utils + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse + +case class TaskExecutionResult(success: Boolean, message: String, exception: Option[Exception] = None) { + + def toMap = Map("success" -> success, "message" -> message) + + def responseCode: Int = DefaultGoApiResponse.SUCCESS_RESPONSE_CODE +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala new file mode 100644 index 0000000..7af4a2c --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala @@ -0,0 +1,200 @@ +package com.indix.gocd.utils.store + +import java.io.File + +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials, InstanceProfileCredentialsProvider} +import com.amazonaws.services.s3.model._ +import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} +import com.indix.gocd.models.{Artifact, ResponseMetadataConstants, Revision, RevisionStatus} +import com.indix.gocd.utils.Constants._ +import com.indix.gocd.utils.GoEnvironment +import com.indix.gocd.utils.store.S3ArtifactStore.getS3client + +import scala.collection.JavaConversions._ + +/** + * Lists objects, finds latest, gets objects from S3 + * + * @param client + * @param bucket + * @param storageClass + */ +class S3ArtifactStore(client: AmazonS3 = getS3client(new GoEnvironment()), bucket: String, storageClass: StorageClass = StorageClass.Standard) { + + def withStorageClass(storageClass: String): S3ArtifactStore = { + val key = storageClass.toLowerCase() + if (S3ArtifactStore.STORAGE_CLASSES.contains(key)) { + new S3ArtifactStore(client, bucket, S3ArtifactStore.STORAGE_CLASSES(key)) + } + else { + throw new IllegalArgumentException("Invalid storage class specified for S3 - " + storageClass + ". Accepted values are standard, standard-ia, rrs and glacier") + } + } + + /** + * Puts object request in client + * + * @param putObjectRequest + * @return + */ + def put(putObjectRequest: PutObjectRequest): PutObjectResult = { + putObjectRequest.setStorageClass(this.storageClass) + client.putObject(putObjectRequest) + } + + def put(from: String, to: String, metadata: ObjectMetadata): PutObjectResult = { + put(new PutObjectRequest(bucket, to, new File(from)).withMetadata(metadata)) + } + + def put(from: String, to: String): PutObjectResult = { + put(new PutObjectRequest(bucket, to, new File(from))) + } + + def pathString(pathOnS3: String): String = String.format("s3://%s/%s", bucket, pathOnS3) + + /** + * Gets objects from path and puts in the destination + * + * @param from + * @param to + * @return + */ + def get(from: String, to: String): ObjectMetadata = { + val getObjectRequest = new GetObjectRequest(bucket, from) + val destinationFile = new File(to) + destinationFile.getParentFile.mkdirs + client.getObject(getObjectRequest, destinationFile) + } + + def getMetadata(key: String): ObjectMetadata = client.getObjectMetadata(bucket, key) + + def getPrefix(prefix: String, to: String): Unit = { + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix) + getAll(client.listObjects(listObjectsRequest)) + + def getAll(objectListing: ObjectListing): Unit = { + if (objectListing.isTruncated) None + else { + for (objectSummary <- objectListing.getObjectSummaries) { + val destinationPath = to + "/" + objectSummary.getKey.replace(prefix + "/", "") + if (objectSummary.getSize > 0) { + val x = get(objectSummary.getKey, destinationPath) + } + } + listObjectsRequest.setMarker(objectListing.getNextMarker) + getAll(client.listObjects(listObjectsRequest)) + } + } + } + + def bucketExists: Boolean = { + try { + client.listObjects(new ListObjectsRequest(bucket, null, null, null, 0)) + true + } + catch { + case e: Exception => false + } + } + + def exists(bucket: String, key: String): Boolean = { + val listObjectRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key).withDelimiter("/") + try { + val listing = client.listObjects(listObjectRequest) + listing != null && !listing.getCommonPrefixes.isEmpty + } + catch { + case e: Exception => false + } + } + + private def isComplete(prefix: String): Boolean = client.getObjectMetadata(bucket, prefix).getUserMetadata.contains(ResponseMetadataConstants.COMPLETED) + + private def mostRecentRevision(objectListing: ObjectListing): Revision = { + val prefixes = objectListing.getCommonPrefixes.filter(isComplete) + val revisions = prefixes.map(prefix => Revision(prefix.split("/").last)) + if (revisions.isEmpty) Revision.base + else revisions.max + } + + private def latestOfInternal(objectListing: ObjectListing, latestRevision: Revision): Revision = { + if (!objectListing.isTruncated) latestRevision + else { + val objects = client.listNextBatchOfObjects(objectListing) + val mostRecent = mostRecentRevision(objects) + latestOfInternal(objectListing, latestRevision = { + if (latestRevision.compareTo(mostRecent) > 0) latestRevision + else mostRecent + }) + } + } + + private def latestOf(objectListing: ObjectListing) = latestOfInternal(objectListing, mostRecentRevision(objectListing)) + + def getLatest(artifact: Artifact): Option[RevisionStatus] = { + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(artifact.prefix).withDelimiter("/") + + val objectListing = client.listObjects(listObjectsRequest) + if (objectListing != null) { + val recent = latestOf(objectListing) + + val getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, artifact.withRevision(recent).prefixWithRevision) + val metadata = client.getObjectMetadata(getObjectMetadataRequest) + val userMetadata = metadata.getUserMetadata + + val tracebackUrl = userMetadata(ResponseMetadataConstants.TRACEBACK_URL) + val user = userMetadata(ResponseMetadataConstants.USER) + val revisionLabel = userMetadata.getOrDefault(ResponseMetadataConstants.GO_PIPELINE_LABEL, "") + + Some(RevisionStatus(recent, metadata.getLastModified, tracebackUrl, user, revisionLabel)) + } + else None + } + + def getLatestPrefix(pipeline: String, stage: String, job: String, pipelineCounter: String): Option[String] = { + val prefix = String.format("%s/%s/%s/%s.", pipeline, stage, job, pipelineCounter) + + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix).withDelimiter("/") + val objectListing = client.listObjects(listObjectsRequest) + + if (objectListing != null) { + val stageCounters = objectListing.getCommonPrefixes.map(input => input.replaceAll(prefix, "").replaceAll("/", "")) + + if (stageCounters.nonEmpty) Some(prefix + stageCounters.maxBy(counter => Integer.valueOf(counter))) + else None + } + else None + } + + def this(env: GoEnvironment, bucket: String) = this(getS3client(env), bucket, StorageClass.Standard) + + def this(client: AmazonS3, bucket: String) = this(client, bucket, StorageClass.Standard) +} + +object S3ArtifactStore { + val STORAGE_CLASSES = Map( + STORAGE_CLASS_STANDARD -> StorageClass.Standard, + STORAGE_CLASS_STANDARD_IA -> StorageClass.StandardInfrequentAccess, + STORAGE_CLASS_RRS -> StorageClass.ReducedRedundancy, + STORAGE_CLASS_GLACIER -> StorageClass.Glacier + ) + + def getS3client(env: GoEnvironment): AmazonS3 = { + + def withRegion(clientBuilder: AmazonS3ClientBuilder) = { + if (env.has(AWS_REGION)) clientBuilder.withRegion(env.get(AWS_REGION)) + else clientBuilder + } + + def withCredentials(clientBuilder: AmazonS3ClientBuilder) = { + if (env.hasAWSUseIamRole) clientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false)) + else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) { + val basicAWSCredentials = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY)) + clientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) + } + else clientBuilder + } + + withCredentials(withRegion(AmazonS3ClientBuilder.standard())).build() + } +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala new file mode 100644 index 0000000..a161f4a --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala @@ -0,0 +1,15 @@ +package com.indix.gocd.utils.utils + +trait Function[I, O] { + def apply(input: I): O +} + +abstract class VoidFunction[I] extends Function[I, Nothing]{ + def execute(i: I): Nothing + override def apply(input: I): Nothing = execute(input) +} + +abstract class Predicate[T] extends Function[T, Boolean] { + def execute(i: T): Boolean + override def apply(input: T): Boolean = execute(input) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala new file mode 100644 index 0000000..b09148e --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala @@ -0,0 +1,23 @@ +package com.indix.gocd.utils.utils + +import java.util + +import scala.collection.JavaConversions._ + +object Maps { + case class MapBuilder[K, V](internal: Map[K, V]) { + + def _with(k: K, v: V) = MapBuilder(internal ++ Map[K, V](k -> v)) + + def remove(k: K): MapBuilder[K, V] = { + val newInternal = internal.filterNot(tuple => k.equals(tuple._1)) + MapBuilder(newInternal) + } + + def buildJavaMap(): util.Map[K, V] = mapAsJavaMap(internal) + + def build: Map[K, V] = internal + } + + def builder[K, V] = MapBuilder(Map[K, V]()) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala new file mode 100644 index 0000000..5a64126 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala @@ -0,0 +1,6 @@ +package com.indix.gocd.utils.utils + +case class Tuple2[Left, Right](internal: (Left, Right)) { + def _1: Left = internal._1 + def _2: Right = internal._2 +} diff --git a/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala b/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala new file mode 100644 index 0000000..529df65 --- /dev/null +++ b/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala @@ -0,0 +1,71 @@ +package com.indix.gocd.utils + +import com.indix.gocd.utils.Constants.GO_SERVER_DASHBOARD_URL +import org.scalatest.{FlatSpec, Matchers} + +class GoEnvironmentSpec extends FlatSpec with Matchers { + + val mockEnvironmentVariables = Map( + GO_SERVER_DASHBOARD_URL -> "http://go.server:8153", + "GO_SERVER_URL" -> "https://localhost:8154/go", + "GO_PIPELINE_NAME" -> "s3-publish-test", + "GO_PIPELINE_COUNTER" -> "20", + "GO_STAGE_NAME" -> "build-and-publish", + "GO_STAGE_COUNTER" -> "1", + "GO_JOB_NAME" -> "publish", + "GO_TRIGGER_USER" -> "Krishna") + + val goEnvironment = new GoEnvironment(Some(mockEnvironmentVariables)) + + "traceBackUrl" should "generate trace back url" in { + goEnvironment.traceBackUrl should be("http://go.server:8153/go/tab/build/detail/s3-publish-test/20/build-and-publish/1/publish") + } + + "triggeredUser" should "return triggered user" in { + goEnvironment.triggeredUser should be("Krishna") + } + + "artifactsLocationTemplate" should "generate artifacts location template" in { + goEnvironment.artifactsLocationTemplate should be("s3-publish-test/build-and-publish/publish/20.1") + } + + "asMap" should "return as map" in { + mockEnvironmentVariables.foreach(tuple => goEnvironment.get(tuple._1) should be(tuple._2)) + } + + "replaceVariables" should "replace the env variables with respective values in the template string" in { + val template = "COUNT:${GO_STAGE_COUNTER} Name:${GO_STAGE_NAME} COUNT2:${GO_STAGE_COUNTER}" + val replaced = goEnvironment.replaceVariables(template) + replaced should be("COUNT:1 Name:build-and-publish COUNT2:1") + } + + it should "not replace unknown env variables in the template string" in { + val template = "COUNT:${GO_STAGE_COUNTER} ${DOESNT_EXIST}" + val replaced = goEnvironment.replaceVariables(template) + replaced should be("COUNT:1 ${DOESNT_EXIST}") + } + + "hasAWSUseIamRole" should "return true if it is set to true" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "True")) + env.hasAWSUseIamRole should be(true) + } + + it should "return false if it is set to false" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "False")) + env.hasAWSUseIamRole should be(false) + } + + it should "throw exception if the value is not a valid boolean" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "Blue")) + + try { + env.hasAWSUseIamRole + fail("Expected Exception") + } + catch { + case e: Exception => e.getMessage should be + "Unexpected value in AWS_USE_IAM_ROLE environment variable; was Blue, but expected one of " + + "the following [true, false, yes, no, on, off]" + } + } +} diff --git a/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala b/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala new file mode 100644 index 0000000..6eec2e6 --- /dev/null +++ b/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala @@ -0,0 +1,114 @@ +package com.indix.gocd.utils.store + +import java.io.File + +import com.amazonaws.services.s3.AmazonS3Client +import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing, PutObjectRequest} +import org.mockito.Matchers.any +import org.mockito.Mockito.{times, verify, when} +import org.mockito.{ArgumentCaptor, Mock, Mockito} +import org.scalatest.mockito.MockitoSugar +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + +import scala.collection.JavaConversions._ + +class S3ArtifactStoreSpec extends FlatSpec with Matchers with BeforeAndAfterEach { + + @Mock + private val mockClient = MockitoSugar.mock[AmazonS3Client] + private val putCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest]) + private val listingCaptor: ArgumentCaptor[ListObjectsRequest] = ArgumentCaptor.forClass(classOf[ListObjectsRequest]) + private val store = new S3ArtifactStore(mockClient, "foo-bar") + + override def afterEach(): Unit = { + super.afterEach() + Mockito.reset(mockClient) + } + + "put" should "use standard storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("STANDARD") + } + + it should "use standard-ia storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("standard-ia") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("STANDARD_IA") + } + + it should "use reduced redundancy class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("rrs") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("REDUCED_REDUNDANCY") + } + + it should "use glacier storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("glacier") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("GLACIER") + } + + "bucketExists" should "successfully check if bucket exists" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(new ObjectListing) + store.bucketExists should be(true) + } + + it should "return false when bucket does not exist" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenThrow(new RuntimeException("Bucket does not exist")) + try { + store.bucketExists + } + catch { + case e: RuntimeException => e.getMessage should be("Bucket does not exist") + } + } + + "listObjects" should "return the right objects list" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(null) + store.getLatestPrefix("pipeline", "stage", "job", "1") + verify(mockClient).listObjects(listingCaptor.capture()) + + val request = listingCaptor.getValue + request.getBucketName should be("foo-bar") + request.getPrefix should be("pipeline/stage/job/1.") + request.getDelimiter should be("/") + } + + "getLatestPrefix" should "not be defined when object listing is null" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(null) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(false) + } + + it should "not be defined when object listing size is 0" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(new ObjectListing) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(false) + } + + it should "return the latest stage counter from the listing" in { + val listing = new ObjectListing + listing.setCommonPrefixes(List("pipeline/stage/job/1.2", "pipeline/stage/job/1.1", "pipeline/stage/job/1.7")) + + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(listing) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(true) + latestPrefix.get should be("pipeline/stage/job/1.7") + } +}