From 3f254031df5ea064b06d447fb4ef50d7cf8013d7 Mon Sep 17 00:00:00 2001 From: Rucha Date: Wed, 26 Dec 2018 13:55:33 +0530 Subject: [PATCH 1/5] move utils module here from gocd-s3-artifacts plugin --- build.sbt | 22 +- publish.sh | 1 + .../java/com/indix/gocd/models/Artifact.java | 37 +++ .../models/ResponseMetadataConstants.java | 9 + .../java/com/indix/gocd/models/Revision.java | 49 ++++ .../com/indix/gocd/models/RevisionStatus.java | 41 +++ .../java/com/indix/gocd/utils/Constants.java | 38 +++ .../java/com/indix/gocd/utils/Context.java | 39 +++ .../com/indix/gocd/utils/GoEnvironment.java | 124 +++++++++ .../com/indix/gocd/utils/MaterialResult.java | 34 +++ .../indix/gocd/utils/TaskExecutionResult.java | 41 +++ .../gocd/utils/store/S3ArtifactStore.java | 245 ++++++++++++++++++ .../com/indix/gocd/utils/utils/Function.java | 5 + .../com/indix/gocd/utils/utils/Functions.java | 19 ++ .../com/indix/gocd/utils/utils/Lists.java | 66 +++++ .../java/com/indix/gocd/utils/utils/Maps.java | 32 +++ .../com/indix/gocd/utils/utils/Tuple2.java | 39 +++ .../indix/gocd/utils/GoEnvironmentTest.java | 114 ++++++++ .../indix/gocd/utils/mocks/MockContext.java | 28 ++ .../gocd/utils/store/S3ArtifactStoreTest.java | 132 ++++++++++ .../com/indix/gocd/utils/utils/ListsTest.java | 38 +++ 21 files changed, 1151 insertions(+), 2 deletions(-) create mode 100644 util-gocd/src/main/java/com/indix/gocd/models/Artifact.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/models/Revision.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/Constants.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/Context.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java create mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java create mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java create mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java create mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java create mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java diff --git a/build.sbt b/build.sbt index c653152..1be97fb 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ lazy val commonSettings = Seq( organizationHomepage := Some(url("http://www.indix.com")), scalaVersion := "2.11.11", scalacOptions ++= Seq("-encoding", "UTF-8", "-deprecation", "-unchecked"), - javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.7"), + javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.8"), resolvers ++= Seq( "Clojars" at "http://clojars.org/repo", "Concurrent Maven Repo" at "http://conjars.org/repo", @@ -68,7 +68,7 @@ lazy val utils = (project in file(".")). site.addMappingsToSiteDir(mappings in(ScalaUnidoc, packageDoc), "latest/api"), git.remoteRepo := "git@github.com:indix/utils.git" ). - aggregate(coreUtils, storeUtils, sparkUtils) + aggregate(coreUtils, storeUtils, sparkUtils, gocdUtils) lazy val coreUtils = (project in file("util-core")). settings(commonSettings: _*). @@ -118,3 +118,21 @@ lazy val sparkUtils = (project in file("util-spark")). "org.bdgenomics.utils" %% "utils-misc" % "0.2.13" ) ) + +lazy val gocdUtils = (project in file("util-gocd")). + settings(commonSettings: _*). + settings(publishSettings: _*). + settings( + name := "util-gocd", + crossScalaVersions := Seq("2.10.6", "2.11.11"), + libraryDependencies ++= Seq( + "org.apache.commons" % "commons-lang3" % "3.1", + "commons-io" % "commons-io" % "1.3.2", + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.127", + "cd.go.plugin" % "go-plugin-api" % "17.2.0" % Provided, + "com.google.code.gson" % "gson" % "2.2.3", + "junit" % "junit" % "4.12" % Test, + "com.novocode" % "junit-interface" % "0.11" % Test, + "org.mockito" % "mockito-all" % "1.10.19" % Test + ) + ) diff --git a/publish.sh b/publish.sh index de9f7b1..703c624 100755 --- a/publish.sh +++ b/publish.sh @@ -5,6 +5,7 @@ set -ex sbt "project coreUtils" +publishSigned sbt "project sparkUtils" +publishSigned sbt "project storeUtils" +publishSigned +sbt "project gocdUtils" +publishSigned sbt sonatypeReleaseAll echo "Released" \ No newline at end of file diff --git a/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java b/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java new file mode 100644 index 0000000..dcd83d5 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java @@ -0,0 +1,37 @@ +package com.indix.gocd.models; + +public class Artifact { + String pipelineName; + String stageName; + String jobName; + Revision revision; + + public Artifact(String pipelineName, String stageName, String jobName) { + this.pipelineName = pipelineName; + this.stageName = stageName; + this.jobName = jobName; + } + + public Artifact(String pipelineName, String stageName, String jobName, Revision revision) { + this.pipelineName = pipelineName; + this.stageName = stageName; + this.jobName = jobName; + this.revision = revision; + } + + public Artifact withRevision(Revision revision) { + this.revision = revision; + return this; + } + + public String prefix(){ + return String.format("%s/%s/%s/", pipelineName, stageName, jobName); + } + + public String prefixWithRevision(){ + if(revision != null) + return String.format("%s/%s/%s/%s/", pipelineName, stageName, jobName, revision.getRevision()); + else + return prefix(); + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java b/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java new file mode 100644 index 0000000..cdeec55 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java @@ -0,0 +1,9 @@ +package com.indix.gocd.models; + +public class ResponseMetadataConstants { + public static final String TRACEBACK_URL = "traceback_url"; + public static final String USER = "user"; + public static final String REVISION_COMMENT = "revision_comment"; + public static final String COMPLETED = "completed"; + public static final String GO_PIPELINE_LABEL = "go_pipeline_label"; +} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/Revision.java b/util-gocd/src/main/java/com/indix/gocd/models/Revision.java new file mode 100644 index 0000000..9b2829c --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/models/Revision.java @@ -0,0 +1,49 @@ +package com.indix.gocd.models; + +public class Revision implements Comparable { + private String revision; + private String[] parts; + private Integer major; + private Integer minor; + private Integer patch; + + public Revision(String revision) { + this.revision = revision; + this.parts = revision.split("\\."); + this.major = Integer.valueOf(parts[0]); + this.minor = Integer.valueOf(parts[1]); + if (parts.length == 3) { + this.patch = Integer.valueOf(parts[2]); + } else { + this.patch = 0; + } + } + + public static Revision base() { + return new Revision("0.0.0"); + } + + @Override + public int compareTo(Object otherInstance) { + if(! (otherInstance instanceof Revision)) + throw new RuntimeException("Cannot compare a non-Revision type with Revision type"); + Revision that = (Revision)otherInstance; + int majorDiff = this.major.compareTo(that.major); + int minorDiff = this.minor.compareTo(that.minor); + int patchDiff = this.patch.compareTo(that.patch); + + if(majorDiff != 0) + return majorDiff; + else if(minorDiff != 0) + return minorDiff; + else if(patchDiff != 0) + return patchDiff; + else + return 0; + } + + public String getRevision() { + return revision; + } + +} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java b/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java new file mode 100644 index 0000000..4fd38a7 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java @@ -0,0 +1,41 @@ +package com.indix.gocd.models; + +import com.amazonaws.util.StringUtils; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class RevisionStatus { + public Revision revision; + public Date lastModified; + public String tracebackUrl; + public String user; + public String revisionLabel; + private static final String DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + + public RevisionStatus(Revision revision, Date lastModified, String tracebackUrl, String user) { + this(revision, lastModified, tracebackUrl, user, ""); + } + + public RevisionStatus(Revision revision, Date lastModified, String tracebackUrl, String user, String revisionLabel) { + this.revision = revision; + this.lastModified = lastModified; + this.tracebackUrl = tracebackUrl; + this.user = user; + this.revisionLabel = revisionLabel; + } + + public Map toMap() { + final HashMap result = new HashMap(); + result.put("revision", revision.getRevision()); + result.put("timestamp", new SimpleDateFormat(DATE_PATTERN).format(lastModified)); + result.put("user", user); + result.put("revisionComment", String.format("Original revision number: %s", + StringUtils.isNullOrEmpty(revisionLabel) ? "unavailable" : revisionLabel)); + result.put("trackbackUrl", tracebackUrl); + + return result; + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java b/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java new file mode 100644 index 0000000..25ecd82 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java @@ -0,0 +1,38 @@ +package com.indix.gocd.utils; + +public class Constants { + public static final String METADATA_USER = "user"; + public static final String METADATA_TRACEBACK_URL = "traceback_url"; + public static final String COMPLETED = "completed"; + + public static final String GO_ARTIFACTS_S3_BUCKET = "GO_ARTIFACTS_S3_BUCKET"; + public static final String GO_SERVER_DASHBOARD_URL = "GO_SERVER_DASHBOARD_URL"; + + public static final String SOURCEDESTINATIONS = "sourceDestinations"; + public static final String DESTINATION_PREFIX = "destinationPrefix"; + public static final String ARTIFACTS_BUCKET = "artifactsBucket"; + + public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"; + public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"; + public static final String AWS_REGION = "AWS_REGION"; + public static final String AWS_USE_IAM_ROLE = "AWS_USE_IAM_ROLE"; + public static final String AWS_STORAGE_CLASS = "AWS_STORAGE_CLASS"; + public static final String STORAGE_CLASS_STANDARD = "standard"; + public static final String STORAGE_CLASS_STANDARD_IA = "standard-ia"; + public static final String STORAGE_CLASS_RRS = "rrs"; + public static final String STORAGE_CLASS_GLACIER = "glacier"; + + public static final String GO_PIPELINE_LABEL = "GO_PIPELINE_LABEL"; + + public static final String MATERIAL_TYPE = "MaterialType"; + public static final String REPO = "Repo"; + public static final String PACKAGE = "Package"; + public static final String MATERIAL = "Material"; + public static final String JOB = "Job"; + public static final String STAGE = "Stage"; + public static final String SOURCE = "Source"; + public static final String SOURCE_PREFIX = "SourcePrefix"; + public static final String DESTINATION = "Destination"; + + public static final String REQUIRED_FIELD_MESSAGE = "This field is required"; +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/Context.java b/util-gocd/src/main/java/com/indix/gocd/utils/Context.java new file mode 100644 index 0000000..fe45405 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/Context.java @@ -0,0 +1,39 @@ +package com.indix.gocd.utils; + + +import com.thoughtworks.go.plugin.api.task.JobConsoleLogger; + +import java.nio.file.Paths; +import java.util.Map; + +public class Context { + private final Map environmentVariables; + private final String workingDir; + private final JobConsoleLogger console; + + public Context(Map context) { + environmentVariables = (Map) context.get("environmentVariables"); + workingDir = (String) context.get("workingDirectory"); + console = new JobConsoleLogger() {}; + } + + public void printMessage(String message) { + console.printLine(message); + } + + public void printEnvironment() { + console.printEnvironment(environmentVariables); + } + + public Map getEnvironmentVariables() { + return environmentVariables; + } + + public String getWorkingDir() { + return workingDir; + } + + public String getAbsoluteWorkingDir() { + return Paths.get("").toAbsolutePath().resolve(workingDir).toString(); + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java b/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java new file mode 100644 index 0000000..f4680f5 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java @@ -0,0 +1,124 @@ +package com.indix.gocd.utils; + +import org.apache.commons.lang3.BooleanUtils; + +import java.lang.StringBuffer; + +import java.util.*; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.indix.gocd.utils.Constants.*; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + +/** + * Wrapper around Go's Environment variables + */ +public class GoEnvironment { + private Pattern envPat = Pattern.compile("\\$\\{(\\w+)\\}"); + private Map environment = new HashMap<>(); + + public GoEnvironment() { + this.environment.putAll(System.getenv()); + } + + public GoEnvironment(Map defaultEnvironment) { + this(); + this.environment.putAll(defaultEnvironment); + } + + public GoEnvironment putAll(Map existing) { + environment.putAll(existing); + return this; + } + + public Map asMap() { return environment; } + + public String get(String name) { + return environment.get(name); + } + + public String getOrElse(String name, String defaultValue) { + if(has(name)) return get(name); + else return defaultValue; + } + + public boolean has(String name) { + return environment.containsKey(name) && isNotEmpty(get(name)); + } + + public boolean isAbsent(String name) { + return !has(name); + } + + public String traceBackUrl() { + String serverUrl = get(GO_SERVER_DASHBOARD_URL); + String pipelineName = get("GO_PIPELINE_NAME"); + String pipelineCounter = get("GO_PIPELINE_COUNTER"); + String stageName = get("GO_STAGE_NAME"); + String stageCounter = get("GO_STAGE_COUNTER"); + String jobName = get("GO_JOB_NAME"); + return String.format("%s/go/tab/build/detail/%s/%s/%s/%s/%s", serverUrl, pipelineName, pipelineCounter, stageName, stageCounter, jobName); + } + + public String triggeredUser() { + return get("GO_TRIGGER_USER"); + } + + public String replaceVariables(String str) { + Matcher m = envPat.matcher(str); + + StringBuffer sb = new StringBuffer(); + while (m.find()) { + String replacement = get(m.group(1)); + if(replacement != null) { + m.appendReplacement(sb, replacement); + } + } + + m.appendTail(sb); + + return sb.toString(); + } + + /** + * Version Format on S3 is pipeline/stage/job/pipeline_counter.stage_counter + */ + public String artifactsLocationTemplate() { + String pipeline = get("GO_PIPELINE_NAME"); + String stageName = get("GO_STAGE_NAME"); + String jobName = get("GO_JOB_NAME"); + + String pipelineCounter = get("GO_PIPELINE_COUNTER"); + String stageCounter = get("GO_STAGE_COUNTER"); + return artifactsLocationTemplate(pipeline, stageName, jobName, pipelineCounter, stageCounter); + } + + public String artifactsLocationTemplate(String pipeline, String stageName, String jobName, String pipelineCounter, String stageCounter) { + return String.format("%s/%s/%s/%s.%s", pipeline, stageName, jobName, pipelineCounter, stageCounter); + } + + private static final List validUseIamRoleValues = new ArrayList(Arrays.asList("true", "false", "yes", "no", "on", "off")); + public boolean hasAWSUseIamRole() { + if (!has(AWS_USE_IAM_ROLE)) { + return false; + } + + String useIamRoleValue = get(AWS_USE_IAM_ROLE); + Boolean result = BooleanUtils.toBooleanObject(useIamRoleValue); + if (result == null) { + throw new IllegalArgumentException(getEnvInvalidFormatMessage(AWS_USE_IAM_ROLE, + useIamRoleValue, validUseIamRoleValues.toString())); + } + else { + return result.booleanValue(); + } + } + + private String getEnvInvalidFormatMessage(String environmentVariable, String value, String expected){ + return String.format( + "Unexpected value in %s environment variable; was %s, but expected one of the following %s", + environmentVariable, value, expected); + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java b/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java new file mode 100644 index 0000000..f6d62d5 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java @@ -0,0 +1,34 @@ +package com.indix.gocd.utils; + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MaterialResult { + private boolean success; + private List messages; + + public MaterialResult(boolean success) { + this.success = success; + messages = new ArrayList<>(); + } + + public MaterialResult(boolean success, String message) { + this(success); + messages.add(message); + } + + public Map toMap() { + final HashMap result = new HashMap(); + result.put("status", success ? "success" : "failure"); + result.put("messages", messages); + return result; + } + + public int responseCode() { + return DefaultGoApiResponse.SUCCESS_RESPONSE_CODE; + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java b/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java new file mode 100644 index 0000000..30852a6 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java @@ -0,0 +1,41 @@ +package com.indix.gocd.utils; + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse; + +import java.util.HashMap; +import java.util.Map; + +public class TaskExecutionResult { + private boolean success; + private String message; + private Exception exception; + + public TaskExecutionResult(boolean success, String message) { + this.success = success; + this.message = message; + } + + public TaskExecutionResult(boolean success, String message, Exception exception) { + this(success, message); + this.exception = exception; + } + + public Map toMap() { + final HashMap result = new HashMap(); + result.put("success", success); + result.put("message", message); + return result; + } + + public int responseCode() { + return DefaultGoApiResponse.SUCCESS_RESPONSE_CODE; + } + + public boolean isSuccessful() { + return success; + } + + public String message() { + return message; + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java b/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java new file mode 100644 index 0000000..ac8bbd7 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java @@ -0,0 +1,245 @@ +package com.indix.gocd.utils.store; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.*; +import com.indix.gocd.models.Artifact; +import com.indix.gocd.models.ResponseMetadataConstants; +import com.indix.gocd.models.Revision; +import com.indix.gocd.models.RevisionStatus; +import com.indix.gocd.utils.GoEnvironment; +import com.indix.gocd.utils.utils.Function; +import com.indix.gocd.utils.utils.Functions; +import com.indix.gocd.utils.utils.Lists; +import com.indix.gocd.utils.utils.Maps; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static com.indix.gocd.utils.Constants.*; + +public class S3ArtifactStore { + + private static Map STORAGE_CLASSES = Maps.builder() + .with(STORAGE_CLASS_STANDARD, StorageClass.Standard) + .with(STORAGE_CLASS_STANDARD_IA, StorageClass.StandardInfrequentAccess) + .with(STORAGE_CLASS_RRS, StorageClass.ReducedRedundancy) + .with(STORAGE_CLASS_GLACIER, StorageClass.Glacier) + .build(); + + private AmazonS3 client; + private String bucket; + private StorageClass storageClass = StorageClass.Standard; + + public S3ArtifactStore(AmazonS3 client, String bucket) { + this.client = client; + this.bucket = bucket; + } + + public S3ArtifactStore(GoEnvironment env, String bucket) { + this(getS3client(env), bucket); + } + + public S3ArtifactStore(String bucket) { + this(getS3client(new GoEnvironment()), bucket); + } + + public void setStorageClass(String storageClass) { + String key = StringUtils.lowerCase(storageClass); + if (STORAGE_CLASSES.containsKey(key)) { + this.storageClass = STORAGE_CLASSES.get(key); + } else { + throw new IllegalArgumentException("Invalid storage class specified for S3 - " + storageClass + ". Accepted values are standard, standard-ia, rrs and glacier"); + } + } + + public void put(String from, String to) { + put(new PutObjectRequest(bucket, to, new File(from))); + } + + public void put(String from, String to, ObjectMetadata metadata) { + put(new PutObjectRequest(bucket, to, new File(from)) + .withMetadata(metadata)); + } + + public void put(PutObjectRequest putObjectRequest) { + putObjectRequest.setStorageClass(this.storageClass); + client.putObject(putObjectRequest); + } + + public String pathString(String pathOnS3) { + return String.format("s3://%s/%s", bucket, pathOnS3); + } + + public void get(String from, String to) { + GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, from); + File destinationFile = new File(to); + destinationFile.getParentFile().mkdirs(); + client.getObject(getObjectRequest, destinationFile); + } + + public ObjectMetadata getMetadata(String key) { + return client.getObjectMetadata(bucket, key); + } + + public void getPrefix(String prefix, String to) { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucket) + .withPrefix(prefix); + + ObjectListing objectListing; + do { + objectListing = client.listObjects(listObjectsRequest); + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + String destinationPath = to + "/" + objectSummary.getKey().replace(prefix + "/", ""); + long size = objectSummary.getSize(); + if (size > 0) { + get(objectSummary.getKey(), destinationPath); + } + } + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } while (objectListing.isTruncated()); + + } + + public boolean bucketExists() { + try { + client.listObjects(new ListObjectsRequest(bucket, null, null, null, 0)); + return true; + } catch (Exception ex) { + return false; + } + } + + public boolean exists(String bucket, String key) { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucket) + .withPrefix(key) + .withDelimiter("/"); + try { + ObjectListing objectListing = client.listObjects(listObjectsRequest); + return objectListing != null && objectListing.getCommonPrefixes().size() > 0; + } catch (Exception ex) { + return false; + } + } + + private Boolean isComplete(String prefix) { + return client.getObjectMetadata(bucket, prefix).getUserMetadata().containsKey(ResponseMetadataConstants.COMPLETED); + } + + private Revision mostRecentRevision(ObjectListing listing) { + List prefixes = Lists.filter(listing.getCommonPrefixes(), new Functions.Predicate() { + @Override + public Boolean execute(String input) { + return isComplete(input); + } + }); + + List revisions = Lists.map(prefixes, new Function() { + @Override + public Revision apply(String prefix) { + String[] parts = prefix.split("/"); + String last = parts[parts.length - 1]; + return new Revision(last); + } + }); + + if (revisions.size() > 0) + return Collections.max(revisions); + else + return Revision.base(); + } + + private Revision latestOfInternal(ObjectListing listing, Revision latestSoFar) { + if (!listing.isTruncated()) { + return latestSoFar; + } else { + ObjectListing objects = client.listNextBatchOfObjects(listing); + Revision mostRecent = mostRecentRevision(objects); + if (latestSoFar.compareTo(mostRecent) > 0) + mostRecent = latestSoFar; + return latestOfInternal(objects, mostRecent); + } + } + + private Revision latestOf(ObjectListing listing) { + return latestOfInternal(listing, mostRecentRevision(listing)); + } + + public RevisionStatus getLatest(Artifact artifact) { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucket) + .withPrefix(artifact.prefix()) + .withDelimiter("/"); + + ObjectListing listing = client.listObjects(listObjectsRequest); + if (listing != null) { + Revision recent = latestOf(listing); + Artifact artifactWithRevision = artifact.withRevision(recent); + GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, artifactWithRevision.prefixWithRevision()); + ObjectMetadata metadata = client.getObjectMetadata(objectMetadataRequest); + Map userMetadata = metadata.getUserMetadata(); + String tracebackUrl = userMetadata.get(ResponseMetadataConstants.TRACEBACK_URL); + String user = userMetadata.get(ResponseMetadataConstants.USER); + String revisionLabel = userMetadata.containsKey(ResponseMetadataConstants.GO_PIPELINE_LABEL) ? + userMetadata.get(ResponseMetadataConstants.GO_PIPELINE_LABEL) + : ""; + return new RevisionStatus(recent, metadata.getLastModified(), tracebackUrl, user, revisionLabel); + } + return null; + } + + public String getLatestPrefix(String pipeline, String stage, String job, String pipelineCounter) { + String prefix = String.format("%s/%s/%s/%s.", pipeline, stage, job, pipelineCounter); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucket) + .withPrefix(prefix) + .withDelimiter("/"); + + ObjectListing listing = client.listObjects(listObjectsRequest); + + if (listing != null) { + List commonPrefixes = listing.getCommonPrefixes(); + List stageCounters = Lists.map(commonPrefixes, + input -> + input.replaceAll(prefix, "").replaceAll("/", "")); + if (stageCounters.size() > 0) { + int maxStageCounter = Integer.valueOf(stageCounters.get(0)); + + for (int i = 1; i < stageCounters.size(); i++) { + int stageCounter = Integer.valueOf(stageCounters.get(i)); + if (stageCounter > maxStageCounter) { + maxStageCounter = stageCounter; + } + } + + return prefix + maxStageCounter; + } + } + return null; + } + + public static AmazonS3 getS3client(GoEnvironment env) { + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard(); + + if (env.has(AWS_REGION)) { + amazonS3ClientBuilder.withRegion(env.get(AWS_REGION)); + } + if (env.hasAWSUseIamRole()) { + amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false)); + } else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) { + BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY)); + amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds)); + } + + return amazonS3ClientBuilder.build(); + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java new file mode 100644 index 0000000..e0820b8 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java @@ -0,0 +1,5 @@ +package com.indix.gocd.utils.utils; + +public interface Function { + public O apply(I input); +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java new file mode 100644 index 0000000..fd96e16 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java @@ -0,0 +1,19 @@ +package com.indix.gocd.utils.utils; + +public class Functions { + public static abstract class VoidFunction implements Function { + public abstract void execute(I input); + + public Void apply(I input) { + execute(input); + return null; + } + } + + public static abstract class Predicate implements Function { + public abstract Boolean execute(T input); + public Boolean apply(T input){ + return execute(input); + } + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java new file mode 100644 index 0000000..787e8cd --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java @@ -0,0 +1,66 @@ +package com.indix.gocd.utils.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class Lists { + + public static void foreach(List originalList, Function function) { + for (T item : originalList) { + function.apply(item); + } + } + + public static void foreach(T[] array, Function function) { + foreach(Arrays.asList(array), function); + } + + public static List flatMap(List originalList, Function> transformer) { + ArrayList flatMapped = new ArrayList(); + for (T item : originalList) { + flatMapped.addAll(transformer.apply(item)); + } + return flatMapped; + } + + public static List flatMap(T[] array, Function> transformer) { + return flatMap(Arrays.asList(array), transformer); + } + + public static List of(T... items) { + ArrayList listToReturn = new ArrayList(); + Collections.addAll(listToReturn, items); + return Collections.unmodifiableList(listToReturn); + } + + public static List map(T[] array, Function transformer) { + return map(Arrays.asList(array), transformer); + } + + public static List map(List elements, Function transformer) { + List mapped = new ArrayList(); + for (T item: elements) { + mapped.add(transformer.apply(item)); + } + return mapped; + } + + public static List filter(List elements, Functions.Predicate predicate) { + List filtered = new ArrayList(); + for(T element: elements){ + if(predicate.execute(element)) + filtered.add(element); + } + return filtered; + } + + public static Boolean exists(List elements, Functions.Predicate predicate) { + for (T element: elements) { + if(predicate.execute(element)) + return true; + } + return false; + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java new file mode 100644 index 0000000..dcb258a --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java @@ -0,0 +1,32 @@ +package com.indix.gocd.utils.utils; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class Maps { + public static MapBuilder builder() { + return new MapBuilder(); + } + public static boolean isEmpty(Map map) { + return map.isEmpty(); + } + + public static class MapBuilder { + private Map internal = new HashMap(); + + public MapBuilder with(K key, V value) { + internal.put(key, value); + return this; + } + + public MapBuilder remove(K key) { + internal.remove(key); + return this; + } + + public Map build() { + return Collections.unmodifiableMap(internal); + } + } +} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java new file mode 100644 index 0000000..eff9ec6 --- /dev/null +++ b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java @@ -0,0 +1,39 @@ +package com.indix.gocd.utils.utils; + +public class Tuple2 { + private Left _1; + private Right _2; + + public Tuple2(Left _1, Right _2) { + this._1 = _1; + this._2 = _2; + } + + public Left _1() { + return _1; + } + + public Right _2() { + return _2; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Tuple2 tuple2 = (Tuple2) o; + + if (!_1.equals(tuple2._1)) return false; + if (!_2.equals(tuple2._2)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = _1.hashCode(); + result = 31 * result + _2.hashCode(); + return result; + } +} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java new file mode 100644 index 0000000..09161ed --- /dev/null +++ b/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java @@ -0,0 +1,114 @@ +package com.indix.gocd.utils; + +import static com.indix.gocd.utils.Constants.AWS_USE_IAM_ROLE; +import static org.hamcrest.CoreMatchers.is; + +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.util.HashMap; +import java.util.Map; +import com.indix.gocd.utils.utils.Maps; +import static com.indix.gocd.utils.Constants.GO_SERVER_DASHBOARD_URL; +import static org.junit.Assert.fail; + + +public class GoEnvironmentTest { + private GoEnvironment goEnvironment; + Maps.MapBuilder mockEnvironmentVariables; + + @Before + public void setUp() throws Exception { + mockEnvironmentVariables = Maps.builder() + .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153") + .with("GO_SERVER_URL", "https://localhost:8154/go") + .with("GO_PIPELINE_NAME", "s3-publish-test") + .with("GO_PIPELINE_COUNTER", "20") + .with("GO_STAGE_NAME", "build-and-publish") + .with("GO_STAGE_COUNTER", "1") + .with("GO_JOB_NAME", "publish") + .with("GO_TRIGGER_USER", "Krishna"); + goEnvironment = new GoEnvironment(new HashMap()).putAll(mockEnvironmentVariables.build()); + } + + @Test + public void shouldGenerateTracebackUrl() { + assertThat(goEnvironment.traceBackUrl(), is("http://go.server:8153/go/tab/build/detail/s3-publish-test/20/build-and-publish/1/publish")); + } + + @Test + public void shouldReturnTriggeredUser() { + assertThat(goEnvironment.triggeredUser(), is("Krishna")); + } + + @Test + public void shouldGenerateArtifactLocationTemplate() { + assertThat(goEnvironment.artifactsLocationTemplate(), is("s3-publish-test/build-and-publish/publish/20.1")); + } + + @Test + public void shouldReturnAsMap() { + for(Map.Entry entry : mockEnvironmentVariables.build().entrySet()) { + assertEquals(entry.getValue(), goEnvironment.asMap().get(entry.getKey())); + } + } + + @Test + public void shouldReplaceWithEnvVariables() { + final String envTestTemplate = "COUNT:${GO_STAGE_COUNTER} Name:${GO_STAGE_NAME} COUNT2:${GO_STAGE_COUNTER}"; + final String replaced = goEnvironment.replaceVariables(envTestTemplate); + + assertThat(replaced, is("COUNT:1 Name:build-and-publish COUNT2:1")); + } + + @Test + public void shouldNotReplaceUnknownEnvVariables() { + final String envTestTemplate = "COUNT:${GO_STAGE_COUNTER} ${DOESNT_EXIST}"; + final String replaced = goEnvironment.replaceVariables(envTestTemplate); + + assertThat(replaced, is("COUNT:1 ${DOESNT_EXIST}")); + } + + @Test + public void shouldGetHasAWSUseIamRoleTrueIfSetToTrue() { + GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables + .with(AWS_USE_IAM_ROLE,"True") + .build()); + + Boolean result = sut.hasAWSUseIamRole(); + + assertThat(result, is(Boolean.TRUE)); + } + + @Test + public void shouldGetHasAWSUseIamRoleFalseIfSetToFalse() { + GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables + .with(AWS_USE_IAM_ROLE,"False") + .build()); + + Boolean result = sut.hasAWSUseIamRole(); + + assertThat(result, is(Boolean.FALSE)); + } + + + @Test + public void shouldThrowExceptionIfAWSUseIamRoleNotWithinExpectedValues() { + GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables + .with(AWS_USE_IAM_ROLE,"blue") + .build()); + + try { + sut.hasAWSUseIamRole(); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals( + "Unexpected value in AWS_USE_IAM_ROLE environment variable; was blue, " + + "but expected one of the following [true, false, yes, no, on, off]", + e.getMessage()); + } + + } +} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java b/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java new file mode 100644 index 0000000..aa181ed --- /dev/null +++ b/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java @@ -0,0 +1,28 @@ +package com.indix.gocd.utils.mocks; + +import com.indix.gocd.utils.Context; +import com.thoughtworks.go.plugin.api.task.Console; +import com.thoughtworks.go.plugin.api.task.EnvironmentVariables; +import com.thoughtworks.go.plugin.api.task.TaskExecutionContext; +import org.apache.commons.lang3.StringUtils; + +import java.io.InputStream; +import java.util.Map; + +public class MockContext extends Context { + + public MockContext(Map contextMap) { + super(contextMap); + } + + @Override + public void printMessage(String message) { + + } + + @Override + public void printEnvironment() { + + } + +} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java new file mode 100644 index 0000000..c85e5ca --- /dev/null +++ b/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java @@ -0,0 +1,132 @@ +package com.indix.gocd.utils.store; + +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.indix.gocd.utils.GoEnvironment; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.indix.gocd.utils.Constants.AWS_ACCESS_KEY_ID; +import static com.indix.gocd.utils.Constants.AWS_REGION; +import static com.indix.gocd.utils.Constants.AWS_SECRET_ACCESS_KEY; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.*; + +public class S3ArtifactStoreTest { + + AmazonS3Client mockClient = mock(AmazonS3Client.class); + ArgumentCaptor putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); + ArgumentCaptor listingCaptor = ArgumentCaptor.forClass(ListObjectsRequest.class); + + @Test + public void shouldUseStandardStorageClassAsDefault() { + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); + verify(mockClient, times(1)).putObject(putCaptor.capture()); + PutObjectRequest putRequest = putCaptor.getValue(); + assertThat(putRequest.getStorageClass(), is("STANDARD")); + } + + @Test + public void shouldUseStandardIAStorageClassAsDefault() { + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.setStorageClass("standard-ia"); + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); + verify(mockClient, times(1)).putObject(putCaptor.capture()); + PutObjectRequest putRequest = putCaptor.getValue(); + assertThat(putRequest.getStorageClass(), is("STANDARD_IA")); + } + + @Test + public void shouldUseReducedRedundancyStorageClass() { + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.setStorageClass("rrs"); + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); + verify(mockClient, times(1)).putObject(putCaptor.capture()); + PutObjectRequest putRequest = putCaptor.getValue(); + assertThat(putRequest.getStorageClass(), is("REDUCED_REDUNDANCY")); + } + + @Test + public void shouldUseGlacierStorageClass() { + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.setStorageClass("glacier"); + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); + verify(mockClient, times(1)).putObject(putCaptor.capture()); + PutObjectRequest putRequest = putCaptor.getValue(); + assertThat(putRequest.getStorageClass(), is("GLACIER")); + } + + @Test + public void shouldSuccessfullyCheckIfBucketExists() { + doReturn(new ObjectListing()).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + assertThat(store.bucketExists(), is(true)); + } + + @Test + public void shouldHandleBucketDoesNotExists() { + doThrow(new RuntimeException("Bucket does not exist")).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + assertThat(store.bucketExists(), is(false)); + } + + @Test + public void verifyObjectListingRequestIsRight() { + doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + store.getLatestPrefix("pipeline", "stage", "job", "1"); + + verify(mockClient).listObjects(listingCaptor.capture()); + ListObjectsRequest request = listingCaptor.getValue(); + assertEquals("foo-bar", request.getBucketName()); + assertEquals("pipeline/stage/job/1.", request.getPrefix()); + assertEquals("/", request.getDelimiter()); + } + + @Test + public void shouldReturnNullWhenObjectListingIsNull() { + doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertNull(prefix); + } + + @Test + public void shouldReturnNullWhenObjectListingIsSize0() { + ObjectListing listing = new ObjectListing(); + doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertNull(prefix); + } + + @Test + public void shouldReturnTheLatestStageCounter() { + ObjectListing listing = new ObjectListing(); + List commonPrefixes = new ArrayList<>(); + commonPrefixes.add("pipeline/stage/job/1.2"); + commonPrefixes.add("pipeline/stage/job/1.1"); + commonPrefixes.add("pipeline/stage/job/1.7"); + listing.setCommonPrefixes(commonPrefixes); + + doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); + S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); + + String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); + assertEquals("pipeline/stage/job/1.7", prefix); + } +} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java new file mode 100644 index 0000000..5de101a --- /dev/null +++ b/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java @@ -0,0 +1,38 @@ +package com.indix.gocd.utils.utils; + +import org.junit.Test; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +import static com.indix.gocd.utils.utils.Lists.*; +import static com.indix.gocd.utils.utils.Functions.VoidFunction; + +public class ListsTest { + @Test + public void shouldRunForEachOnceForEveryElement() { + final int[] sum = {0}; + List list = Lists.of(1, 2, 3, 4, 5); + foreach(list, new VoidFunction() { + @Override + public void execute(Integer input) { + sum[0] += (input * 2); + } + }); + + assertThat(sum[0], is(30)); + } + + @Test + public void shouldTransformEveryElementInTheListAndFlatten() { + List duplicateNumbers = flatMap(Lists.of(1, 2, 3, 4, 5), new Function>() { + @Override + public List apply(Integer input) { + return Lists.of(input, input * 2); + } + }); + + assertThat(duplicateNumbers, is(Lists.of(1, 2, 2, 4, 3, 6, 4, 8, 5, 10))); + } +} From d5adc63870957395004e7738112d9dac3e4e143a Mon Sep 17 00:00:00 2001 From: Rucha Date: Wed, 2 Jan 2019 10:36:56 +0530 Subject: [PATCH 2/5] rewrite the module in scala --- build.sbt | 5 +- .../java/com/indix/gocd/models/Artifact.java | 37 --- .../models/ResponseMetadataConstants.java | 9 - .../java/com/indix/gocd/models/Revision.java | 49 ---- .../com/indix/gocd/models/RevisionStatus.java | 41 --- .../java/com/indix/gocd/utils/Constants.java | 38 --- .../java/com/indix/gocd/utils/Context.java | 39 --- .../com/indix/gocd/utils/GoEnvironment.java | 124 --------- .../com/indix/gocd/utils/MaterialResult.java | 34 --- .../indix/gocd/utils/TaskExecutionResult.java | 41 --- .../gocd/utils/store/S3ArtifactStore.java | 245 ------------------ .../com/indix/gocd/utils/utils/Function.java | 5 - .../com/indix/gocd/utils/utils/Functions.java | 19 -- .../com/indix/gocd/utils/utils/Lists.java | 66 ----- .../java/com/indix/gocd/utils/utils/Maps.java | 32 --- .../com/indix/gocd/utils/utils/Tuple2.java | 39 --- .../com/indix/gocd/models/Artifact.scala | 25 ++ .../models/ResponseMetadataConstants.scala | 9 + .../com/indix/gocd/models/Revision.scala | 37 +++ .../indix/gocd/models/RevisionStatus.scala | 22 ++ .../com/indix/gocd/utils/Constants.scala | 38 +++ .../scala/com/indix/gocd/utils/Context.scala | 22 ++ .../com/indix/gocd/utils/GoEnvironment.scala | 85 ++++++ .../com/indix/gocd/utils/MaterialResult.scala | 22 ++ .../gocd/utils/TaskExecutionResult.scala | 10 + .../gocd/utils/store/S3ArtifactStore.scala | 180 +++++++++++++ .../indix/gocd/utils/utils/Functions.scala | 15 ++ .../com/indix/gocd/utils/utils/Maps.scala | 23 ++ .../com/indix/gocd/utils/utils/Tuple2.scala | 6 + .../indix/gocd/utils/GoEnvironmentTest.java | 114 -------- .../indix/gocd/utils/mocks/MockContext.java | 28 -- .../gocd/utils/store/S3ArtifactStoreTest.java | 132 ---------- .../com/indix/gocd/utils/utils/ListsTest.java | 38 --- .../indix/gocd/utils/GoEnvironmentSpec.scala | 71 +++++ .../utils/store/S3ArtifactStoreSpec.scala | 114 ++++++++ 35 files changed, 682 insertions(+), 1132 deletions(-) delete mode 100644 util-gocd/src/main/java/com/indix/gocd/models/Artifact.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/models/Revision.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/Constants.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/Context.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java delete mode 100644 util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java create mode 100644 util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala create mode 100644 util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala delete mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java delete mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java delete mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java delete mode 100644 util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java create mode 100644 util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala create mode 100644 util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala diff --git a/build.sbt b/build.sbt index 1be97fb..92cc478 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ lazy val commonSettings = Seq( organizationHomepage := Some(url("http://www.indix.com")), scalaVersion := "2.11.11", scalacOptions ++= Seq("-encoding", "UTF-8", "-deprecation", "-unchecked"), - javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.8"), + javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.6"), resolvers ++= Seq( "Clojars" at "http://clojars.org/repo", "Concurrent Maven Repo" at "http://conjars.org/repo", @@ -133,6 +133,7 @@ lazy val gocdUtils = (project in file("util-gocd")). "com.google.code.gson" % "gson" % "2.2.3", "junit" % "junit" % "4.12" % Test, "com.novocode" % "junit-interface" % "0.11" % Test, - "org.mockito" % "mockito-all" % "1.10.19" % Test + "org.mockito" % "mockito-all" % "1.10.19" % Test, + "org.scalatest" %% "scalatest" % "3.0.3" % Test ) ) diff --git a/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java b/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java deleted file mode 100644 index dcd83d5..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/models/Artifact.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.indix.gocd.models; - -public class Artifact { - String pipelineName; - String stageName; - String jobName; - Revision revision; - - public Artifact(String pipelineName, String stageName, String jobName) { - this.pipelineName = pipelineName; - this.stageName = stageName; - this.jobName = jobName; - } - - public Artifact(String pipelineName, String stageName, String jobName, Revision revision) { - this.pipelineName = pipelineName; - this.stageName = stageName; - this.jobName = jobName; - this.revision = revision; - } - - public Artifact withRevision(Revision revision) { - this.revision = revision; - return this; - } - - public String prefix(){ - return String.format("%s/%s/%s/", pipelineName, stageName, jobName); - } - - public String prefixWithRevision(){ - if(revision != null) - return String.format("%s/%s/%s/%s/", pipelineName, stageName, jobName, revision.getRevision()); - else - return prefix(); - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java b/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java deleted file mode 100644 index cdeec55..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/models/ResponseMetadataConstants.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.indix.gocd.models; - -public class ResponseMetadataConstants { - public static final String TRACEBACK_URL = "traceback_url"; - public static final String USER = "user"; - public static final String REVISION_COMMENT = "revision_comment"; - public static final String COMPLETED = "completed"; - public static final String GO_PIPELINE_LABEL = "go_pipeline_label"; -} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/Revision.java b/util-gocd/src/main/java/com/indix/gocd/models/Revision.java deleted file mode 100644 index 9b2829c..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/models/Revision.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.indix.gocd.models; - -public class Revision implements Comparable { - private String revision; - private String[] parts; - private Integer major; - private Integer minor; - private Integer patch; - - public Revision(String revision) { - this.revision = revision; - this.parts = revision.split("\\."); - this.major = Integer.valueOf(parts[0]); - this.minor = Integer.valueOf(parts[1]); - if (parts.length == 3) { - this.patch = Integer.valueOf(parts[2]); - } else { - this.patch = 0; - } - } - - public static Revision base() { - return new Revision("0.0.0"); - } - - @Override - public int compareTo(Object otherInstance) { - if(! (otherInstance instanceof Revision)) - throw new RuntimeException("Cannot compare a non-Revision type with Revision type"); - Revision that = (Revision)otherInstance; - int majorDiff = this.major.compareTo(that.major); - int minorDiff = this.minor.compareTo(that.minor); - int patchDiff = this.patch.compareTo(that.patch); - - if(majorDiff != 0) - return majorDiff; - else if(minorDiff != 0) - return minorDiff; - else if(patchDiff != 0) - return patchDiff; - else - return 0; - } - - public String getRevision() { - return revision; - } - -} diff --git a/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java b/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java deleted file mode 100644 index 4fd38a7..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/models/RevisionStatus.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.indix.gocd.models; - -import com.amazonaws.util.StringUtils; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -public class RevisionStatus { - public Revision revision; - public Date lastModified; - public String tracebackUrl; - public String user; - public String revisionLabel; - private static final String DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; - - public RevisionStatus(Revision revision, Date lastModified, String tracebackUrl, String user) { - this(revision, lastModified, tracebackUrl, user, ""); - } - - public RevisionStatus(Revision revision, Date lastModified, String tracebackUrl, String user, String revisionLabel) { - this.revision = revision; - this.lastModified = lastModified; - this.tracebackUrl = tracebackUrl; - this.user = user; - this.revisionLabel = revisionLabel; - } - - public Map toMap() { - final HashMap result = new HashMap(); - result.put("revision", revision.getRevision()); - result.put("timestamp", new SimpleDateFormat(DATE_PATTERN).format(lastModified)); - result.put("user", user); - result.put("revisionComment", String.format("Original revision number: %s", - StringUtils.isNullOrEmpty(revisionLabel) ? "unavailable" : revisionLabel)); - result.put("trackbackUrl", tracebackUrl); - - return result; - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java b/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java deleted file mode 100644 index 25ecd82..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/Constants.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.indix.gocd.utils; - -public class Constants { - public static final String METADATA_USER = "user"; - public static final String METADATA_TRACEBACK_URL = "traceback_url"; - public static final String COMPLETED = "completed"; - - public static final String GO_ARTIFACTS_S3_BUCKET = "GO_ARTIFACTS_S3_BUCKET"; - public static final String GO_SERVER_DASHBOARD_URL = "GO_SERVER_DASHBOARD_URL"; - - public static final String SOURCEDESTINATIONS = "sourceDestinations"; - public static final String DESTINATION_PREFIX = "destinationPrefix"; - public static final String ARTIFACTS_BUCKET = "artifactsBucket"; - - public static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"; - public static final String AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"; - public static final String AWS_REGION = "AWS_REGION"; - public static final String AWS_USE_IAM_ROLE = "AWS_USE_IAM_ROLE"; - public static final String AWS_STORAGE_CLASS = "AWS_STORAGE_CLASS"; - public static final String STORAGE_CLASS_STANDARD = "standard"; - public static final String STORAGE_CLASS_STANDARD_IA = "standard-ia"; - public static final String STORAGE_CLASS_RRS = "rrs"; - public static final String STORAGE_CLASS_GLACIER = "glacier"; - - public static final String GO_PIPELINE_LABEL = "GO_PIPELINE_LABEL"; - - public static final String MATERIAL_TYPE = "MaterialType"; - public static final String REPO = "Repo"; - public static final String PACKAGE = "Package"; - public static final String MATERIAL = "Material"; - public static final String JOB = "Job"; - public static final String STAGE = "Stage"; - public static final String SOURCE = "Source"; - public static final String SOURCE_PREFIX = "SourcePrefix"; - public static final String DESTINATION = "Destination"; - - public static final String REQUIRED_FIELD_MESSAGE = "This field is required"; -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/Context.java b/util-gocd/src/main/java/com/indix/gocd/utils/Context.java deleted file mode 100644 index fe45405..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/Context.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.indix.gocd.utils; - - -import com.thoughtworks.go.plugin.api.task.JobConsoleLogger; - -import java.nio.file.Paths; -import java.util.Map; - -public class Context { - private final Map environmentVariables; - private final String workingDir; - private final JobConsoleLogger console; - - public Context(Map context) { - environmentVariables = (Map) context.get("environmentVariables"); - workingDir = (String) context.get("workingDirectory"); - console = new JobConsoleLogger() {}; - } - - public void printMessage(String message) { - console.printLine(message); - } - - public void printEnvironment() { - console.printEnvironment(environmentVariables); - } - - public Map getEnvironmentVariables() { - return environmentVariables; - } - - public String getWorkingDir() { - return workingDir; - } - - public String getAbsoluteWorkingDir() { - return Paths.get("").toAbsolutePath().resolve(workingDir).toString(); - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java b/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java deleted file mode 100644 index f4680f5..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/GoEnvironment.java +++ /dev/null @@ -1,124 +0,0 @@ -package com.indix.gocd.utils; - -import org.apache.commons.lang3.BooleanUtils; - -import java.lang.StringBuffer; - -import java.util.*; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.indix.gocd.utils.Constants.*; -import static org.apache.commons.lang3.StringUtils.isNotEmpty; - -/** - * Wrapper around Go's Environment variables - */ -public class GoEnvironment { - private Pattern envPat = Pattern.compile("\\$\\{(\\w+)\\}"); - private Map environment = new HashMap<>(); - - public GoEnvironment() { - this.environment.putAll(System.getenv()); - } - - public GoEnvironment(Map defaultEnvironment) { - this(); - this.environment.putAll(defaultEnvironment); - } - - public GoEnvironment putAll(Map existing) { - environment.putAll(existing); - return this; - } - - public Map asMap() { return environment; } - - public String get(String name) { - return environment.get(name); - } - - public String getOrElse(String name, String defaultValue) { - if(has(name)) return get(name); - else return defaultValue; - } - - public boolean has(String name) { - return environment.containsKey(name) && isNotEmpty(get(name)); - } - - public boolean isAbsent(String name) { - return !has(name); - } - - public String traceBackUrl() { - String serverUrl = get(GO_SERVER_DASHBOARD_URL); - String pipelineName = get("GO_PIPELINE_NAME"); - String pipelineCounter = get("GO_PIPELINE_COUNTER"); - String stageName = get("GO_STAGE_NAME"); - String stageCounter = get("GO_STAGE_COUNTER"); - String jobName = get("GO_JOB_NAME"); - return String.format("%s/go/tab/build/detail/%s/%s/%s/%s/%s", serverUrl, pipelineName, pipelineCounter, stageName, stageCounter, jobName); - } - - public String triggeredUser() { - return get("GO_TRIGGER_USER"); - } - - public String replaceVariables(String str) { - Matcher m = envPat.matcher(str); - - StringBuffer sb = new StringBuffer(); - while (m.find()) { - String replacement = get(m.group(1)); - if(replacement != null) { - m.appendReplacement(sb, replacement); - } - } - - m.appendTail(sb); - - return sb.toString(); - } - - /** - * Version Format on S3 is pipeline/stage/job/pipeline_counter.stage_counter - */ - public String artifactsLocationTemplate() { - String pipeline = get("GO_PIPELINE_NAME"); - String stageName = get("GO_STAGE_NAME"); - String jobName = get("GO_JOB_NAME"); - - String pipelineCounter = get("GO_PIPELINE_COUNTER"); - String stageCounter = get("GO_STAGE_COUNTER"); - return artifactsLocationTemplate(pipeline, stageName, jobName, pipelineCounter, stageCounter); - } - - public String artifactsLocationTemplate(String pipeline, String stageName, String jobName, String pipelineCounter, String stageCounter) { - return String.format("%s/%s/%s/%s.%s", pipeline, stageName, jobName, pipelineCounter, stageCounter); - } - - private static final List validUseIamRoleValues = new ArrayList(Arrays.asList("true", "false", "yes", "no", "on", "off")); - public boolean hasAWSUseIamRole() { - if (!has(AWS_USE_IAM_ROLE)) { - return false; - } - - String useIamRoleValue = get(AWS_USE_IAM_ROLE); - Boolean result = BooleanUtils.toBooleanObject(useIamRoleValue); - if (result == null) { - throw new IllegalArgumentException(getEnvInvalidFormatMessage(AWS_USE_IAM_ROLE, - useIamRoleValue, validUseIamRoleValues.toString())); - } - else { - return result.booleanValue(); - } - } - - private String getEnvInvalidFormatMessage(String environmentVariable, String value, String expected){ - return String.format( - "Unexpected value in %s environment variable; was %s, but expected one of the following %s", - environmentVariable, value, expected); - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java b/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java deleted file mode 100644 index f6d62d5..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/MaterialResult.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.indix.gocd.utils; - -import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class MaterialResult { - private boolean success; - private List messages; - - public MaterialResult(boolean success) { - this.success = success; - messages = new ArrayList<>(); - } - - public MaterialResult(boolean success, String message) { - this(success); - messages.add(message); - } - - public Map toMap() { - final HashMap result = new HashMap(); - result.put("status", success ? "success" : "failure"); - result.put("messages", messages); - return result; - } - - public int responseCode() { - return DefaultGoApiResponse.SUCCESS_RESPONSE_CODE; - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java b/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java deleted file mode 100644 index 30852a6..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/TaskExecutionResult.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.indix.gocd.utils; - -import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse; - -import java.util.HashMap; -import java.util.Map; - -public class TaskExecutionResult { - private boolean success; - private String message; - private Exception exception; - - public TaskExecutionResult(boolean success, String message) { - this.success = success; - this.message = message; - } - - public TaskExecutionResult(boolean success, String message, Exception exception) { - this(success, message); - this.exception = exception; - } - - public Map toMap() { - final HashMap result = new HashMap(); - result.put("success", success); - result.put("message", message); - return result; - } - - public int responseCode() { - return DefaultGoApiResponse.SUCCESS_RESPONSE_CODE; - } - - public boolean isSuccessful() { - return success; - } - - public String message() { - return message; - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java b/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java deleted file mode 100644 index ac8bbd7..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/store/S3ArtifactStore.java +++ /dev/null @@ -1,245 +0,0 @@ -package com.indix.gocd.utils.store; - -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.*; -import com.indix.gocd.models.Artifact; -import com.indix.gocd.models.ResponseMetadataConstants; -import com.indix.gocd.models.Revision; -import com.indix.gocd.models.RevisionStatus; -import com.indix.gocd.utils.GoEnvironment; -import com.indix.gocd.utils.utils.Function; -import com.indix.gocd.utils.utils.Functions; -import com.indix.gocd.utils.utils.Lists; -import com.indix.gocd.utils.utils.Maps; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static com.indix.gocd.utils.Constants.*; - -public class S3ArtifactStore { - - private static Map STORAGE_CLASSES = Maps.builder() - .with(STORAGE_CLASS_STANDARD, StorageClass.Standard) - .with(STORAGE_CLASS_STANDARD_IA, StorageClass.StandardInfrequentAccess) - .with(STORAGE_CLASS_RRS, StorageClass.ReducedRedundancy) - .with(STORAGE_CLASS_GLACIER, StorageClass.Glacier) - .build(); - - private AmazonS3 client; - private String bucket; - private StorageClass storageClass = StorageClass.Standard; - - public S3ArtifactStore(AmazonS3 client, String bucket) { - this.client = client; - this.bucket = bucket; - } - - public S3ArtifactStore(GoEnvironment env, String bucket) { - this(getS3client(env), bucket); - } - - public S3ArtifactStore(String bucket) { - this(getS3client(new GoEnvironment()), bucket); - } - - public void setStorageClass(String storageClass) { - String key = StringUtils.lowerCase(storageClass); - if (STORAGE_CLASSES.containsKey(key)) { - this.storageClass = STORAGE_CLASSES.get(key); - } else { - throw new IllegalArgumentException("Invalid storage class specified for S3 - " + storageClass + ". Accepted values are standard, standard-ia, rrs and glacier"); - } - } - - public void put(String from, String to) { - put(new PutObjectRequest(bucket, to, new File(from))); - } - - public void put(String from, String to, ObjectMetadata metadata) { - put(new PutObjectRequest(bucket, to, new File(from)) - .withMetadata(metadata)); - } - - public void put(PutObjectRequest putObjectRequest) { - putObjectRequest.setStorageClass(this.storageClass); - client.putObject(putObjectRequest); - } - - public String pathString(String pathOnS3) { - return String.format("s3://%s/%s", bucket, pathOnS3); - } - - public void get(String from, String to) { - GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, from); - File destinationFile = new File(to); - destinationFile.getParentFile().mkdirs(); - client.getObject(getObjectRequest, destinationFile); - } - - public ObjectMetadata getMetadata(String key) { - return client.getObjectMetadata(bucket, key); - } - - public void getPrefix(String prefix, String to) { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucket) - .withPrefix(prefix); - - ObjectListing objectListing; - do { - objectListing = client.listObjects(listObjectsRequest); - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { - String destinationPath = to + "/" + objectSummary.getKey().replace(prefix + "/", ""); - long size = objectSummary.getSize(); - if (size > 0) { - get(objectSummary.getKey(), destinationPath); - } - } - listObjectsRequest.setMarker(objectListing.getNextMarker()); - } while (objectListing.isTruncated()); - - } - - public boolean bucketExists() { - try { - client.listObjects(new ListObjectsRequest(bucket, null, null, null, 0)); - return true; - } catch (Exception ex) { - return false; - } - } - - public boolean exists(String bucket, String key) { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucket) - .withPrefix(key) - .withDelimiter("/"); - try { - ObjectListing objectListing = client.listObjects(listObjectsRequest); - return objectListing != null && objectListing.getCommonPrefixes().size() > 0; - } catch (Exception ex) { - return false; - } - } - - private Boolean isComplete(String prefix) { - return client.getObjectMetadata(bucket, prefix).getUserMetadata().containsKey(ResponseMetadataConstants.COMPLETED); - } - - private Revision mostRecentRevision(ObjectListing listing) { - List prefixes = Lists.filter(listing.getCommonPrefixes(), new Functions.Predicate() { - @Override - public Boolean execute(String input) { - return isComplete(input); - } - }); - - List revisions = Lists.map(prefixes, new Function() { - @Override - public Revision apply(String prefix) { - String[] parts = prefix.split("/"); - String last = parts[parts.length - 1]; - return new Revision(last); - } - }); - - if (revisions.size() > 0) - return Collections.max(revisions); - else - return Revision.base(); - } - - private Revision latestOfInternal(ObjectListing listing, Revision latestSoFar) { - if (!listing.isTruncated()) { - return latestSoFar; - } else { - ObjectListing objects = client.listNextBatchOfObjects(listing); - Revision mostRecent = mostRecentRevision(objects); - if (latestSoFar.compareTo(mostRecent) > 0) - mostRecent = latestSoFar; - return latestOfInternal(objects, mostRecent); - } - } - - private Revision latestOf(ObjectListing listing) { - return latestOfInternal(listing, mostRecentRevision(listing)); - } - - public RevisionStatus getLatest(Artifact artifact) { - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucket) - .withPrefix(artifact.prefix()) - .withDelimiter("/"); - - ObjectListing listing = client.listObjects(listObjectsRequest); - if (listing != null) { - Revision recent = latestOf(listing); - Artifact artifactWithRevision = artifact.withRevision(recent); - GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, artifactWithRevision.prefixWithRevision()); - ObjectMetadata metadata = client.getObjectMetadata(objectMetadataRequest); - Map userMetadata = metadata.getUserMetadata(); - String tracebackUrl = userMetadata.get(ResponseMetadataConstants.TRACEBACK_URL); - String user = userMetadata.get(ResponseMetadataConstants.USER); - String revisionLabel = userMetadata.containsKey(ResponseMetadataConstants.GO_PIPELINE_LABEL) ? - userMetadata.get(ResponseMetadataConstants.GO_PIPELINE_LABEL) - : ""; - return new RevisionStatus(recent, metadata.getLastModified(), tracebackUrl, user, revisionLabel); - } - return null; - } - - public String getLatestPrefix(String pipeline, String stage, String job, String pipelineCounter) { - String prefix = String.format("%s/%s/%s/%s.", pipeline, stage, job, pipelineCounter); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucket) - .withPrefix(prefix) - .withDelimiter("/"); - - ObjectListing listing = client.listObjects(listObjectsRequest); - - if (listing != null) { - List commonPrefixes = listing.getCommonPrefixes(); - List stageCounters = Lists.map(commonPrefixes, - input -> - input.replaceAll(prefix, "").replaceAll("/", "")); - if (stageCounters.size() > 0) { - int maxStageCounter = Integer.valueOf(stageCounters.get(0)); - - for (int i = 1; i < stageCounters.size(); i++) { - int stageCounter = Integer.valueOf(stageCounters.get(i)); - if (stageCounter > maxStageCounter) { - maxStageCounter = stageCounter; - } - } - - return prefix + maxStageCounter; - } - } - return null; - } - - public static AmazonS3 getS3client(GoEnvironment env) { - AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard(); - - if (env.has(AWS_REGION)) { - amazonS3ClientBuilder.withRegion(env.get(AWS_REGION)); - } - if (env.hasAWSUseIamRole()) { - amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false)); - } else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) { - BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY)); - amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds)); - } - - return amazonS3ClientBuilder.build(); - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java deleted file mode 100644 index e0820b8..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Function.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.indix.gocd.utils.utils; - -public interface Function { - public O apply(I input); -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java deleted file mode 100644 index fd96e16..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Functions.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.indix.gocd.utils.utils; - -public class Functions { - public static abstract class VoidFunction implements Function { - public abstract void execute(I input); - - public Void apply(I input) { - execute(input); - return null; - } - } - - public static abstract class Predicate implements Function { - public abstract Boolean execute(T input); - public Boolean apply(T input){ - return execute(input); - } - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java deleted file mode 100644 index 787e8cd..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Lists.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.indix.gocd.utils.utils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -public class Lists { - - public static void foreach(List originalList, Function function) { - for (T item : originalList) { - function.apply(item); - } - } - - public static void foreach(T[] array, Function function) { - foreach(Arrays.asList(array), function); - } - - public static List flatMap(List originalList, Function> transformer) { - ArrayList flatMapped = new ArrayList(); - for (T item : originalList) { - flatMapped.addAll(transformer.apply(item)); - } - return flatMapped; - } - - public static List flatMap(T[] array, Function> transformer) { - return flatMap(Arrays.asList(array), transformer); - } - - public static List of(T... items) { - ArrayList listToReturn = new ArrayList(); - Collections.addAll(listToReturn, items); - return Collections.unmodifiableList(listToReturn); - } - - public static List map(T[] array, Function transformer) { - return map(Arrays.asList(array), transformer); - } - - public static List map(List elements, Function transformer) { - List mapped = new ArrayList(); - for (T item: elements) { - mapped.add(transformer.apply(item)); - } - return mapped; - } - - public static List filter(List elements, Functions.Predicate predicate) { - List filtered = new ArrayList(); - for(T element: elements){ - if(predicate.execute(element)) - filtered.add(element); - } - return filtered; - } - - public static Boolean exists(List elements, Functions.Predicate predicate) { - for (T element: elements) { - if(predicate.execute(element)) - return true; - } - return false; - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java deleted file mode 100644 index dcb258a..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Maps.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.indix.gocd.utils.utils; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class Maps { - public static MapBuilder builder() { - return new MapBuilder(); - } - public static boolean isEmpty(Map map) { - return map.isEmpty(); - } - - public static class MapBuilder { - private Map internal = new HashMap(); - - public MapBuilder with(K key, V value) { - internal.put(key, value); - return this; - } - - public MapBuilder remove(K key) { - internal.remove(key); - return this; - } - - public Map build() { - return Collections.unmodifiableMap(internal); - } - } -} diff --git a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java b/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java deleted file mode 100644 index eff9ec6..0000000 --- a/util-gocd/src/main/java/com/indix/gocd/utils/utils/Tuple2.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.indix.gocd.utils.utils; - -public class Tuple2 { - private Left _1; - private Right _2; - - public Tuple2(Left _1, Right _2) { - this._1 = _1; - this._2 = _2; - } - - public Left _1() { - return _1; - } - - public Right _2() { - return _2; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Tuple2 tuple2 = (Tuple2) o; - - if (!_1.equals(tuple2._1)) return false; - if (!_2.equals(tuple2._2)) return false; - - return true; - } - - @Override - public int hashCode() { - int result = _1.hashCode(); - result = 31 * result + _2.hashCode(); - return result; - } -} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala new file mode 100644 index 0000000..6b9494a --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Artifact.scala @@ -0,0 +1,25 @@ +package com.indix.gocd.models + +/** + * S3 Artifact + * + * A gocd pipeline can create an s3 artifact at a location that can + * be identified using the pipeline details like pipeline name, + * stage name, job name, etc + * + * @param pipelineName + * @param stageName + * @param jobName + * @param revision + */ +case class Artifact(pipelineName: String, stageName: String, jobName: String, revision: Option[Revision] = None) { + + def withRevision(revision: Revision): Artifact = Artifact(pipelineName, stageName, jobName, Some(revision)) + + def prefix: String = String.format("%s/%s/%s/", pipelineName, stageName, jobName) + + def prefixWithRevision: String = { + if (revision.isDefined) String.format("%s/%s/%s/%s/", pipelineName, stageName, jobName, revision.get.revision) + else prefix + } +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala b/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala new file mode 100644 index 0000000..f906a41 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/ResponseMetadataConstants.scala @@ -0,0 +1,9 @@ +package com.indix.gocd.models + +object ResponseMetadataConstants { + val TRACEBACK_URL = "traceback_url" + val USER = "user" + val REVISION_COMMENT = "revision_comment" + val COMPLETED = "completed" + val GO_PIPELINE_LABEL = "go_pipeline_label" +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala new file mode 100644 index 0000000..1639774 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala @@ -0,0 +1,37 @@ +package com.indix.gocd.models + +case class Revision(revision: String) extends Ordered[Revision] { + + val parts: Seq[String] = revision.split("\\.") + val major: Int = Integer.valueOf(parts.head) + val minor: Int = Integer.valueOf(parts(1)) + val patch: Int = { + if (parts.size == 3) Integer.valueOf(parts(2)) + else 0 + } + +// override def compareTo(that: Revision): Int = { +// val majorDiff = this.major.compareTo(that.major) +// val minorDiff = this.minor.compareTo(that.minor) +// val patchDiff = this.patch.compareTo(that.patch) +// +// if (majorDiff != 0) majorDiff +// else if (minorDiff != 0) minorDiff +// else if (patchDiff != 0) patchDiff +// else 0 +// } + override def compare(that: Revision): Int = { + val majorDiff = this.major.compareTo(that.major) + val minorDiff = this.minor.compareTo(that.minor) + val patchDiff = this.patch.compareTo(that.patch) + + if (majorDiff != 0) majorDiff + else if (minorDiff != 0) minorDiff + else if (patchDiff != 0) patchDiff + else 0 + } +} + +object Revision { + def base: Revision = Revision("0.0.0") +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala new file mode 100644 index 0000000..5c5ca6d --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala @@ -0,0 +1,22 @@ +package com.indix.gocd.models + +import java.text.SimpleDateFormat +import java.util.Date + +import com.amazonaws.util.StringUtils + +case class RevisionStatus(revision: Revision, lastModified: Date, trackbackUrl: String, user: String, revisionLabel: String = "") { + + val DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" + + def toMap: Map[String, String] = { + Map( + "revision" -> revision.revision, + "timestamp" -> new SimpleDateFormat(DATE_PATTERN).format(lastModified), + "user" -> user, + "revisionComment" -> String.format("Original revision number: %s", if (StringUtils.isNullOrEmpty(revisionLabel)) "unavailable" + else revisionLabel), + "trackbackUrl" -> trackbackUrl + ) + } +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala new file mode 100644 index 0000000..26c3b16 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/Constants.scala @@ -0,0 +1,38 @@ +package com.indix.gocd.utils + +object Constants { + val METADATA_USER = "user" + val METADATA_TRACEBACK_URL = "traceback_url" + val COMPLETED = "completed" + + val GO_ARTIFACTS_S3_BUCKET = "GO_ARTIFACTS_S3_BUCKET" + val GO_SERVER_DASHBOARD_URL = "GO_SERVER_DASHBOARD_URL" + + val SOURCEDESTINATIONS = "sourceDestinations" + val DESTINATION_PREFIX = "destinationPrefix" + val ARTIFACTS_BUCKET = "artifactsBucket" + + val AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" + val AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" + val AWS_REGION = "AWS_REGION" + val AWS_USE_IAM_ROLE = "AWS_USE_IAM_ROLE" + val AWS_STORAGE_CLASS = "AWS_STORAGE_CLASS" + val STORAGE_CLASS_STANDARD = "standard" + val STORAGE_CLASS_STANDARD_IA = "standard-ia" + val STORAGE_CLASS_RRS = "rrs" + val STORAGE_CLASS_GLACIER = "glacier" + + val GO_PIPELINE_LABEL = "GO_PIPELINE_LABEL" + + val MATERIAL_TYPE = "MaterialType" + val REPO = "Repo" + val PACKAGE = "Package" + val MATERIAL = "Material" + val JOB = "Job" + val STAGE = "Stage" + val SOURCE = "Source" + val SOURCE_PREFIX = "SourcePrefix" + val DESTINATION = "Destination" + + val REQUIRED_FIELD_MESSAGE = "This field is required" +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala new file mode 100644 index 0000000..1d632ae --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala @@ -0,0 +1,22 @@ +package com.indix.gocd.utils + +import java.nio.file.Paths + +import com.thoughtworks.go.plugin.api.task.JobConsoleLogger + +import scala.collection.JavaConverters._ + +case class Context(environmentVariables: Map[String, String], workingDir: String, console: JobConsoleLogger) { + + def printMessage(message: String): Unit = console.printLine(message) + + def printEnvironment(): Unit = console.printEnvironment(environmentVariables.asJava) + + def getAbsoluteWorkingDir: String = Paths.get("").toAbsolutePath.resolve(workingDir).toString + +// def this(context: java.util.Map[String, Object]) = this(context.asScala) +} + +object Context { + def apply(context: Map[String, _]): Context = Context(context("environmentVariables").asInstanceOf[Map[String, String]], context("workingDirectory").asInstanceOf[String], JobConsoleLogger.getConsoleLogger) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala new file mode 100644 index 0000000..007d5e2 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/GoEnvironment.scala @@ -0,0 +1,85 @@ +package com.indix.gocd.utils + +import java.util.regex.Pattern + +import com.indix.gocd.utils.Constants.{AWS_USE_IAM_ROLE, GO_SERVER_DASHBOARD_URL} +import org.apache.commons.lang3.BooleanUtils +import org.apache.commons.lang3.StringUtils.isNotEmpty + +import scala.collection.JavaConversions._ + +class GoEnvironment(input: Option[Map[String, String]] = None) { + + val environment: Map[String, String] = { + if (input.isDefined) + System.getenv().toMap ++ input.get + else + System.getenv().toMap + } + val envPat: Pattern = Pattern.compile("\\$\\{(\\w+)\\}") + + def putAll(existing: Map[String, String]): GoEnvironment = new GoEnvironment(Some(existing ++ environment)) + + def asMap: Map[String, String] = environment + + def get(name: String): String = environment(name) + + def getOrElse(name: String, defaultValue: String): String = environment.getOrElse(name, defaultValue) + + def has(name: String): Boolean = environment.containsKey(name) && isNotEmpty(get(name)) + + def isAbsent(name: String): Boolean = !has(name) + + def traceBackUrl: String = { + val serverUrl = get(GO_SERVER_DASHBOARD_URL) + val pipelineName = get("GO_PIPELINE_NAME") + val pipelineCounter = get("GO_PIPELINE_COUNTER") + val stageName = get("GO_STAGE_NAME") + val stageCounter = get("GO_STAGE_COUNTER") + val jobName = get("GO_JOB_NAME") + String.format("%s/go/tab/build/detail/%s/%s/%s/%s/%s", serverUrl, pipelineName, pipelineCounter, stageName, stageCounter, jobName) + } + + def triggeredUser: String = get("GO_TRIGGER_USER") + + def replaceVariables(str: String): String = { + val m = envPat.matcher(str) + val sb = new StringBuffer + while (m.find()) { + val replacement = environment.get(m.group(1)) + if (replacement.isDefined) { + m.appendReplacement(sb, replacement.get) + } + } + m.appendTail(sb) + sb.toString + } + + /** + * Version Format on S3 is pipeline/stage/job/pipeline_counter.stage_counter + */ + def artifactsLocationTemplate: String = { + val pipeline = get("GO_PIPELINE_NAME") + val stageName = get("GO_STAGE_NAME") + val jobName = get("GO_JOB_NAME") + val pipelineCounter = get("GO_PIPELINE_COUNTER") + val stageCounter = get("GO_STAGE_COUNTER") + artifactsLocationTemplate(pipeline, stageName, jobName, pipelineCounter, stageCounter) + } + + def artifactsLocationTemplate(pipeline: String, stageName: String, jobName: String, pipelineCounter: String, stageCounter: String): String = String.format("%s/%s/%s/%s.%s", pipeline, stageName, jobName, pipelineCounter, stageCounter) + + private val validUseIamRoleValues: List[String] = List("true", "false", "yes", "no", "on", "off") + + def hasAWSUseIamRole: Boolean = { + if (!has(AWS_USE_IAM_ROLE)) false + val useIamRoleValue = get(AWS_USE_IAM_ROLE) + val result = BooleanUtils.toBooleanObject(useIamRoleValue) + if (result == null) throw new IllegalArgumentException(getEnvInvalidFormatMessage(AWS_USE_IAM_ROLE, useIamRoleValue, validUseIamRoleValues.mkString("[", ", ", "]"))) + else result.booleanValue + } + + private def getEnvInvalidFormatMessage(environmentVariable: String, value: String, expected: String) = String.format("Unexpected value in %s environment variable; was %s, but expected one of the following %s", environmentVariable, value, expected) + + def this() = this(None) +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala new file mode 100644 index 0000000..8e53fce --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/MaterialResult.scala @@ -0,0 +1,22 @@ +package com.indix.gocd.utils + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse + +case class MaterialResult(success: Boolean, messages: Seq[String]) { + + def toMap = Map( + "status" -> { + if (success) "success" else "failure" + }, + "messages" -> messages + ) + + def responseCode: Int = DefaultGoApiResponse.SUCCESS_RESPONSE_CODE +} + +object MaterialResult { + + def apply(success: Boolean) : MaterialResult = MaterialResult(success, Seq()) + + def apply(success: Boolean, message: String) : MaterialResult = MaterialResult(success, Seq(message)) +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala new file mode 100644 index 0000000..0de507f --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/TaskExecutionResult.scala @@ -0,0 +1,10 @@ +package com.indix.gocd.utils + +import com.thoughtworks.go.plugin.api.response.DefaultGoApiResponse + +case class TaskExecutionResult(success: Boolean, message: String, exception: Option[Exception] = None) { + + def toMap = Map("success" -> success, "message" -> message) + + def responseCode: Int = DefaultGoApiResponse.SUCCESS_RESPONSE_CODE +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala new file mode 100644 index 0000000..3cbc135 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala @@ -0,0 +1,180 @@ +package com.indix.gocd.utils.store + +import java.io.File + +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials, InstanceProfileCredentialsProvider} +import com.amazonaws.services.s3.model._ +import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} +import com.indix.gocd.models.{Artifact, ResponseMetadataConstants, Revision, RevisionStatus} +import com.indix.gocd.utils.Constants._ +import com.indix.gocd.utils.GoEnvironment +import com.indix.gocd.utils.store.S3ArtifactStore.getS3client + +import scala.collection.JavaConversions._ + +class S3ArtifactStore(client: AmazonS3 = getS3client(new GoEnvironment()), bucket: String, storageClass: StorageClass = StorageClass.Standard) { + + def withStorageClass(storageClass: String): S3ArtifactStore = { + val key = storageClass.toLowerCase() + if (S3ArtifactStore.STORAGE_CLASSES.contains(key)) { + new S3ArtifactStore(client, bucket, S3ArtifactStore.STORAGE_CLASSES(key)) + } + else { + throw new IllegalArgumentException("Invalid storage class specified for S3 - " + storageClass + ". Accepted values are standard, standard-ia, rrs and glacier") + } + } + + def put(putObjectRequest: PutObjectRequest): PutObjectResult = { + putObjectRequest.setStorageClass(this.storageClass) + client.putObject(putObjectRequest) + } + + def put(from: String, to: String, metadata: ObjectMetadata): PutObjectResult = { + put(new PutObjectRequest(bucket, to, new File(from)).withMetadata(metadata)) + } + + def put(from: String, to: String): PutObjectResult = { + put(new PutObjectRequest(bucket, to, new File(from))) + } + + def pathString(pathOnS3: String): String = String.format("s3://%s/%s", bucket, pathOnS3) + + def get(from: String, to: String): ObjectMetadata = { + val getObjectRequest = new GetObjectRequest(bucket, from) + val destinationFile = new File(to) + destinationFile.getParentFile.mkdirs + client.getObject(getObjectRequest, destinationFile) + } + + def getMetadata(key: String): ObjectMetadata = client.getObjectMetadata(bucket, key) + + def getPrefix(prefix: String, to: String): Unit = { + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix) + getAll(client.listObjects(listObjectsRequest)) + + def getAll(objectListing: ObjectListing): Unit = { + if (objectListing.isTruncated) None + else { + for (objectSummary <- objectListing.getObjectSummaries) { + val destinationPath = to + "/" + objectSummary.getKey.replace(prefix + "/", "") + if (objectSummary.getSize > 0) { + val x = get(objectSummary.getKey, destinationPath) + } + } + listObjectsRequest.setMarker(objectListing.getNextMarker) + getAll(client.listObjects(listObjectsRequest)) + } + } + } + + def bucketExists: Boolean = { + try { + client.listObjects(new ListObjectsRequest(bucket, null, null, null, 0)) + true + } + catch { + case e: Exception => false + } + } + + def exists(bucket: String, key: String): Boolean = { + val listObjectRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key).withDelimiter("/") + try { + val listing = client.listObjects(listObjectRequest) + listing != null && !listing.getCommonPrefixes.isEmpty + } + catch { + case e: Exception => false + } + } + + private def isComplete(prefix: String): Boolean = client.getObjectMetadata(bucket, prefix).getUserMetadata.contains(ResponseMetadataConstants.COMPLETED) + + private def mostRecentRevision(objectListing: ObjectListing): Revision = { + val prefixes = objectListing.getCommonPrefixes.filter(isComplete) + val revisions = prefixes.map(prefix => Revision(prefix.split("/").last)) + if (revisions.isEmpty) Revision.base + else revisions.max + } + + private def latestOfInternal(objectListing: ObjectListing, latestRevision: Revision): Revision = { + if (!objectListing.isTruncated) latestRevision + else { + val objects = client.listNextBatchOfObjects(objectListing) + val mostRecent = mostRecentRevision(objects) + latestOfInternal(objectListing, latestRevision = { + if (latestRevision.compareTo(mostRecent) > 0) latestRevision + else mostRecent + }) + } + } + + private def latestOf(objectListing: ObjectListing) = latestOfInternal(objectListing, mostRecentRevision(objectListing)) + + def getLatest(artifact: Artifact): Option[RevisionStatus] = { + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(artifact.prefix).withDelimiter("/") + + val objectListing = client.listObjects(listObjectsRequest) + if (objectListing != null) { + val recent = latestOf(objectListing) + + val getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, artifact.withRevision(recent).prefixWithRevision) + val metadata = client.getObjectMetadata(getObjectMetadataRequest) + val userMetadata = metadata.getUserMetadata + + val tracebackUrl = userMetadata(ResponseMetadataConstants.TRACEBACK_URL) + val user = userMetadata(ResponseMetadataConstants.USER) + val revisionLabel = userMetadata.getOrDefault(ResponseMetadataConstants.GO_PIPELINE_LABEL, "") + + Some(RevisionStatus(recent, metadata.getLastModified, tracebackUrl, user, revisionLabel)) + } + else None + } + + def getLatestPrefix(pipeline: String, stage: String, job: String, pipelineCounter: String): Option[String] = { + val prefix = String.format("%s/%s/%s/%s.", pipeline, stage, job, pipelineCounter) + + val listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix).withDelimiter("/") + val objectListing = client.listObjects(listObjectsRequest) + + if (objectListing != null) { + val stageCounters = objectListing.getCommonPrefixes.map(input => input.replaceAll(prefix, "").replaceAll("/", "")) + + if (stageCounters.nonEmpty) Some(prefix + stageCounters.maxBy(counter => Integer.valueOf(counter))) + else None + } + else None + } + + def this(env: GoEnvironment, bucket: String) = this(getS3client(env), bucket, StorageClass.Standard) + + def this(client: AmazonS3, bucket: String) = this(client, bucket, StorageClass.Standard) +} + +object S3ArtifactStore { + val STORAGE_CLASSES = Map( + STORAGE_CLASS_STANDARD -> StorageClass.Standard, + STORAGE_CLASS_STANDARD_IA -> StorageClass.StandardInfrequentAccess, + STORAGE_CLASS_RRS -> StorageClass.ReducedRedundancy, + STORAGE_CLASS_GLACIER -> StorageClass.Glacier + ) + + def getS3client(env: GoEnvironment): AmazonS3 = { + + def withRegion(clientBuilder: AmazonS3ClientBuilder) = { + if (env.has(AWS_REGION)) clientBuilder.withRegion(env.get(AWS_REGION)) + else clientBuilder + } + + def withCredentials(clientBuilder: AmazonS3ClientBuilder) = { + if (env.hasAWSUseIamRole) clientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false)) + else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) { + val basicAWSCredentials = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY)) + clientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicAWSCredentials)) + } + else clientBuilder + } + + withCredentials(withRegion(AmazonS3ClientBuilder.standard())).build() + } +} \ No newline at end of file diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala new file mode 100644 index 0000000..a161f4a --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Functions.scala @@ -0,0 +1,15 @@ +package com.indix.gocd.utils.utils + +trait Function[I, O] { + def apply(input: I): O +} + +abstract class VoidFunction[I] extends Function[I, Nothing]{ + def execute(i: I): Nothing + override def apply(input: I): Nothing = execute(input) +} + +abstract class Predicate[T] extends Function[T, Boolean] { + def execute(i: T): Boolean + override def apply(input: T): Boolean = execute(input) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala new file mode 100644 index 0000000..b09148e --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Maps.scala @@ -0,0 +1,23 @@ +package com.indix.gocd.utils.utils + +import java.util + +import scala.collection.JavaConversions._ + +object Maps { + case class MapBuilder[K, V](internal: Map[K, V]) { + + def _with(k: K, v: V) = MapBuilder(internal ++ Map[K, V](k -> v)) + + def remove(k: K): MapBuilder[K, V] = { + val newInternal = internal.filterNot(tuple => k.equals(tuple._1)) + MapBuilder(newInternal) + } + + def buildJavaMap(): util.Map[K, V] = mapAsJavaMap(internal) + + def build: Map[K, V] = internal + } + + def builder[K, V] = MapBuilder(Map[K, V]()) +} diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala new file mode 100644 index 0000000..5a64126 --- /dev/null +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/utils/Tuple2.scala @@ -0,0 +1,6 @@ +package com.indix.gocd.utils.utils + +case class Tuple2[Left, Right](internal: (Left, Right)) { + def _1: Left = internal._1 + def _2: Right = internal._2 +} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java deleted file mode 100644 index 09161ed..0000000 --- a/util-gocd/src/test/java/com/indix/gocd/utils/GoEnvironmentTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.indix.gocd.utils; - -import static com.indix.gocd.utils.Constants.AWS_USE_IAM_ROLE; -import static org.hamcrest.CoreMatchers.is; - -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -import java.util.HashMap; -import java.util.Map; -import com.indix.gocd.utils.utils.Maps; -import static com.indix.gocd.utils.Constants.GO_SERVER_DASHBOARD_URL; -import static org.junit.Assert.fail; - - -public class GoEnvironmentTest { - private GoEnvironment goEnvironment; - Maps.MapBuilder mockEnvironmentVariables; - - @Before - public void setUp() throws Exception { - mockEnvironmentVariables = Maps.builder() - .with(GO_SERVER_DASHBOARD_URL, "http://go.server:8153") - .with("GO_SERVER_URL", "https://localhost:8154/go") - .with("GO_PIPELINE_NAME", "s3-publish-test") - .with("GO_PIPELINE_COUNTER", "20") - .with("GO_STAGE_NAME", "build-and-publish") - .with("GO_STAGE_COUNTER", "1") - .with("GO_JOB_NAME", "publish") - .with("GO_TRIGGER_USER", "Krishna"); - goEnvironment = new GoEnvironment(new HashMap()).putAll(mockEnvironmentVariables.build()); - } - - @Test - public void shouldGenerateTracebackUrl() { - assertThat(goEnvironment.traceBackUrl(), is("http://go.server:8153/go/tab/build/detail/s3-publish-test/20/build-and-publish/1/publish")); - } - - @Test - public void shouldReturnTriggeredUser() { - assertThat(goEnvironment.triggeredUser(), is("Krishna")); - } - - @Test - public void shouldGenerateArtifactLocationTemplate() { - assertThat(goEnvironment.artifactsLocationTemplate(), is("s3-publish-test/build-and-publish/publish/20.1")); - } - - @Test - public void shouldReturnAsMap() { - for(Map.Entry entry : mockEnvironmentVariables.build().entrySet()) { - assertEquals(entry.getValue(), goEnvironment.asMap().get(entry.getKey())); - } - } - - @Test - public void shouldReplaceWithEnvVariables() { - final String envTestTemplate = "COUNT:${GO_STAGE_COUNTER} Name:${GO_STAGE_NAME} COUNT2:${GO_STAGE_COUNTER}"; - final String replaced = goEnvironment.replaceVariables(envTestTemplate); - - assertThat(replaced, is("COUNT:1 Name:build-and-publish COUNT2:1")); - } - - @Test - public void shouldNotReplaceUnknownEnvVariables() { - final String envTestTemplate = "COUNT:${GO_STAGE_COUNTER} ${DOESNT_EXIST}"; - final String replaced = goEnvironment.replaceVariables(envTestTemplate); - - assertThat(replaced, is("COUNT:1 ${DOESNT_EXIST}")); - } - - @Test - public void shouldGetHasAWSUseIamRoleTrueIfSetToTrue() { - GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables - .with(AWS_USE_IAM_ROLE,"True") - .build()); - - Boolean result = sut.hasAWSUseIamRole(); - - assertThat(result, is(Boolean.TRUE)); - } - - @Test - public void shouldGetHasAWSUseIamRoleFalseIfSetToFalse() { - GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables - .with(AWS_USE_IAM_ROLE,"False") - .build()); - - Boolean result = sut.hasAWSUseIamRole(); - - assertThat(result, is(Boolean.FALSE)); - } - - - @Test - public void shouldThrowExceptionIfAWSUseIamRoleNotWithinExpectedValues() { - GoEnvironment sut = new GoEnvironment(mockEnvironmentVariables - .with(AWS_USE_IAM_ROLE,"blue") - .build()); - - try { - sut.hasAWSUseIamRole(); - fail("Expected exception"); - } catch (IllegalArgumentException e) { - assertEquals( - "Unexpected value in AWS_USE_IAM_ROLE environment variable; was blue, " + - "but expected one of the following [true, false, yes, no, on, off]", - e.getMessage()); - } - - } -} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java b/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java deleted file mode 100644 index aa181ed..0000000 --- a/util-gocd/src/test/java/com/indix/gocd/utils/mocks/MockContext.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.indix.gocd.utils.mocks; - -import com.indix.gocd.utils.Context; -import com.thoughtworks.go.plugin.api.task.Console; -import com.thoughtworks.go.plugin.api.task.EnvironmentVariables; -import com.thoughtworks.go.plugin.api.task.TaskExecutionContext; -import org.apache.commons.lang3.StringUtils; - -import java.io.InputStream; -import java.util.Map; - -public class MockContext extends Context { - - public MockContext(Map contextMap) { - super(contextMap); - } - - @Override - public void printMessage(String message) { - - } - - @Override - public void printEnvironment() { - - } - -} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java deleted file mode 100644 index c85e5ca..0000000 --- a/util-gocd/src/test/java/com/indix/gocd/utils/store/S3ArtifactStoreTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.indix.gocd.utils.store; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.indix.gocd.utils.GoEnvironment; -import org.junit.Test; -import org.mockito.ArgumentCaptor; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.indix.gocd.utils.Constants.AWS_ACCESS_KEY_ID; -import static com.indix.gocd.utils.Constants.AWS_REGION; -import static com.indix.gocd.utils.Constants.AWS_SECRET_ACCESS_KEY; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.*; - -public class S3ArtifactStoreTest { - - AmazonS3Client mockClient = mock(AmazonS3Client.class); - ArgumentCaptor putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); - ArgumentCaptor listingCaptor = ArgumentCaptor.forClass(ListObjectsRequest.class); - - @Test - public void shouldUseStandardStorageClassAsDefault() { - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); - verify(mockClient, times(1)).putObject(putCaptor.capture()); - PutObjectRequest putRequest = putCaptor.getValue(); - assertThat(putRequest.getStorageClass(), is("STANDARD")); - } - - @Test - public void shouldUseStandardIAStorageClassAsDefault() { - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - store.setStorageClass("standard-ia"); - store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); - verify(mockClient, times(1)).putObject(putCaptor.capture()); - PutObjectRequest putRequest = putCaptor.getValue(); - assertThat(putRequest.getStorageClass(), is("STANDARD_IA")); - } - - @Test - public void shouldUseReducedRedundancyStorageClass() { - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - store.setStorageClass("rrs"); - store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); - verify(mockClient, times(1)).putObject(putCaptor.capture()); - PutObjectRequest putRequest = putCaptor.getValue(); - assertThat(putRequest.getStorageClass(), is("REDUCED_REDUNDANCY")); - } - - @Test - public void shouldUseGlacierStorageClass() { - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - store.setStorageClass("glacier"); - store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))); - verify(mockClient, times(1)).putObject(putCaptor.capture()); - PutObjectRequest putRequest = putCaptor.getValue(); - assertThat(putRequest.getStorageClass(), is("GLACIER")); - } - - @Test - public void shouldSuccessfullyCheckIfBucketExists() { - doReturn(new ObjectListing()).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - assertThat(store.bucketExists(), is(true)); - } - - @Test - public void shouldHandleBucketDoesNotExists() { - doThrow(new RuntimeException("Bucket does not exist")).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - assertThat(store.bucketExists(), is(false)); - } - - @Test - public void verifyObjectListingRequestIsRight() { - doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - store.getLatestPrefix("pipeline", "stage", "job", "1"); - - verify(mockClient).listObjects(listingCaptor.capture()); - ListObjectsRequest request = listingCaptor.getValue(); - assertEquals("foo-bar", request.getBucketName()); - assertEquals("pipeline/stage/job/1.", request.getPrefix()); - assertEquals("/", request.getDelimiter()); - } - - @Test - public void shouldReturnNullWhenObjectListingIsNull() { - doReturn(null).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - - String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); - assertNull(prefix); - } - - @Test - public void shouldReturnNullWhenObjectListingIsSize0() { - ObjectListing listing = new ObjectListing(); - doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - - String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); - assertNull(prefix); - } - - @Test - public void shouldReturnTheLatestStageCounter() { - ObjectListing listing = new ObjectListing(); - List commonPrefixes = new ArrayList<>(); - commonPrefixes.add("pipeline/stage/job/1.2"); - commonPrefixes.add("pipeline/stage/job/1.1"); - commonPrefixes.add("pipeline/stage/job/1.7"); - listing.setCommonPrefixes(commonPrefixes); - - doReturn(listing).when(mockClient).listObjects(any(ListObjectsRequest.class)); - S3ArtifactStore store = new S3ArtifactStore(mockClient, "foo-bar"); - - String prefix = store.getLatestPrefix("pipeline", "stage", "job", "1"); - assertEquals("pipeline/stage/job/1.7", prefix); - } -} diff --git a/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java b/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java deleted file mode 100644 index 5de101a..0000000 --- a/util-gocd/src/test/java/com/indix/gocd/utils/utils/ListsTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.indix.gocd.utils.utils; - -import org.junit.Test; -import java.util.List; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -import static com.indix.gocd.utils.utils.Lists.*; -import static com.indix.gocd.utils.utils.Functions.VoidFunction; - -public class ListsTest { - @Test - public void shouldRunForEachOnceForEveryElement() { - final int[] sum = {0}; - List list = Lists.of(1, 2, 3, 4, 5); - foreach(list, new VoidFunction() { - @Override - public void execute(Integer input) { - sum[0] += (input * 2); - } - }); - - assertThat(sum[0], is(30)); - } - - @Test - public void shouldTransformEveryElementInTheListAndFlatten() { - List duplicateNumbers = flatMap(Lists.of(1, 2, 3, 4, 5), new Function>() { - @Override - public List apply(Integer input) { - return Lists.of(input, input * 2); - } - }); - - assertThat(duplicateNumbers, is(Lists.of(1, 2, 2, 4, 3, 6, 4, 8, 5, 10))); - } -} diff --git a/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala b/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala new file mode 100644 index 0000000..529df65 --- /dev/null +++ b/util-gocd/src/test/scala/com/indix/gocd/utils/GoEnvironmentSpec.scala @@ -0,0 +1,71 @@ +package com.indix.gocd.utils + +import com.indix.gocd.utils.Constants.GO_SERVER_DASHBOARD_URL +import org.scalatest.{FlatSpec, Matchers} + +class GoEnvironmentSpec extends FlatSpec with Matchers { + + val mockEnvironmentVariables = Map( + GO_SERVER_DASHBOARD_URL -> "http://go.server:8153", + "GO_SERVER_URL" -> "https://localhost:8154/go", + "GO_PIPELINE_NAME" -> "s3-publish-test", + "GO_PIPELINE_COUNTER" -> "20", + "GO_STAGE_NAME" -> "build-and-publish", + "GO_STAGE_COUNTER" -> "1", + "GO_JOB_NAME" -> "publish", + "GO_TRIGGER_USER" -> "Krishna") + + val goEnvironment = new GoEnvironment(Some(mockEnvironmentVariables)) + + "traceBackUrl" should "generate trace back url" in { + goEnvironment.traceBackUrl should be("http://go.server:8153/go/tab/build/detail/s3-publish-test/20/build-and-publish/1/publish") + } + + "triggeredUser" should "return triggered user" in { + goEnvironment.triggeredUser should be("Krishna") + } + + "artifactsLocationTemplate" should "generate artifacts location template" in { + goEnvironment.artifactsLocationTemplate should be("s3-publish-test/build-and-publish/publish/20.1") + } + + "asMap" should "return as map" in { + mockEnvironmentVariables.foreach(tuple => goEnvironment.get(tuple._1) should be(tuple._2)) + } + + "replaceVariables" should "replace the env variables with respective values in the template string" in { + val template = "COUNT:${GO_STAGE_COUNTER} Name:${GO_STAGE_NAME} COUNT2:${GO_STAGE_COUNTER}" + val replaced = goEnvironment.replaceVariables(template) + replaced should be("COUNT:1 Name:build-and-publish COUNT2:1") + } + + it should "not replace unknown env variables in the template string" in { + val template = "COUNT:${GO_STAGE_COUNTER} ${DOESNT_EXIST}" + val replaced = goEnvironment.replaceVariables(template) + replaced should be("COUNT:1 ${DOESNT_EXIST}") + } + + "hasAWSUseIamRole" should "return true if it is set to true" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "True")) + env.hasAWSUseIamRole should be(true) + } + + it should "return false if it is set to false" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "False")) + env.hasAWSUseIamRole should be(false) + } + + it should "throw exception if the value is not a valid boolean" in { + val env = goEnvironment.putAll(mockEnvironmentVariables ++ Map(Constants.AWS_USE_IAM_ROLE -> "Blue")) + + try { + env.hasAWSUseIamRole + fail("Expected Exception") + } + catch { + case e: Exception => e.getMessage should be + "Unexpected value in AWS_USE_IAM_ROLE environment variable; was Blue, but expected one of " + + "the following [true, false, yes, no, on, off]" + } + } +} diff --git a/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala b/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala new file mode 100644 index 0000000..6eec2e6 --- /dev/null +++ b/util-gocd/src/test/scala/com/indix/gocd/utils/store/S3ArtifactStoreSpec.scala @@ -0,0 +1,114 @@ +package com.indix.gocd.utils.store + +import java.io.File + +import com.amazonaws.services.s3.AmazonS3Client +import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing, PutObjectRequest} +import org.mockito.Matchers.any +import org.mockito.Mockito.{times, verify, when} +import org.mockito.{ArgumentCaptor, Mock, Mockito} +import org.scalatest.mockito.MockitoSugar +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + +import scala.collection.JavaConversions._ + +class S3ArtifactStoreSpec extends FlatSpec with Matchers with BeforeAndAfterEach { + + @Mock + private val mockClient = MockitoSugar.mock[AmazonS3Client] + private val putCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest]) + private val listingCaptor: ArgumentCaptor[ListObjectsRequest] = ArgumentCaptor.forClass(classOf[ListObjectsRequest]) + private val store = new S3ArtifactStore(mockClient, "foo-bar") + + override def afterEach(): Unit = { + super.afterEach() + Mockito.reset(mockClient) + } + + "put" should "use standard storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("STANDARD") + } + + it should "use standard-ia storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("standard-ia") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("STANDARD_IA") + } + + it should "use reduced redundancy class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("rrs") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("REDUCED_REDUNDANCY") + } + + it should "use glacier storage class as default" in { + val store = new S3ArtifactStore(mockClient, "foo-bar").withStorageClass("glacier") + store.put(new PutObjectRequest("foo-bar", "key", new File("/tmp/baz"))) + verify(mockClient, times(1)).putObject(putCaptor.capture) + + val putRequest = putCaptor.getValue + putRequest.getStorageClass should be("GLACIER") + } + + "bucketExists" should "successfully check if bucket exists" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(new ObjectListing) + store.bucketExists should be(true) + } + + it should "return false when bucket does not exist" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenThrow(new RuntimeException("Bucket does not exist")) + try { + store.bucketExists + } + catch { + case e: RuntimeException => e.getMessage should be("Bucket does not exist") + } + } + + "listObjects" should "return the right objects list" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(null) + store.getLatestPrefix("pipeline", "stage", "job", "1") + verify(mockClient).listObjects(listingCaptor.capture()) + + val request = listingCaptor.getValue + request.getBucketName should be("foo-bar") + request.getPrefix should be("pipeline/stage/job/1.") + request.getDelimiter should be("/") + } + + "getLatestPrefix" should "not be defined when object listing is null" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(null) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(false) + } + + it should "not be defined when object listing size is 0" in { + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(new ObjectListing) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(false) + } + + it should "return the latest stage counter from the listing" in { + val listing = new ObjectListing + listing.setCommonPrefixes(List("pipeline/stage/job/1.2", "pipeline/stage/job/1.1", "pipeline/stage/job/1.7")) + + when(mockClient.listObjects(any(classOf[ListObjectsRequest]))).thenReturn(listing) + val latestPrefix = store.getLatestPrefix("pipeline", "stage", "job", "1") + + latestPrefix.isDefined should be(true) + latestPrefix.get should be("pipeline/stage/job/1.7") + } +} From 8356e07bca9da7403d8a645fdeb9308bd7f17ac0 Mon Sep 17 00:00:00 2001 From: Rucha Date: Wed, 2 Jan 2019 10:38:38 +0530 Subject: [PATCH 3/5] revert java back to 1.7 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 92cc478..ea2af59 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ lazy val commonSettings = Seq( organizationHomepage := Some(url("http://www.indix.com")), scalaVersion := "2.11.11", scalacOptions ++= Seq("-encoding", "UTF-8", "-deprecation", "-unchecked"), - javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.6"), + javacOptions ++= Seq("-Xlint:deprecation", "-source", "1.7"), resolvers ++= Seq( "Clojars" at "http://clojars.org/repo", "Concurrent Maven Repo" at "http://conjars.org/repo", From 573454e98928455f5e517d745eea10cf941695b7 Mon Sep 17 00:00:00 2001 From: Rucha Date: Wed, 2 Jan 2019 10:49:08 +0530 Subject: [PATCH 4/5] documentation --- .../com/indix/gocd/models/Revision.scala | 16 ++++++--------- .../indix/gocd/models/RevisionStatus.scala | 9 +++++++++ .../scala/com/indix/gocd/utils/Context.scala | 2 -- .../gocd/utils/store/S3ArtifactStore.scala | 20 +++++++++++++++++++ 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala index 1639774..a55d574 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala @@ -1,5 +1,11 @@ package com.indix.gocd.models +/** + * Represents revision + * Parses revision string to pupulate fields + * + * @param revision + */ case class Revision(revision: String) extends Ordered[Revision] { val parts: Seq[String] = revision.split("\\.") @@ -10,16 +16,6 @@ case class Revision(revision: String) extends Ordered[Revision] { else 0 } -// override def compareTo(that: Revision): Int = { -// val majorDiff = this.major.compareTo(that.major) -// val minorDiff = this.minor.compareTo(that.minor) -// val patchDiff = this.patch.compareTo(that.patch) -// -// if (majorDiff != 0) majorDiff -// else if (minorDiff != 0) minorDiff -// else if (patchDiff != 0) patchDiff -// else 0 -// } override def compare(that: Revision): Int = { val majorDiff = this.major.compareTo(that.major) val minorDiff = this.minor.compareTo(that.minor) diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala index 5c5ca6d..8e88578 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/models/RevisionStatus.scala @@ -5,6 +5,15 @@ import java.util.Date import com.amazonaws.util.StringUtils +/** + * Used to display the status of revisions in gocd + * + * @param revision + * @param lastModified + * @param trackbackUrl + * @param user + * @param revisionLabel + */ case class RevisionStatus(revision: Revision, lastModified: Date, trackbackUrl: String, user: String, revisionLabel: String = "") { val DATE_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala index 1d632ae..bf451b5 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/Context.scala @@ -13,8 +13,6 @@ case class Context(environmentVariables: Map[String, String], workingDir: String def printEnvironment(): Unit = console.printEnvironment(environmentVariables.asJava) def getAbsoluteWorkingDir: String = Paths.get("").toAbsolutePath.resolve(workingDir).toString - -// def this(context: java.util.Map[String, Object]) = this(context.asScala) } object Context { diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala index 3cbc135..4e87e9c 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala @@ -12,6 +12,13 @@ import com.indix.gocd.utils.store.S3ArtifactStore.getS3client import scala.collection.JavaConversions._ +/** + * Lists objects, finds latest in s3 bucket + * + * @param client + * @param bucket + * @param storageClass + */ class S3ArtifactStore(client: AmazonS3 = getS3client(new GoEnvironment()), bucket: String, storageClass: StorageClass = StorageClass.Standard) { def withStorageClass(storageClass: String): S3ArtifactStore = { @@ -24,6 +31,12 @@ class S3ArtifactStore(client: AmazonS3 = getS3client(new GoEnvironment()), bucke } } + /** + * Puts object request in client + * + * @param putObjectRequest + * @return + */ def put(putObjectRequest: PutObjectRequest): PutObjectResult = { putObjectRequest.setStorageClass(this.storageClass) client.putObject(putObjectRequest) @@ -39,6 +52,13 @@ class S3ArtifactStore(client: AmazonS3 = getS3client(new GoEnvironment()), bucke def pathString(pathOnS3: String): String = String.format("s3://%s/%s", bucket, pathOnS3) + /** + * Gets objects from path and puts in the destination + * + * @param from + * @param to + * @return + */ def get(from: String, to: String): ObjectMetadata = { val getObjectRequest = new GetObjectRequest(bucket, from) val destinationFile = new File(to) From c1b9d32caabca72146e9917c52c8ee6aaed59b80 Mon Sep 17 00:00:00 2001 From: Rucha Date: Wed, 2 Jan 2019 10:53:22 +0530 Subject: [PATCH 5/5] documentation --- util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala | 2 +- .../main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala index a55d574..cd4e8c0 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/models/Revision.scala @@ -2,7 +2,7 @@ package com.indix.gocd.models /** * Represents revision - * Parses revision string to pupulate fields + * Parses revision string to populate fields * * @param revision */ diff --git a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala index 4e87e9c..7af4a2c 100644 --- a/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala +++ b/util-gocd/src/main/scala/com/indix/gocd/utils/store/S3ArtifactStore.scala @@ -13,7 +13,7 @@ import com.indix.gocd.utils.store.S3ArtifactStore.getS3client import scala.collection.JavaConversions._ /** - * Lists objects, finds latest in s3 bucket + * Lists objects, finds latest, gets objects from S3 * * @param client * @param bucket