diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cd62d2e --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ + +*.class +.*.swp +.beamer +# Package Files # +*.jar +*.war +*.ear +*.versionsBackup + +# Intellij Files & Dir # +**/*.iml +*.ipr +*.iws +atlassian-ide-plugin.xml +out/ +.DS_Store +./lib/ +.idea + +# Gradle Files & Dir # +build/ +.gradle/ +.stickyStorage +.build/ +target/ + +# Node log +npm-*.log +logs/ + +# Singlenode and test data files. +/templates/ +/data/ +/data-fabric-tests/data/ + +# ANTLR4 +/core/gen +*.tokens +DirectivesLexer.java +DirectivesParser.java +DirectivesBaseListener.java +DirectivesBaseVisitor.java +DirectivesListener.java +DirectivesVisitor.java + +# generated by docs build +*.py + +# Remove release.properties +release.properties + +# Remove dev directory. +dev \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..04c6eda --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,397 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/contrib/pom.xml b/contrib/pom.xml new file mode 100644 index 0000000..1c8fba3 --- /dev/null +++ b/contrib/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + io.cdap.plugin + marketo-entity-generator + 1.0.0-SNAPSHOT + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + 6.1.0-SNAPSHOT + + + + + io.cdap.cdap + cdap-api-common + ${cdap.version} + + + com.google.guava + guava + 28.1-jre + + + com.squareup + javapoet + 1.11.1 + + + io.cdap.plugin + marketo-entity-plugin + system + 1.0.0-SNAPSHOT + ${basedir}/../target/marketo-entity-plugin-1.0.0-SNAPSHOT.jar + + + \ No newline at end of file diff --git a/contrib/src/main/java/io/cdap/plugin/generate/Generate.java b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java new file mode 100644 index 0000000..90642fb --- /dev/null +++ b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java @@ -0,0 +1,445 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.generate; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.reflect.ClassPath; +import com.squareup.javapoet.ClassName; +import com.squareup.javapoet.FieldSpec; +import com.squareup.javapoet.JavaFile; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.TypeSpec; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import javax.lang.model.element.Modifier; + +/** + * Class that generates schema for entities. + * This class is not required anymore in near future and not intended to be used by end users... + * ...but can be used later again if some massive changes to schema will be required. + *

+ * Requires guava. + */ +public class Generate { + public static void main(String... args) throws IOException { + List methods = new ArrayList<>(); + List withoutPaging = getResponseClasses().stream() + .filter(Generate::isNotPaged) + .map(Generate::getItemClsForResponseCls) + .map(aClass -> "EntityType."+aClass.getSimpleName().toUpperCase()) + .collect(Collectors.toList()); + + FieldSpec withoutPagingField = FieldSpec.builder(ClassName.bestGuess("List"), "ENTITIES_WITHOUT_PAGING") + .addModifiers(Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL) + .initializer("$T.of(\n$L\n)", ImmutableList.class, String.join(",\n", withoutPaging)) + .build(); + + MethodSpec supportPagingMethod = MethodSpec.methodBuilder("supportPaging") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(boolean.class) + .addParameter(ClassName.bestGuess("EntityType"), "entityType") + .addStatement("return !ENTITIES_WITHOUT_PAGING.contains(entityType)") + .build(); + + MethodSpec.Builder iteratorForEntityTypeBuilder = MethodSpec.methodBuilder("iteratorForEntityType") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Iterator.class) + .addParameter(Marketo.class, "marketo") + .addParameter(ClassName.bestGuess("EntityType"), "entityType") + .addParameter(int.class, "offset") + .addParameter(int.class, "maxResults") + .beginControlFlow("switch (entityType)"); + + getResponseClasses().forEach(aClass -> { + ParameterizedType baseT = (ParameterizedType) aClass.getGenericSuperclass(); + Class responseItemClass = (Class) baseT.getActualTypeArguments()[0]; + iteratorForEntityTypeBuilder.beginControlFlow("case $L:", + responseItemClass.getSimpleName().toUpperCase()); + iteratorForEntityTypeBuilder.addStatement("return marketo.iteratePage(\n$S,\n $T.class,\n $T::getResult,\n " + + "$T.of(\n\"maxReturn\",\n Integer.toString(maxResults),\n" + + " \"offset\",\n Integer.toString(offset)\n)\n)", + getFetchUrl(aClass), + aClass, aClass, + ImmutableMap.class); + iteratorForEntityTypeBuilder.endControlFlow(); + }); + iteratorForEntityTypeBuilder.beginControlFlow("default:"); + iteratorForEntityTypeBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityType.getValue())"); + iteratorForEntityTypeBuilder.endControlFlow(); + iteratorForEntityTypeBuilder.endControlFlow(); + + methods.add(iteratorForEntityTypeBuilder.build()); + + MethodSpec.Builder schemaForEntityNameBuilder = MethodSpec.methodBuilder("schemaForEntityName") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Schema.class) + .addParameter(String.class, "entityName"); + + schemaForEntityNameBuilder.beginControlFlow("switch(entityName)"); + getEntityClasses().forEach(aClass -> { + schemaForEntityNameBuilder.beginControlFlow("case $S:", aClass.getSimpleName()); + schemaForEntityNameBuilder.addStatement("return $L()", generateSchemaGetterMethod(aClass)); + schemaForEntityNameBuilder.endControlFlow(); + }); + schemaForEntityNameBuilder.beginControlFlow("default:"); + schemaForEntityNameBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityName)"); + schemaForEntityNameBuilder.endControlFlow(); + schemaForEntityNameBuilder.endControlFlow(); + + methods.add(schemaForEntityNameBuilder.build()); + + // individual schemas + getEntityClasses().forEach(aClass -> { + MethodSpec.Builder entitySchemaGenBuilder = MethodSpec.methodBuilder(generateSchemaGetterMethod(aClass)) + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Schema.class) + .addStatement("$T fields = new $T<>()", List.class, ArrayList.class); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + entitySchemaGenBuilder.addStatement("fields.add(Schema.Field.of($S, $L))", field.getName(), + simpleTypeToCdapType(field)); + } else if (isComplexType(field)) { + entitySchemaGenBuilder.addStatement( + "fields.add(Schema.Field.of($S, Schema.nullableOf(EntityHelper.$L())))", + field.getName(), generateSchemaGetterMethod(field.getType())); + } else if (isListType(field)) { + entitySchemaGenBuilder.addStatement( + "fields.add(Schema.Field.of($S, $L))", + field.getName(), listTypeToCdapType(field)); + } else { + throw new RuntimeException("Unknown field type."); + } + } + entitySchemaGenBuilder.addStatement( + "return Schema.recordOf(\"Schema\" + UUID.randomUUID().toString().replace(\"-\", \"\"), fields)"); + methods.add(entitySchemaGenBuilder.build()); + }); + + // entity object to structured record + MethodSpec.Builder recordFromEntityBuilder = MethodSpec.methodBuilder("structuredRecordFromEntity") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(StructuredRecord.class) + .addParameter(String.class, "entityName") + .addParameter(Object.class, "entity") + .addParameter(Schema.class, "schema") + .addStatement("StructuredRecord.Builder builder = StructuredRecord.builder(schema)") + .beginControlFlow("switch(entityName)"); + getEntityClasses().forEach(aClass -> { + recordFromEntityBuilder.beginControlFlow("case $S:", aClass.getSimpleName()); + recordFromEntityBuilder.addStatement("$T $L = ($L) entity", + aClass, aClass.getSimpleName().toLowerCase(), + aClass.getSimpleName()); + + recordFromEntityBuilder.beginControlFlow("if (entity == null)"); + recordFromEntityBuilder.addStatement("break"); + recordFromEntityBuilder.endControlFlow(); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + recordFromEntityBuilder.addStatement("builder.set($S, $L.$L())", + field.getName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass)); + } else if (isComplexType(field)) { + recordFromEntityBuilder.addStatement("builder.set($S, EntityHelper.structuredRecordFromEntity(\n" + + " $S,\n" + + " $L.$L(),\n" + + " schema.getField($S).getSchema().getNonNullable()))", + field.getName(), + field.getType().getSimpleName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass), + field.getName()); + } else if (isListType(field)) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (isSimpleType(listType)) { + recordFromEntityBuilder.addStatement("builder.set($S, $L.$L())", + field.getName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass)); + } else { + recordFromEntityBuilder.addStatement( + "builder.set(\n" + + " $S,\n" + + " $L.$L().stream()\n" + + " .map(ent -> EntityHelper.structuredRecordFromEntity(\n" + + " $S,\n" + + " ent,\n" + + " schema.getField($S).getSchema().getNonNullable().getComponentSchema()))\n" + + " .collect(Collectors.toList())\n" + + ")", + field.getName(), aClass.getSimpleName().toLowerCase(), findGetterMethod(field, aClass), + listType.getSimpleName(), field.getName() + ); + } + } else { + throw new RuntimeException("Unknown field type."); + } + } + recordFromEntityBuilder.addStatement("break"); + recordFromEntityBuilder.endControlFlow(); + }); + + recordFromEntityBuilder.beginControlFlow("default:"); + recordFromEntityBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityName)"); + recordFromEntityBuilder.endControlFlow(); + recordFromEntityBuilder.endControlFlow(); // switch statement + recordFromEntityBuilder.addStatement("return builder.build()"); + methods.add(recordFromEntityBuilder.build()); + + TypeSpec.Builder entityTypeEnumBuilder = TypeSpec.enumBuilder("EntityType") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .addField(String.class, "value", Modifier.PRIVATE, Modifier.FINAL); + + getTopLevelEntityClasses().forEach(aClass -> { + entityTypeEnumBuilder.addEnumConstant( + aClass.getSimpleName().toUpperCase(), + TypeSpec.anonymousClassBuilder("$S", aClass.getSimpleName()).build() + ); + }); + + entityTypeEnumBuilder.addMethod( + MethodSpec.constructorBuilder() + .addParameter(String.class, "value") + .addStatement("this.$N = $N", "value", "value") + .build() + ); + + entityTypeEnumBuilder.addMethod( + MethodSpec.methodBuilder("getValue") + .addModifiers(Modifier.PUBLIC) + .returns(String.class) + .addStatement("return value") + .build() + ); + + entityTypeEnumBuilder.addMethod( + MethodSpec.methodBuilder("fromString") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(ClassName.bestGuess("EntityType")) + .addParameter(String.class, "value") + .beginControlFlow("for (EntityType entityType : EntityType.values())") + .beginControlFlow("if (entityType.value.equals(value))") + .addStatement("return entityType") + .endControlFlow() + .endControlFlow() + .addStatement("throw new IllegalArgumentException(\"Unknown entity type type: \" + value)") + .build() + ); + + TypeSpec schemaHelperSpec = TypeSpec.classBuilder("EntityHelper") + .addModifiers(Modifier.PUBLIC, Modifier.FINAL) + .addField(withoutPagingField) + .addMethod(supportPagingMethod) + .addMethods(methods) + .addType(entityTypeEnumBuilder.build()) + .build(); + + JavaFile javaFile = JavaFile.builder("io.cdap.plugin.marketo.source.batch.entity", schemaHelperSpec) + .skipJavaLangImports(true) + .build(); + System.out.println(javaFile.toString()); + } + + private static final List SIMPLE_TYPES = ImmutableList.of(Boolean.class, Integer.class, int.class, + boolean.class, String.class); + + private static final List COLLECTION_TYPES = ImmutableList.of(List.class); + + public static boolean isSimpleType(Field field) { + return SIMPLE_TYPES.contains(field.getType()); + } + + public static boolean isSimpleType(Class cls) { + return SIMPLE_TYPES.contains(cls); + } + + public static String generateSchemaGetterMethod(Class cls) { + return "get" + capitalize(cls.getSimpleName()) + "Schema"; + } + + public static boolean isListType(Field field) { + return COLLECTION_TYPES.contains(field.getType()); + } + + public static boolean isComplexType(Field field) { + return !SIMPLE_TYPES.contains(field.getType()) && !COLLECTION_TYPES.contains(field.getType()); + } + + public static String simpleTypeToCdapType(Field field) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))"; + } else if (field.getType().equals(String.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.STRING))"; + } else if (field.getType().equals(Integer.class) || field.getType().equals(int.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.INT))"; + } else { + throw new RuntimeException("Unsupported simple type"); + } + } + + public static String listTypeToCdapType(Field field) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (listType.equals(Boolean.class) || listType.equals(boolean.class)) { + return "SSchema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))"; + } else if (listType.equals(String.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING)))"; + } else if (listType.equals(Integer.class) || listType.equals(int.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.INT)))"; + } else { + return "Schema.nullableOf(Schema.arrayOf(EntityHelper." + generateSchemaGetterMethod(listType) + "()))"; + } + } + + public static String findGetterMethod(Field field, Class cls) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException e) { + if (field.getName().startsWith("is")) { + String alternateGetter = "get" + capitalize(field.getName().substring(2)); + try { + cls.getMethod(alternateGetter); + return alternateGetter; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + } + } else { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + throw new RuntimeException(); + } + + public static String capitalize(String str) { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } + + public static List getEntityClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isEntity) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + + public static List getTopLevelEntityClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isTopLevelEntity) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + + public static List getResponseClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isResponses) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + + public static boolean isEntity(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Entity.class)) { + return true; + } + } + return false; + } + + public static boolean isResponses(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + return true; + } + } + return false; + } + + public static String getFetchUrl(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + Response r = (Response) a; + return r.fetchUrl(); + } + } + throw new RuntimeException(); + } + + public static boolean getPaged(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + Response r = (Response) a; + return r.paged(); + } + } + throw new RuntimeException(); + } + + public static boolean isNotPaged(Class cls) { + return !getPaged(cls); + } + + public static Class getItemClsForResponseCls(Class cls) { + ParameterizedType baseT = (ParameterizedType) cls.getGenericSuperclass(); + Class responseItemClass = (Class) baseT.getActualTypeArguments()[0]; + return responseItemClass; + } + + public static boolean isTopLevelEntity(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Entity.class)) { + Entity e = (Entity) a; + if (e.topLevel()) { + return true; + } + } + } + return false; + } +} diff --git a/docs/MarketoEntityPlugin-batchsource.md b/docs/MarketoEntityPlugin-batchsource.md new file mode 100644 index 0000000..4db2a24 --- /dev/null +++ b/docs/MarketoEntityPlugin-batchsource.md @@ -0,0 +1,36 @@ +# Marketo Entity Batch Source + +Description +----------- +This plugin is used to query Leads or Activities entities for specified date range from Marketo. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Rest API endpoint:** Marketo rest API endpoint, unique for each client. + +**Entity Type:** Type of entity. +### Authentication + +**Client ID:** Client ID. + +**Client Secret:** Client secret. + +### Advanced + +**Splits Count:** Parallel splits count. + +**Max Results Per Page:** Maximum number of records to retrieve by single request. + +API limits notes +---------------- +All entities will be fetched at once. + +Total requests count depends on **Max Results Per Page**, leave it with default +value(200 - maximum possible value) to minimize risk of reaching limits. + +Default request limit is 50000 per day, this will allow to fetch ~10.000.000 entities, but you must consider this limit +if there are several pipelines or other applications that uses API. \ No newline at end of file diff --git a/docs/MarketoReportingPlugin-batchsource.md b/docs/MarketoReportingPlugin-batchsource.md new file mode 100644 index 0000000..5451214 --- /dev/null +++ b/docs/MarketoReportingPlugin-batchsource.md @@ -0,0 +1,26 @@ +# Marketo Reporting Batch Source + +Description +----------- +This plugin is used to query Leads or Activities entities for specified date range from Marketo. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Rest API endpoint:** Marketo rest API endpoint, unique for each client. +### Authentication + +**Client ID:** Client ID. + +**Client Secret:** Client secret. + +### Report + +**Report Type:** Type of report. One of 'leads' or 'activities'. + +**Start Date:** Start date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). + +**End Date:** End date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). \ No newline at end of file diff --git a/icons/MarketoEntityPlugin-batchsource.png b/icons/MarketoEntityPlugin-batchsource.png new file mode 100644 index 0000000..4abe7e7 Binary files /dev/null and b/icons/MarketoEntityPlugin-batchsource.png differ diff --git a/icons/MarketoReportingPlugin-batchsource.png b/icons/MarketoReportingPlugin-batchsource.png new file mode 100644 index 0000000..4abe7e7 Binary files /dev/null and b/icons/MarketoReportingPlugin-batchsource.png differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..a1d3cbf --- /dev/null +++ b/pom.xml @@ -0,0 +1,446 @@ + + + + 4.0.0 + + io.cdap.plugin + marketo-entity-plugin + 1.0.0-SNAPSHOT + + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + + + 6.1.0-SNAPSHOT + 2.8.0 + 4.5.9 + 2.3.0-SNAPSHOT + 2.1.3 + + + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-formats + ${cdap.version} + + + org.apache.avro + avro + + + io.thekraken + grok + + + + + io.cdap.plugin + hydrator-common + ${hydrator.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + + io.cdap.cdap + cdap-etl-api-spark + ${cdap.version} + provided + + + org.apache.spark + spark-streaming_2.11 + ${spark2.version} + provided + + + org.apache.spark + spark-core_2.11 + ${spark2.version} + provided + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.hadoop + hadoop-client + + + com.esotericsoftware.reflectasm + reflectasm + + + org.apache.curator + curator-recipes + + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.eclipse.jetty.orbit + javax.servlet + + + + net.java.dev.jets3t + jets3t + + + asm + asm + + + + + + org.apache.httpcomponents + httpclient + ${httpcomponents.version} + + + com.google.code.gson + gson + 2.8.6 + + + org.apache.commons + commons-csv + 1.7 + + + commons-io + commons-io + 2.6 + + + org.awaitility + awaitility + 4.0.1 + + + junit + junit + 4.12 + test + + + com.github.tomakehurst + wiremock-jre8-standalone + 2.25.1 + test + + + io.cdap.cdap + cdap-data-pipeline + ${cdap.version} + test + + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + + + + + + + org.apache.rat + apache-rat-plugin + 0.13 + + + org.apache.maven.doxia + doxia-core + 1.6 + + + xerces + xercesImpl + + + + + + + rat-check + validate + + check + + + + LICENSE*.txt + + *.rst + *.md + **/*.cdap + **/*.yaml + **/*.md + logs/** + .git/** + .idea/** + **/grok/patterns/** + conf/** + data/** + plugins/** + **/*.patch + **/logrotate.d/** + **/limits.d/** + **/*.json + **/*.json.template + **/MANIFEST.MF + + **/org/apache/hadoop/** + + **/resources/** + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 8.18 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + org.apache.felix + maven-bundle-plugin + 3.5.0 + true + + + <_exportcontents>io.cdap.plugin.marketo.* + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java new file mode 100644 index 0000000..bda2387 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -0,0 +1,118 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import io.cdap.plugin.marketo.common.api.entities.DateRange; +import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * Various helper methods. + */ +public class Helpers { + public static String streamToString(InputStream inputStream) { + try { + return IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(String.format("Failed to read stream completely due to '%s'", e.getMessage())); + } + } + + public static T streamToObject(InputStream inputStream, Class cls) { + return Marketo.GSON.fromJson(new InputStreamReader(inputStream), cls); + } + + public static RuntimeException failForMethodAndUri(String method, URI uri, Exception ex) { + String message = ex.getMessage(); + if (Strings.isNullOrEmpty(message)) { + if (ex.getCause() != null) { + message = ex.getCause().getMessage(); + if (Strings.isNullOrEmpty(message)) { + message = "Unknown failure"; + } + } + } + + URIBuilder uriBuilder = new URIBuilder(uri); + List queryParameters = uriBuilder.getQueryParams(); + queryParameters.removeIf(queryParameter -> queryParameter.getName().equals("access_token")); + uriBuilder.setParameters(queryParameters); + try { + String uriString = uriBuilder.build().toString(); + return new RuntimeException(String.format("Failed '%s' '%s' - '%s'", method, uriString, message)); + } catch (URISyntaxException e) { + // this will never happen since we rebuilding already validated uri, just make compiler happy + return new RuntimeException(e); + } + } + + /** + * Splits date range in 30 day ranges. + * Return date range as is if difference is less or equals to 30 days. + * + * @param beginDate + * @param endDate + * @return + */ + public static List getDateRanges(String beginDate, String endDate) { + OffsetDateTime start = OffsetDateTime.parse(beginDate); + OffsetDateTime end = OffsetDateTime.parse(endDate); + + if (start.compareTo(end) > 0) { + throw new IllegalArgumentException("Start date cannot be greater than the end date."); + } + + int compareResult = start.plusDays(30).compareTo(end); + if (compareResult >= 0) { + // we are in range of 30 days, dates are okay + return ImmutableList.of(new DateRange(start.toString(), end.toString())); + } else { + List result = new ArrayList<>(); + OffsetDateTime currentStart = start; + + while (currentStart.compareTo(end) < 0) { + OffsetDateTime nextEnd = currentStart.plusDays(30); + result.add(new DateRange(currentStart.toString(), + min(nextEnd.minusSeconds(1), end).toString())); + currentStart = nextEnd; + } + + return result; + } + } + + public static > T min(T o1, T o2) { + if (o1.compareTo(o2) < 0) { + return o1; + } else { + return o2; + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java new file mode 100644 index 0000000..c49e389 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -0,0 +1,167 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.marketo.common.api.entities.Warning; +import io.cdap.plugin.marketo.common.api.entities.WarningDeserializer; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivityTypeResponse; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribeResponse; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; +import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; +import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Class that expose marketo rest api endpoints. + */ +public class Marketo extends MarketoHttp { + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(Warning.class, new WarningDeserializer()) + .create(); + public static final List ACTIVITY_FIELDS = ImmutableList.of("marketoGUID", "leadId", "activityDate", + "activityTypeId", "campaignId", + "primaryAttributeValueId", + "primaryAttributeValue", "attributes"); + /** + * Job queue will be checked every 60 seconds. + */ + private static final long JOB_QUEUE_POLL_INTERVAL = 60; + /** + * Wait for 10 seconds before trying to enqueue job, this will minimize chance of race condition. + */ + private static final long JOB_QUEUE_POLL_DELAY = 10; + + public Marketo(String marketoEndpoint, String clientId, String clientSecret) { + super(marketoEndpoint, clientId, clientSecret); + } + + public List describeLeads() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribeResponse.class, LeadsDescribeResponse::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + + public List describeBuildInActivities() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.BUILD_IN_ACTIVITIES_TYPES, ActivityTypeResponse.class, ActivityTypeResponse::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + + public LeadsExportJob exportLeads(LeadsExportRequest request) { + LeadsExportResponse export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), + Marketo::streamToLeadsExport, + request, + GSON::toJson); + return new LeadsExportJob(export.singleExport(), this); + } + + public LeadsExport leadsExportJobStatus(String jobId) { + LeadsExportResponse currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), + Collections.emptyMap(), Marketo::streamToLeadsExport); + return currentResp.singleExport(); + } + + public ActivitiesExportJob exportActivities(ActivitiesExportRequest request) { + ActivitiesExportResponse export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), + Marketo::streamToActivitiesExport, + request, + GSON::toJson); + return new ActivitiesExportJob(export.singleExport(), this); + } + + public ActivitiesExport activitiesExportJobStatus(String jobId) { + ActivitiesExportResponse currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_ACTIVITIES_STATUS, jobId), + Collections.emptyMap(), Marketo::streamToActivitiesExport); + return currentResp.singleExport(); + } + + /** + * Waits until bulk extract queue has available slot and executes given action. + * + * @param action action to execute once slot is available + * @param timeoutSeconds timeout in seconds + */ + public void onBulkExtractQueueAvailable(Callable action, long timeoutSeconds) { + try { + Awaitility.given() + .ignoreException(TooManyJobsException.class) // ignore exception in case another reader took our slot + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .pollInterval(JOB_QUEUE_POLL_INTERVAL, TimeUnit.SECONDS) + .pollDelay(JOB_QUEUE_POLL_DELAY, TimeUnit.SECONDS) + .until(action); + } catch (ConditionTimeoutException ex) { + throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); + } + } + + public static LeadsExportResponse streamToLeadsExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, LeadsExportResponse.class); + } + + public static ActivitiesExportResponse streamToActivitiesExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, ActivitiesExportResponse.class); + } + + /** + * Check if job can be enqueued. + * + * @return true, if job can be enqueued + */ + public boolean canEnqueueJob() { + LeadsExportResponse leadsExportResponseJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToLeadsExport + ); + + int jobsInQueue = leadsExportResponseJobs.getResult().size(); + + ActivitiesExportResponse activitiesExportResponceJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToActivitiesExport + ); + jobsInQueue += activitiesExportResponceJobs.getResult().size(); + + LOG.debug("Jobs in queue: {}", jobsInQueue); + + return jobsInQueue < 10; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java new file mode 100644 index 0000000..2b4df11 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -0,0 +1,241 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Class that encapsulates common http functions for marketo rest api. + */ +class MarketoHttp { + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + private static final Gson GSON = new Gson(); + private String marketoEndpoint; + private String clientId; + private String clientSecret; + private MarketoToken token; + private HttpClientContext httpClientContext = HttpClientContext.create(); + + MarketoHttp(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + token = refreshToken(); + } + + private T getPage(String queryUrl, Class pageClass, Map parameters) { + return validatedGet(queryUrl, parameters, + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + + private T getPage(String queryUrl, Class pageClass) { + return validatedGet(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + + T getNextPage(T currentPage, String queryUrl, Class pageClass) { + if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { + return validatedGet(queryUrl, + ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + return null; + } + + public MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter, + Map parameters) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass, parameters), this, queryUrl, pageClass, + resultsGetter); + } + + public MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); + } + + T validatedGet(String queryUrl, Map parameters, + Function deserializer) { + String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return get(queryUri, deserializer); + }); + } + + public T validatedPost(String queryUrl, Map parameters, + Function deserializer, + B body, Function qSerializer) { + String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return post(queryUri, deserializer, body, qSerializer); + }); + } + + // code: 1029, message: Too many jobs (10) in queue + private T retryableValidate(String logUri, Supplier tryQuery) { + T result = tryQuery.get(); + // check for expired token + if (!result.isSuccess()) { + for (Error error : result.getErrors()) { + if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { + // refresh token and retry + token = refreshToken(); + LOG.info("Refreshed token"); + return tryQuery.get(); + } + } + } + + // log warnings if required + if (result.getWarnings().size() > 0) { + String warnings = result.getWarnings().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); + } + + if (!result.isSuccess()) { + String msg = String.format("Errors when calling '%s'", logUri); + // log errors if required + if (result.getErrors().size() > 0) { + String errors = result.getErrors().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + msg = msg + " - " + errors; + LOG.error(msg); + } + throw mapErrorsToException(result.getErrors(), msg); + } + return result; + } + + private RuntimeException mapErrorsToException(List errors, String defaultMessage) { + if (errors.size() == 1) { + Error e = errors.get(0); + String message = e.getMessage(); + if (e.getCode() == 1029 && message != null && message.contains("many jobs")) { + return new TooManyJobsException(); + } else { + // this error don't require specific handling + return new RuntimeException(defaultMessage); + } + } else { + // something outstanding happened and we have more than one error, we can't handle this in specific way + return new RuntimeException(defaultMessage); + } + } + + public T get(URI uri, Function deserializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(uri); + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + checkResponseCode(response); + return deserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForMethodAndUri("GET", uri, e); + } + } + + private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(uri); + if (body != null) { + Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); + request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); + } + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + checkResponseCode(response); + return respDeserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForMethodAndUri("POST", uri, e); + } + } + + private static void checkResponseCode(CloseableHttpResponse response) throws IOException { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode >= 300) { + String responseBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + throw new RuntimeException(String.format("Http code '%s', response '%s'", statusCode, responseBody)); + } + } + + public URI buildUri(String queryUrl, Map parameters) { + return buildUri(queryUrl, parameters, true); + } + + URI buildUri(String queryUrl, Map parameters, boolean includeToken) { + try { + URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); + parameters.forEach(builder::setParameter); + if (includeToken) { + builder.setParameter("access_token", token.getAccessToken()); + } + return builder.build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); + } + } + + MarketoToken getCurrentToken() { + return this.token; + } + + private MarketoToken refreshToken() { + LOG.debug("Requesting marketo token"); + URI getTokenUri = buildUri("/identity/oauth/token", + ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, + "client_secret", clientSecret), false); + return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java new file mode 100644 index 0000000..1de07bf --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +/** + * Marketo page iterator. + * + * @param type of page response + * @param type of page item entity + */ +public class MarketoPageIterator implements Iterator { + private T currentPage; + private MarketoHttp marketo; + private String queryUrl; + private Class pageClass; + private Function> resultsGetter; + private Iterator currentPageResultIterator; + + MarketoPageIterator(T page, MarketoHttp marketo, String queryUrl, Class pageClass, + Function> resultsGetter) { + this.currentPage = page; + this.marketo = marketo; + this.queryUrl = queryUrl; + this.pageClass = pageClass; + this.resultsGetter = resultsGetter; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + } + + @Override + public boolean hasNext() { + if (currentPageResultIterator.hasNext()) { + return true; + } else { + T nextPage = marketo.getNextPage(currentPage, queryUrl, pageClass); + if (nextPage != null) { + currentPage = nextPage; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + return hasNext(); + } else { + return false; + } + } + } + + @Override + public I next() { + if (hasNext()) { + return currentPageResultIterator.next(); + } else { + throw new NoSuchElementException(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java new file mode 100644 index 0000000..ca23321 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +/** + * Exception thrown if too many jobs already queued. + */ +public class TooManyJobsException extends RuntimeException { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java new file mode 100644 index 0000000..32479bb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +/** + * Marketo API urls. + */ +public class Urls { + public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; + public static final String BULK_EXPORT_LEADS_LIST = "/bulk/v1/leads/export.json"; + public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; + public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; + public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; + public static final String BULK_EXPORT_LEADS_FILE = "/bulk/v1/leads/export/%s/file.json"; + public static final String BULK_EXPORT_ACTIVITIES_LIST = "/bulk/v1/activities/export.json"; + public static final String BULK_EXPORT_ACTIVITIES_CREATE = "/bulk/v1/activities/export/create.json"; + public static final String BULK_EXPORT_ACTIVITIES_ENQUEUE = "/bulk/v1/activities/export/%s/enqueue.json"; + public static final String BULK_EXPORT_ACTIVITIES_STATUS = "/bulk/v1/activities/export/%s/status.json"; + public static final String BULK_EXPORT_ACTIVITIES_FILE = "/bulk/v1/activities/export/%s/file.json"; + public static final String BUILD_IN_ACTIVITIES_TYPES = "/rest/v1/activities/types.json"; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java new file mode 100644 index 0000000..2f679ca --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -0,0 +1,81 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +import java.util.Collections; +import java.util.List; + +/** + * Represents common parts for all responses. + */ +public class BaseResponse { + + private boolean success = false; + private List errors = Collections.emptyList(); + private List warnings = Collections.emptyList(); + private String requestId; + private boolean moreResult = false; + private String nextPageToken; + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public List getErrors() { + return errors; + } + + public void setErrors(List errors) { + this.errors = errors; + } + + public List getWarnings() { + return warnings; + } + + public void setWarnings(List warnings) { + this.warnings = warnings; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public boolean isMoreResult() { + return moreResult; + } + + public void setMoreResult(boolean moreResult) { + this.moreResult = moreResult; + } + + public String getNextPageToken() { + return nextPageToken; + } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java new file mode 100644 index 0000000..9e71503 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents date range. + */ +public class DateRange { + String endAt; + String startAt; + + public DateRange() { + + } + + public DateRange(String startAt, String endAt) { + this.endAt = endAt; + this.startAt = startAt; + } + + public String getEndAt() { + return endAt; + } + + public String getStartAt() { + return startAt; + } + + @Override + public String toString() { + return startAt + " -- " + endAt; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java new file mode 100644 index 0000000..594e9cb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents error message. + */ +public class Error { + private int code; + private String message; + + public Error(int code, String message) { + this.code = code; + this.message = message; + } + + public Error() { + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java new file mode 100644 index 0000000..258d93f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +import com.google.gson.annotations.SerializedName; + +/** + * Represents marketo token response. + */ +public class MarketoToken { + @SerializedName("access_token") + private String accessToken; + private String scope; + @SerializedName("expires_in") + private String expiresIn; + @SerializedName("token_type") + private String tokenType; + + public MarketoToken(String accessToken, String scope, String expiresIn, String tokenType) { + this.accessToken = accessToken; + this.scope = scope; + this.expiresIn = expiresIn; + this.tokenType = tokenType; + } + + public String getAccessToken() { + return accessToken; + } + + public String getScope() { + return scope; + } + + public String getExpiresIn() { + return expiresIn; + } + + public String getTokenType() { + return tokenType; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java new file mode 100644 index 0000000..b1167fc --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents warning message. + */ +public class Warning { + private int code; + private String message; + + public Warning(int code, String message) { + this.code = code; + this.message = message; + } + + public Warning() { + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java new file mode 100644 index 0000000..b5b013d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; + +import java.lang.reflect.Type; + +/** + * Deserializer for warning messages that can handle simple string warning messages and code+message warnings. + */ +public class WarningDeserializer implements JsonDeserializer { + @Override + public Warning deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) + throws JsonParseException { + if (jsonElement.isJsonPrimitive()) { + return new Warning(-1, jsonElement.getAsString()); + } else if (jsonElement.isJsonObject()) { + JsonObject obj = jsonElement.getAsJsonObject(); + int code = -1; + String message = ""; + + if (obj.has("code")) { + code = obj.get("code").getAsInt(); + } + + if (obj.has("message")) { + message = obj.get("message").getAsString(); + } + + return new Warning(code, message); + } else { + throw new RuntimeException("Failed to deserialize warning message: " + jsonElement.toString()); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java new file mode 100644 index 0000000..2cfc72e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +/** + * Represents export response item. + */ +public class ActivitiesExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java new file mode 100644 index 0000000..31fc681 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import java.util.List; +import java.util.Map; + +/** + * Represents activities bulk export request. + */ +public class ActivitiesExportRequest { + + Map columnHeaderNames = null; + List fields = null; + ExportActivityFilter filter = null; + String format = "CSV"; + + public ActivitiesExportRequest(List fields, ExportActivityFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java new file mode 100644 index 0000000..0b19058 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents activities bulk export response. + */ +public class ActivitiesExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public ActivitiesExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException( + String.format("Expected single export job result, but found '%s' results.", result.size())); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java new file mode 100644 index 0000000..b853484 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java @@ -0,0 +1,95 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Activity type response. + */ +public class ActivityTypeResponse extends BaseResponse { + /** + * Activity type attribute. + */ + public static class ActivityTypeAttribute { + private String apiName; + private String dataType; + private String name; + + public String getApiName() { + return apiName; + } + + public String getDataType() { + return dataType; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return String.format("%s(%s, %s)", getApiName(), getDataType(), getName()); + } + } + + /** + * Attribute type. + */ + public static class ActivityType { + private String apiName; + private List attributes; + private String description; + private Integer id; + private String name; + private ActivityTypeAttribute primaryAttribute; + + public String getApiName() { + return apiName; + } + + public List getAttributes() { + return attributes; + } + + public String getDescription() { + return description; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public ActivityTypeAttribute getPrimaryAttribute() { + return primaryAttribute; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java new file mode 100644 index 0000000..5e225cf --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.DateRange; + +import java.util.List; + +/** + * Represents request filter. + */ +public class ExportActivityFilter { + private List activityTypeIds; + private DateRange createdAt; + + /** + * Builder. + */ + public static class Builder { + + private List activityTypeIds = null; + private DateRange createdAt = null; + + public Builder() { + } + + Builder(List activityTypeIds, DateRange createdAt) { + this.activityTypeIds = activityTypeIds; + this.createdAt = createdAt; + } + + public Builder activityTypeIds(List activityTypeIds) { + this.activityTypeIds = activityTypeIds; + return Builder.this; + } + + public Builder addActivityTypeIds(Integer activityTypeIds) { + this.activityTypeIds.add(activityTypeIds); + return Builder.this; + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public ExportActivityFilter build() { + + return new ExportActivityFilter(this); + } + } + + private ExportActivityFilter(Builder builder) { + this.activityTypeIds = builder.activityTypeIds; + this.createdAt = builder.createdAt; + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java new file mode 100644 index 0000000..d7da1d2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java @@ -0,0 +1,309 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Email entity. + */ +@Entity(topLevel = true) +public class Email { + String createdAt; + String description; + FolderDescriptor folder; + EmailField fromEmail; + EmailField fromName; + Integer id; + String name; + Boolean operational; + Boolean publishToMSI; + EmailField replyEmail; + String status; + EmailField subject; + Integer template; + Boolean textOnly; + String updatedAt; + String url; + Integer version; + Boolean webView; + String workspace; + Boolean autoCopyToText; + List ccFields = Collections.emptyList(); + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public EmailField getFromEmail() { + return fromEmail; + } + + public EmailField getFromName() { + return fromName; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public Boolean getOperational() { + return operational; + } + + public Boolean getPublishToMSI() { + return publishToMSI; + } + + public EmailField getReplyEmail() { + return replyEmail; + } + + public String getStatus() { + return status; + } + + public EmailField getSubject() { + return subject; + } + + public Integer getTemplate() { + return template; + } + + public Boolean getTextOnly() { + return textOnly; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public Integer getVersion() { + return version; + } + + public Boolean getWebView() { + return webView; + } + + public String getWorkspace() { + return workspace; + } + + public Boolean getAutoCopyToText() { + return autoCopyToText; + } + + public List getCcFields() { + return ccFields; + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for Email entity. + */ + public static class Builder { + + private String createdAt; + private String description; + private FolderDescriptor folder; + private EmailField fromEmail; + private EmailField fromName; + private Integer id; + private String name; + private Boolean operational; + private Boolean publishToMSI; + private EmailField replyEmail; + private String status; + private EmailField subject; + private Integer template; + private Boolean textOnly; + private String updatedAt; + private String url; + private Integer version; + private Boolean webView; + private String workspace; + private Boolean autoCopyToText; + private List ccFields = new ArrayList<>(); + + public Builder() { + } + + public Builder createdAt(String createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder description(String description) { + this.description = description; + return Builder.this; + } + + public Builder folder(FolderDescriptor folder) { + this.folder = folder; + return Builder.this; + } + + public Builder fromEmail(EmailField fromEmail) { + this.fromEmail = fromEmail; + return Builder.this; + } + + public Builder fromName(EmailField fromName) { + this.fromName = fromName; + return Builder.this; + } + + public Builder id(Integer id) { + this.id = id; + return Builder.this; + } + + public Builder name(String name) { + this.name = name; + return Builder.this; + } + + public Builder operational(Boolean operational) { + this.operational = operational; + return Builder.this; + } + + public Builder publishToMSI(Boolean publishToMSI) { + this.publishToMSI = publishToMSI; + return Builder.this; + } + + public Builder replyEmail(EmailField replyEmail) { + this.replyEmail = replyEmail; + return Builder.this; + } + + public Builder status(String status) { + this.status = status; + return Builder.this; + } + + public Builder subject(EmailField subject) { + this.subject = subject; + return Builder.this; + } + + public Builder template(Integer template) { + this.template = template; + return Builder.this; + } + + public Builder textOnly(Boolean textOnly) { + this.textOnly = textOnly; + return Builder.this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public Builder url(String url) { + this.url = url; + return Builder.this; + } + + public Builder version(Integer version) { + this.version = version; + return Builder.this; + } + + public Builder webView(Boolean webView) { + this.webView = webView; + return Builder.this; + } + + public Builder workspace(String workspace) { + this.workspace = workspace; + return Builder.this; + } + + public Builder autoCopyToText(Boolean autoCopyToText) { + this.autoCopyToText = autoCopyToText; + return Builder.this; + } + + public Builder ccFields(List ccFields) { + this.ccFields = ccFields; + return Builder.this; + } + + public Builder addCcFields(EmailCCField ccFields) { + this.ccFields.add(ccFields); + return Builder.this; + } + + public Email build() { + + return new Email(this); + } + } + + private Email(Builder builder) { + this.createdAt = builder.createdAt; + this.description = builder.description; + this.folder = builder.folder; + this.fromEmail = builder.fromEmail; + this.fromName = builder.fromName; + this.id = builder.id; + this.name = builder.name; + this.operational = builder.operational; + this.publishToMSI = builder.publishToMSI; + this.replyEmail = builder.replyEmail; + this.status = builder.status; + this.subject = builder.subject; + this.template = builder.template; + this.textOnly = builder.textOnly; + this.updatedAt = builder.updatedAt; + this.url = builder.url; + this.version = builder.version; + this.webView = builder.webView; + this.workspace = builder.workspace; + this.autoCopyToText = builder.autoCopyToText; + this.ccFields = builder.ccFields; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java new file mode 100644 index 0000000..d59a89c --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Email CC field. + */ +@Entity +public class EmailCCField { + String attributeId; + String objectName; + String displayName; + String apiName; + + public EmailCCField() { + } + + public EmailCCField(String attributeId, String objectName, String displayName, String apiName) { + this.attributeId = attributeId; + this.objectName = objectName; + this.displayName = displayName; + this.apiName = apiName; + } + + public String getAttributeId() { + return attributeId; + } + + public String getObjectName() { + return objectName; + } + + public String getDisplayName() { + return displayName; + } + + public String getApiName() { + return apiName; + } + + @Override + public String toString() { + return "EmailCCField{" + + "attributeId='" + attributeId + '\'' + + ", objectName='" + objectName + '\'' + + ", displayName='" + displayName + '\'' + + ", apiName='" + apiName + '\'' + + '}'; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java new file mode 100644 index 0000000..5cf78b7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Email field. + */ +@Entity +public class EmailField { + String type; + String value; + + public String getType() { + return type; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java new file mode 100644 index 0000000..ef8fa78 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/emails.json + */ +@Response(fetchUrl = "/rest/asset/v1/emails.json") +public class EmailResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java new file mode 100644 index 0000000..b54e3f7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Email template entity. + */ +@Entity(topLevel = true) +public class EmailTemplate { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + Integer version; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public Integer getVersion() { + return version; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java new file mode 100644 index 0000000..0ba9987 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/emailTemplates.json + */ +@Response(fetchUrl = "/rest/asset/v1/emailTemplates.json") +public class EmailTemplateResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java new file mode 100644 index 0000000..3e54994 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * File entity. + */ +@Entity(topLevel = true) +public class File { + String createdAt; + String description; + FileFolder folder; + Integer id; + String mimeType; + String name; + Integer size; + String updatedAt; + String url; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FileFolder getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getMimeType() { + return mimeType; + } + + public String getName() { + return name; + } + + public Integer getSize() { + return size; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java new file mode 100644 index 0000000..9dec989 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * File folder descriptor. + */ +@Entity +public class FileFolder { + Integer id; + String name; + String type; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java new file mode 100644 index 0000000..f6eed56 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/files.json + */ +@Response(fetchUrl = "/rest/asset/v1/files.json") +public class FileResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java new file mode 100644 index 0000000..84eb790 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java @@ -0,0 +1,96 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Folder entity. + */ +@Entity(topLevel = true) +public class Folder { + Integer accessZoneId; + String createdAt; + String description; + FolderDescriptor folderId; + String folderType; + Integer id; + Boolean isArchive; + Boolean isSystem; + String name; + FolderDescriptor parent; + String path; + String updatedAt; + String url; + String workspace; + + public Integer getAccessZoneId() { + return accessZoneId; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolderId() { + return folderId; + } + + public String getFolderType() { + return folderType; + } + + public Integer getId() { + return id; + } + + public Boolean getArchive() { + return isArchive; + } + + public Boolean getSystem() { + return isSystem; + } + + public String getName() { + return name; + } + + public FolderDescriptor getParent() { + return parent; + } + + public String getPath() { + return path; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java new file mode 100644 index 0000000..6d8d5b7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Folder descriptor. + */ +@Entity +public class FolderDescriptor { + String id; + String type; + String folderName; + + public FolderDescriptor(String id, String type, String folderName) { + this.id = id; + this.type = type; + this.folderName = folderName; + } + + public String getId() { + return id; + } + + public String getType() { + return type; + } + + public String getFolderName() { + return folderName; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java new file mode 100644 index 0000000..efac548 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/folders.json + */ +@Response(fetchUrl = "/rest/asset/v1/folders.json") +public class FolderResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java new file mode 100644 index 0000000..3893896 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java @@ -0,0 +1,129 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.util.Collections; +import java.util.List; + +/** + * Form entity. + */ +@Entity(topLevel = true) +public class Form { + String buttonLabel; + Integer buttonLocation; + String createdAt; + String description; + FolderDescriptor folder; + String fontFamily; + String fontSize; + Integer id; + FormKnownVisitorDTO knownVisitor; + String labelPosition; + String language; + String locale; + String name; + Boolean progressiveProfiling; + String status; + List thankYouList = Collections.emptyList(); + String theme; + String updatedAt; + String url; + String waitingLabel; + + public String getButtonLabel() { + return buttonLabel; + } + + public Integer getButtonLocation() { + return buttonLocation; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getFontFamily() { + return fontFamily; + } + + public String getFontSize() { + return fontSize; + } + + public Integer getId() { + return id; + } + + public FormKnownVisitorDTO getKnownVisitor() { + return knownVisitor; + } + + public String getLabelPosition() { + return labelPosition; + } + + public String getLanguage() { + return language; + } + + public String getLocale() { + return locale; + } + + public String getName() { + return name; + } + + public Boolean getProgressiveProfiling() { + return progressiveProfiling; + } + + public String getStatus() { + return status; + } + + public List getThankYouList() { + return thankYouList; + } + + public String getTheme() { + return theme; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWaitingLabel() { + return waitingLabel; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java new file mode 100644 index 0000000..a268fac --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java @@ -0,0 +1,188 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Form filed entity. + */ +@Entity(topLevel = true) +public class FormField { + String dataType; + String defaultValue; + String description; + String fieldMaskValues; + Integer fieldWidth; + String id; + Boolean initiallyChecked; + Boolean isLabelToRight; + Boolean isMultiselect; + Boolean isRequired; + Integer labelWidth; + Integer maxLength; + String maximumNumber; + String minimumNumber; + String picklistValues; + String placeholderText; + String validationMessage; + Integer visibleRows; + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getFieldMaskValues() { + return fieldMaskValues; + } + + public void setFieldMaskValues(String fieldMaskValues) { + this.fieldMaskValues = fieldMaskValues; + } + + public Integer getFieldWidth() { + return fieldWidth; + } + + public void setFieldWidth(Integer fieldWidth) { + this.fieldWidth = fieldWidth; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Boolean getInitiallyChecked() { + return initiallyChecked; + } + + public void setInitiallyChecked(Boolean initiallyChecked) { + this.initiallyChecked = initiallyChecked; + } + + public Boolean getLabelToRight() { + return isLabelToRight; + } + + public void setLabelToRight(Boolean labelToRight) { + isLabelToRight = labelToRight; + } + + public Boolean getMultiselect() { + return isMultiselect; + } + + public void setMultiselect(Boolean multiselect) { + isMultiselect = multiselect; + } + + public Boolean getRequired() { + return isRequired; + } + + public void setRequired(Boolean required) { + isRequired = required; + } + + public Integer getLabelWidth() { + return labelWidth; + } + + public void setLabelWidth(Integer labelWidth) { + this.labelWidth = labelWidth; + } + + public Integer getMaxLength() { + return maxLength; + } + + public void setMaxLength(Integer maxLength) { + this.maxLength = maxLength; + } + + public String getMaximumNumber() { + return maximumNumber; + } + + public void setMaximumNumber(String maximumNumber) { + this.maximumNumber = maximumNumber; + } + + public String getMinimumNumber() { + return minimumNumber; + } + + public void setMinimumNumber(String minimumNumber) { + this.minimumNumber = minimumNumber; + } + + public String getPicklistValues() { + return picklistValues; + } + + public void setPicklistValues(String picklistValues) { + this.picklistValues = picklistValues; + } + + public String getPlaceholderText() { + return placeholderText; + } + + public void setPlaceholderText(String placeholderText) { + this.placeholderText = placeholderText; + } + + public String getValidationMessage() { + return validationMessage; + } + + public void setValidationMessage(String validationMessage) { + this.validationMessage = validationMessage; + } + + public Integer getVisibleRows() { + return visibleRows; + } + + public void setVisibleRows(Integer visibleRows) { + this.visibleRows = visibleRows; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java new file mode 100644 index 0000000..ff406bc --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/form/fields.json + */ +@Response(fetchUrl = "/rest/asset/v1/form/fields.json") +public class FormFieldsResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java new file mode 100644 index 0000000..aba9e2e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java @@ -0,0 +1,36 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Known visitor behavior for the form. + */ +@Entity +public class FormKnownVisitorDTO { + String template; + String type; + + public String getTemplate() { + return template; + } + + public String getType() { + return type; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java new file mode 100644 index 0000000..6351369 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/forms.json + */ +@Response(fetchUrl = "/rest/asset/v1/forms.json") +public class FormResponse extends SimpleBaseResponse

{ + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java new file mode 100644 index 0000000..8ee904f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java @@ -0,0 +1,61 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.util.Collections; +import java.util.List; + +/** + * Thank you page behaviors for the form. + */ +@Entity +public class FormThankYouPageDTO { + @SerializedName("default") + Boolean isDefault; + String followupType; + String followupValue; + String operator; + String subjectField; + List values = Collections.emptyList(); + + public Boolean getDefault() { + return isDefault; + } + + public String getFollowupType() { + return followupType; + } + + public String getFollowupValue() { + return followupValue; + } + + public String getOperator() { + return operator; + } + + public String getSubjectField() { + return subjectField; + } + + public List getValues() { + return values; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java new file mode 100644 index 0000000..f12e8fa --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java @@ -0,0 +1,116 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Landing page entity. + */ +@Entity(topLevel = true) +public class LandingPage { + String url; + String computedUrl; + String createdAt; + String customHeadHTML; + String description; + String facebookOgTags; + FolderDescriptor folder; + Boolean formPrefill; + Integer id; + String keywords; + Boolean mobileEnabled; + String name; + String robots; + String status; + Integer template; + String title; + String updatedAt; + String workspace; + + public String getUrl() { + return url; + } + + public String getComputedUrl() { + return computedUrl; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getCustomHeadHTML() { + return customHeadHTML; + } + + public String getDescription() { + return description; + } + + public String getFacebookOgTags() { + return facebookOgTags; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Boolean getFormPrefill() { + return formPrefill; + } + + public Integer getId() { + return id; + } + + public String getKeywords() { + return keywords; + } + + public Boolean getMobileEnabled() { + return mobileEnabled; + } + + public String getName() { + return name; + } + + public String getRobots() { + return robots; + } + + public String getStatus() { + return status; + } + + public Integer getTemplate() { + return template; + } + + public String getTitle() { + return title; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java new file mode 100644 index 0000000..c577992 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/landingPages.json + */ +@Response(fetchUrl = "/rest/asset/v1/landingPages.json") +public class LandingPageResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java new file mode 100644 index 0000000..ea692cb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java @@ -0,0 +1,125 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Landing page template entity. + */ +@Entity(topLevel = true) +public class LandingPageTemplate { + String createdAt; + String description; + Boolean enableMunchkin; + FolderDescriptor folder; + Integer id; + String name; + String status; + String templateType; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Boolean getEnableMunchkin() { + return enableMunchkin; + } + + public void setEnableMunchkin(Boolean enableMunchkin) { + this.enableMunchkin = enableMunchkin; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public void setFolder(FolderDescriptor folder) { + this.folder = folder; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getTemplateType() { + return templateType; + } + + public void setTemplateType(String templateType) { + this.templateType = templateType; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(String updatedAt) { + this.updatedAt = updatedAt; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getWorkspace() { + return workspace; + } + + public void setWorkspace(String workspace) { + this.workspace = workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java new file mode 100644 index 0000000..47fbfca --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/landingPageTemplates.json + */ +@Response(fetchUrl = "/rest/asset/v1/landingPageTemplates.json") +public class LandingPageTemplateResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java new file mode 100644 index 0000000..4dbb36d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java @@ -0,0 +1,207 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Program entity. + */ +@Entity(topLevel = true) +public class Program { + String channel; + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String sfdcId; + String sfdcName; + String status; + String type; + String updatedAt; + String url; + String workspace; + + public Program() { + } + + public String getChannel() { + return channel; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getSfdcId() { + return sfdcId; + } + + public String getSfdcName() { + return sfdcName; + } + + public String getStatus() { + return status; + } + + public String getType() { + return type; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for Program entity. + */ + public static class Builder { + + private String channel; + private String createdAt; + private String description; + private FolderDescriptor folder; + private Integer id; + private String name; + private String sfdcId; + private String sfdcName; + private String status; + private String type; + private String updatedAt; + private String url; + private String workspace; + + private Builder() { + } + + public Builder channel(String channel) { + this.channel = channel; + return Builder.this; + } + + public Builder createdAt(String createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder description(String description) { + this.description = description; + return Builder.this; + } + + public Builder folder(FolderDescriptor folder) { + this.folder = folder; + return Builder.this; + } + + public Builder id(Integer id) { + this.id = id; + return Builder.this; + } + + public Builder name(String name) { + this.name = name; + return Builder.this; + } + + public Builder sfdcId(String sfdcId) { + this.sfdcId = sfdcId; + return Builder.this; + } + + public Builder sfdcName(String sfdcName) { + this.sfdcName = sfdcName; + return Builder.this; + } + + public Builder status(String status) { + this.status = status; + return Builder.this; + } + + public Builder type(String type) { + this.type = type; + return Builder.this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public Builder url(String url) { + this.url = url; + return Builder.this; + } + + public Builder workspace(String workspace) { + this.workspace = workspace; + return Builder.this; + } + + public Program build() { + + return new Program(this); + } + } + + private Program(Builder builder) { + this.channel = builder.channel; + this.createdAt = builder.createdAt; + this.description = builder.description; + this.folder = builder.folder; + this.id = builder.id; + this.name = builder.name; + this.sfdcId = builder.sfdcId; + this.sfdcName = builder.sfdcName; + this.status = builder.status; + this.type = builder.type; + this.updatedAt = builder.updatedAt; + this.url = builder.url; + this.workspace = builder.workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java new file mode 100644 index 0000000..eb827a5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/programs.json + */ +@Response(fetchUrl = "/rest/asset/v1/programs.json") +public class ProgramResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java new file mode 100644 index 0000000..0ca6eb1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.util.Collections; +import java.util.List; + +/** + * Recurrence descriptor. + */ +@Entity +public class Recurrence { + String startAt; + String endAt; + String intervalType; + Integer interval; + Boolean weekdayOnly; + List weekdayMask = Collections.emptyList(); + Integer dayOfMonth; + Integer dayOfWeek; + Integer weekOfMonth; + + public String getStartAt() { + return startAt; + } + + public String getEndAt() { + return endAt; + } + + public String getIntervalType() { + return intervalType; + } + + public Integer getInterval() { + return interval; + } + + public Boolean getWeekdayOnly() { + return weekdayOnly; + } + + public List getWeekdayMask() { + return weekdayMask; + } + + public Integer getDayOfMonth() { + return dayOfMonth; + } + + public Integer getDayOfWeek() { + return dayOfWeek; + } + + public Integer getWeekOfMonth() { + return weekOfMonth; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java new file mode 100644 index 0000000..0dcbeb7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Segmentation entity. + */ +@Entity(topLevel = true) +public class Segmentation { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java new file mode 100644 index 0000000..e91ae07 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/segmentation.json + * NO PAGING + */ +@Response(fetchUrl = "/rest/asset/v1/segmentation.json", paged = false) +public class SegmentationResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java new file mode 100644 index 0000000..e7c3138 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Base response that contains only additional results in 'result' field. + * @param + */ +public class SimpleBaseResponse extends BaseResponse { + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java new file mode 100644 index 0000000..d6e9a8d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * SmartCampaign entity. + */ +@Entity(topLevel = true) +public class SmartCampaign { + Integer id; + String name; + String description; + String type; + Boolean isSystem; + Boolean isActive; + Boolean isRequestable; + Recurrence recurrence; + String qualificationRuleType; + Integer qualificationRuleInterval; + String qualificationRuleUnit; + Integer maxMembers; + Boolean isCommunicationLimitEnabled; + Integer smartListId; + Integer flowId; + FolderDescriptor folder; + String createdAt; + String updatedAt; + String workspace; + String status; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getType() { + return type; + } + + public Boolean getSystem() { + return isSystem; + } + + public Boolean getActive() { + return isActive; + } + + public Boolean getRequestable() { + return isRequestable; + } + + public Recurrence getRecurrence() { + return recurrence; + } + + public String getQualificationRuleType() { + return qualificationRuleType; + } + + public Integer getQualificationRuleInterval() { + return qualificationRuleInterval; + } + + public String getQualificationRuleUnit() { + return qualificationRuleUnit; + } + + public Integer getMaxMembers() { + return maxMembers; + } + + public Boolean getCommunicationLimitEnabled() { + return isCommunicationLimitEnabled; + } + + public Integer getSmartListId() { + return smartListId; + } + + public Integer getFlowId() { + return flowId; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getWorkspace() { + return workspace; + } + + public String getStatus() { + return status; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java new file mode 100644 index 0000000..55a3828 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/smartCampaigns.json + */ +@Response(fetchUrl = "/rest/asset/v1/smartCampaigns.json") +public class SmartCampaignsResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java new file mode 100644 index 0000000..e64bdc4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * SmartList entity. + */ +@Entity(topLevel = true) +public class SmartList { + Integer id; + String name; + String description; + String createdAt; + String updatedAt; + String url; + FolderDescriptor folder; + String workspace; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(String updatedAt) { + this.updatedAt = updatedAt; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public void setFolder(FolderDescriptor folder) { + this.folder = folder; + } + + public String getWorkspace() { + return workspace; + } + + public void setWorkspace(String workspace) { + this.workspace = workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java new file mode 100644 index 0000000..612afe4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/smartLists.json + */ +@Response(fetchUrl = "/rest/asset/v1/smartLists.json") +public class SmartListsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java new file mode 100644 index 0000000..b4dbea6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Snippet entity. + */ +@Entity(topLevel = true) +public class Snippet { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java new file mode 100644 index 0000000..0a83ccd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/snippets.json + */ +@Response(fetchUrl = "/rest/asset/v1/snippets.json") +public class SnippetsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java new file mode 100644 index 0000000..7737703 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * StaticList entity. + */ +@Entity(topLevel = true) +public class StaticList { + Integer id; + String name; + String description; + String createdAt; + String updatedAt; + String url; + FolderDescriptor folder; + String workspace; + String computedUrl; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getWorkspace() { + return workspace; + } + + public String getComputedUrl() { + return computedUrl; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java new file mode 100644 index 0000000..3cb9c29 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/staticLists.json + */ +@Response(fetchUrl = "/rest/asset/v1/staticLists.json") +public class StaticListsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java new file mode 100644 index 0000000..b001370 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +/** + * Tag entity. + */ +@Entity(topLevel = true) +public class Tag { + String applicableProgramTypes; + String required; + String tagType; + + public String getApplicableProgramTypes() { + return applicableProgramTypes; + } + + public String getRequired() { + return required; + } + + public String getTagType() { + return tagType; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java new file mode 100644 index 0000000..862aee6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + +/** + * GET /rest/asset/v1/tagTypes.json + */ +@Response(fetchUrl = "/rest/asset/v1/tagTypes.json") +public class TagsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java new file mode 100644 index 0000000..9e28c1a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset.gen; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks class as entity for contrib/Generate.java utility. + */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Entity { + boolean topLevel() default false; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java new file mode 100644 index 0000000..ae0cc4a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset.gen; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks class as response class. + */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Response { + String fetchUrl(); + boolean paged() default true; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java new file mode 100644 index 0000000..afd89ec --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads describe response. + */ +public class LeadsDescribeResponse extends BaseResponse { + /** + * Represents lead field description. + */ + public static class LeadAttribute { + String dataType; + String displayName; + int id; + int length; + LeadMapAttribute rest; + LeadMapAttribute soap; + + public String getDataType() { + return dataType; + } + + public String getDisplayName() { + return displayName; + } + + public int getId() { + return id; + } + + public int getLength() { + return length; + } + + public LeadMapAttribute getRest() { + return rest; + } + + public LeadMapAttribute getSoap() { + return soap; + } + } + + /** + * Represents leads field name. + */ + public static class LeadMapAttribute { + String name; + boolean readOnly = true; + + public String getName() { + return name; + } + + public boolean isReadOnly() { + return readOnly; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java new file mode 100644 index 0000000..f833207 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +/** + * Represents export response item. + */ +public class LeadsExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java new file mode 100644 index 0000000..e2035ba --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -0,0 +1,112 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.DateRange; + +import java.util.List; +import java.util.Map; + +/** + * Represents leads bulk export request. + */ +public class LeadsExportRequest { + /** + * Represents request filter. + */ + public static class ExportLeadFilter { + DateRange createdAt = null; + Integer smartListId = null; + String smartListName = null; + Integer staticListId = null; + Integer staticListName = null; + DateRange updatedAt = null; + + /** + * Builder for ExportLeadFilter. + */ + public static class Builder { + private DateRange createdAt = null; + private Integer smartListId = null; + private String smartListName = null; + private Integer staticListId = null; + private Integer staticListName = null; + private DateRange updatedAt = null; + + public Builder() { + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder smartListId(Integer smartListId) { + this.smartListId = smartListId; + return Builder.this; + } + + public Builder smartListName(String smartListName) { + this.smartListName = smartListName; + return Builder.this; + } + + public Builder staticListId(Integer staticListId) { + this.staticListId = staticListId; + return Builder.this; + } + + public Builder staticListName(Integer staticListName) { + this.staticListName = staticListName; + return Builder.this; + } + + public Builder updatedAt(DateRange updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public ExportLeadFilter build() { + + return new ExportLeadFilter(this); + } + } + + private ExportLeadFilter(Builder builder) { + this.createdAt = builder.createdAt; + this.smartListId = builder.smartListId; + this.smartListName = builder.smartListName; + this.staticListId = builder.staticListId; + this.staticListName = builder.staticListName; + this.updatedAt = builder.updatedAt; + } + + public static Builder builder() { + return new Builder(); + } + } + + Map columnHeaderNames = null; + List fields = null; + ExportLeadFilter filter = null; + String format = "CSV"; + + public LeadsExportRequest(List fields, ExportLeadFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java new file mode 100644 index 0000000..bb58467 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads bulk export response. + */ +public class LeadsExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public LeadsExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java new file mode 100644 index 0000000..de21ca8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -0,0 +1,138 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Base bulk export job wrapper. + * + * @param object that represents job status + */ +public abstract class AbstractBulkExportJob { + private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final String ENQUEUE_ABLE_STATUS = "Created"; + private static final String COMPLETED_STATUS = "Completed"; + + private String jobId; + private T lastState; + private Marketo marketo; + + /** + * @param jobId job id + * @param lastState last status + * @param marketo marketo api instance + */ + public AbstractBulkExportJob(String jobId, + T lastState, + Marketo marketo) { + + this.jobId = jobId; + this.lastState = lastState; + this.marketo = marketo; + getLogger().info("{} - created job '{}'", getLogPrefix(), this.jobId); + } + + public abstract Logger getLogger(); + + public abstract T getFreshState(); + + public abstract String getStateStatus(T state); + + public abstract String getFileUrlTemplate(); + + public abstract String getLogPrefix(); + + protected abstract T enqueueImpl(); + + public T getLastState() { + return lastState; + } + + public Marketo getMarketo() { + return marketo; + } + + public void waitCompletion() { + if (!WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))) { + throw new IllegalStateException("Job must be enqueued before waiting for completion."); + } + + do { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + } catch (InterruptedException e) { + throw new IllegalStateException("Failed to wait for job completion - interrupted"); + } + + T newState = getFreshState(); + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + lastState = newState; + } while (WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))); + + if (!getStateStatus(getLastState()).equals(COMPLETED_STATUS)) { + throw new IllegalStateException("Job expected to be in Completed state, but was in " + + getStateStatus(getLastState())); + } + } + + public boolean enqueue() { + if (!getStateStatus(getLastState()).equals(ENQUEUE_ABLE_STATUS)) { + throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + + getStateStatus(getLastState())); + } + + if (!marketo.canEnqueueJob()) { + return false; + } + + T newState = enqueueImpl(); + + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + + if (!(getStateStatus(newState).equals("Queued") || getStateStatus(newState).equals("Processing"))) { + throw new IllegalStateException( + String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, getStateStatus(newState))); + } + + lastState = newState; + + return true; + } + + private void logStatusChange(String oldStatus, String newStatus) { + if (!oldStatus.equals(newStatus)) { + getLogger().info("{} - job '{}' changed state '{}' -> '{}'", getLogPrefix(), jobId, oldStatus, newStatus); + } + } + + public String getFile() { + return marketo.get(marketo.buildUri(String.format(getFileUrlTemplate(), jobId), Collections.emptyMap()), + Helpers::streamToString); + } + + public String getJobId() { + return jobId; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java new file mode 100644 index 0000000..7a6c35d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Activities export job. + */ +public class ActivitiesExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(ActivitiesExportJob.class); + + public ActivitiesExportJob(ActivitiesExport lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public ActivitiesExport getFreshState() { + return getMarketo().activitiesExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(ActivitiesExport state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_ACTIVITIES_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK ACTIVITIES EXPORT"; + } + + @Override + protected ActivitiesExport enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_ACTIVITIES_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, ActivitiesExportResponse.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java new file mode 100644 index 0000000..f96587f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Leads export job. + */ +public class LeadsExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); + + public LeadsExportJob(LeadsExport lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public LeadsExport getFreshState() { + return getMarketo().leadsExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(LeadsExport state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_LEADS_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK LEADS EXPORT"; + } + + @Override + protected LeadsExport enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExportResponse.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java new file mode 100644 index 0000000..096b842 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.Helpers; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * InputFormat for mapreduce job, which provides a single split of data. + */ +public class MarketoInputFormat extends InputFormat { + private static final Gson GSON = new Gson(); + + @Override + public List getSplits(JobContext jobContext) { + Configuration conf = jobContext.getConfiguration(); + MarketoReportingSourceConfig config = GSON.fromJson( + conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON), MarketoReportingSourceConfig.class); + + return Helpers.getDateRanges(config.getStartDate(), config.getEndDate()).stream() + .map(dateRange -> new MarketoReportingSplit(dateRange.getStartAt(), dateRange.getEndAt())) + .collect(Collectors.toList()); + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + MarketoReportingSplit split = (MarketoReportingSplit) inputSplit; + return new MarketoRecordReader(split.getBeginDate(), split.getEndDate()); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java new file mode 100644 index 0000000..8a70ed1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class MarketoInputFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.reporter.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + + MarketoInputFormatProvider(MarketoReportingSourceConfig config) { + this.conf = Collections.unmodifiableMap(new HashMap() {{ + put(PROPERTY_CONFIG_JSON, gson.toJson(config)); + }}); + } + + @Override + public String getInputFormatClassName() { + return MarketoInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java new file mode 100644 index 0000000..4af14f6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -0,0 +1,138 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.DateRange; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ExportActivityFilter; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.job.AbstractBulkExportJob; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * RecordReader implementation, which reads events from Marketo api. + */ +public class MarketoRecordReader extends RecordReader> { + private static final Logger LOG = LoggerFactory.getLogger(MarketoRecordReader.class); + private static final Gson GSON = new GsonBuilder().create(); + /** + * Wait for 25 minutes for available slot in queue. + */ + private static final long JOB_ENQUEUE_TIMEOUT = 25 * 60; + private Map current = null; + private Iterator iterator = null; + private String beginDate; + private String endDate; + + public MarketoRecordReader(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; + } + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + Configuration conf = taskAttemptContext.getConfiguration(); + String configJson = conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON); + MarketoReportingSourceConfig config = GSON.fromJson(configJson, MarketoReportingSourceConfig.class); + + Marketo marketo = config.getMarketo(); + + DateRange dateRange = new DateRange(beginDate, endDate); + AbstractBulkExportJob job; + switch (config.getReportType()) { + case LEADS: + List leadsFields = config.getSchema().getFields().stream() + .map(Schema.Field::getName) + .collect(Collectors.toList()); + + LeadsExportRequest.ExportLeadFilter leadsFilter = LeadsExportRequest.ExportLeadFilter.builder() + .createdAt(dateRange) + .build(); + + job = marketo.exportLeads(new LeadsExportRequest(leadsFields, leadsFilter)); + break; + case ACTIVITIES: + ExportActivityFilter activitiesFilter = ExportActivityFilter.builder() + .createdAt(dateRange) + .build(); + + job = marketo.exportActivities(new ActivitiesExportRequest(Marketo.ACTIVITY_FIELDS, activitiesFilter)); + break; + default: + throw new IllegalArgumentException("Invalid report type " + config.getReportType()); + } + + LOG.info("BULK EXPORT JOB - job '{}' has date range '{}'", job.getJobId(), dateRange); + + // wait for 25 minutes for available slot and enqueue job + marketo.onBulkExtractQueueAvailable(job::enqueue, JOB_ENQUEUE_TIMEOUT); + + job.waitCompletion(); + + String data = job.getFile(); + CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); + iterator = parser.iterator(); + } + + @Override + public boolean nextKeyValue() { + if (iterator.hasNext()) { + current = iterator.next().toMap(); + return true; + } + return false; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public Map getCurrentValue() { + return current; + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void close() { + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java new file mode 100644 index 0000000..585814d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.hadoop.io.NullWritable; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Plugin that reads entities from Marketo api. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(MarketoReportingPlugin.NAME) +@Description("Reads Leads or Activities from Marketo.") +public class MarketoReportingPlugin extends BatchSource, StructuredRecord> { + public static final String NAME = "MarketoReportingPlugin"; + + private final MarketoReportingSourceConfig config; + + public MarketoReportingPlugin(MarketoReportingSourceConfig config) { + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + validateConfiguration(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Marketo entities", + Objects.requireNonNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new MarketoInputFormatProvider(config))); + } + + @Override + public void transform(KeyValue> input, Emitter emitter) { + emitter.emit(MarketoReportingSchemaHelper.getRecord(config.getSchema(), input.getValue())); + } + + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java new file mode 100644 index 0000000..0a578fd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Various methods to deal with schema and record. + */ +public class MarketoReportingSchemaHelper { + public static Schema getActivitySchema() { + List fields = Marketo.ACTIVITY_FIELDS.stream() + .map(s -> Schema.Field.of(s, Schema.nullableOf(Schema.of(Schema.Type.STRING)))) + .collect(Collectors.toList()); + return Schema.recordOf("activityRecord", fields); + } + + public static Schema getSchema(MarketoReportingSourceConfig config) { + switch (config.getReportType()) { + case LEADS: + List fields = config.getMarketo().describeLeads().stream().map( + leadAttribute -> { + if (leadAttribute.getRest() != null) { + return Schema.Field.of(leadAttribute.getRest().getName(), + Schema.nullableOf(Schema.of(Schema.Type.STRING))); + } else { + return null; + } + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + + return Schema.recordOf("LeadsRecord", fields); + case ACTIVITIES: + return getActivitySchema(); + } + throw new IllegalArgumentException("Failed to get schema for type " + config.getReportType()); + } + + public static StructuredRecord getRecord(Schema schema, Map fields) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + schema.getFields().forEach( + field -> { + if (fields.containsKey(field.getName())) { + builder.set(field.getName(), fields.get(field.getName())); + } + } + ); + return builder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java new file mode 100644 index 0000000..e7282aa --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -0,0 +1,199 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.net.MalformedURLException; +import java.net.URL; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; + +/** + * Provides all required configuration for reading Marketo entities. + */ +public class MarketoReportingSourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_CLIENT_ID = "clientId"; + public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; + public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; + public static final String PROPERTY_REPORT_TYPE = "reportType"; + public static final String PROPERTY_START_DATE = "startDate"; + public static final String PROPERTY_END_DATE = "endDate"; + + @Name(PROPERTY_CLIENT_ID) + @Description("Marketo Client ID.") + @Macro + protected String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Description("Marketo Client secret.") + @Macro + protected String clientSecret; + + @Name(PROPERTY_REST_API_ENDPOINT) + @Description("REST API endpoint URL.") + @Macro + protected String restApiEndpoint; + + @Name(PROPERTY_REPORT_TYPE) + @Description("Report type format, leads or activities.") + @Macro + protected String reportType; + + @Name(PROPERTY_START_DATE) + @Description("Start date for the report.") + @Macro + protected String startDate; + + @Name(PROPERTY_END_DATE) + @Description("End date for the report.") + @Macro + protected String endDate; + + private transient Schema schema = null; + private transient Marketo marketo = null; + + public MarketoReportingSourceConfig(String referenceName) { + super(referenceName); + } + + public Schema getSchema() { + if (schema == null) { + schema = MarketoReportingSchemaHelper.getSchema(this); + } + return schema; + } + + public Marketo getMarketo() { + if (marketo == null) { + marketo = new Marketo(getRestApiEndpoint(), getClientId(), getClientSecret()); + } + return marketo; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestApiEndpoint() { + return restApiEndpoint; + } + + public String getStartDate() { + return startDate; + } + + public String getEndDate() { + return endDate; + } + + public ReportType getReportType() { + return ReportType.fromString(reportType); + } + + void validate(FailureCollector failureCollector) { + IdUtils.validateReferenceName(referenceName, failureCollector); + validateDate(failureCollector); + validateReportType(failureCollector); + validateMarketoEndpoint(failureCollector); + validateSecrets(failureCollector); + } + + void validateDate(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_START_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse start date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_START_DATE); + } + } + + if (!containsMacro(PROPERTY_END_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse end date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_END_DATE); + } + } + + if (!(containsMacro(PROPERTY_START_DATE) && containsMacro(PROPERTY_END_DATE))) { + try { + OffsetDateTime start = OffsetDateTime.parse(getStartDate()); + OffsetDateTime end = OffsetDateTime.parse(getEndDate()); + + if (start.compareTo(end) > 0) { + failureCollector.addFailure("Start date cannot be greater than the end date.", "Swap dates.") + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + } + } catch (DateTimeParseException ex) { + // silently ignore parsing exceptions, we already pushed messages for malformed dates + } + } + } + + void validateReportType(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REPORT_TYPE)) { + try { + getReportType(); + } catch (IllegalArgumentException ex) { + failureCollector.addFailure(String.format("Incorrect reporting type '%s'.", reportType), + "Set reporting type to 'activities' or 'leads'.") + .withConfigProperty(PROPERTY_REPORT_TYPE); + } + } + } + + void validateSecrets(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { + failureCollector.addFailure("Client ID is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { + failureCollector.addFailure("Client Secret is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + } + + void validateMarketoEndpoint(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REST_API_ENDPOINT)) { + try { + new URL(getRestApiEndpoint()); + } catch (MalformedURLException e) { + failureCollector + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), null) + .withConfigProperty(PROPERTY_REST_API_ENDPOINT); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java new file mode 100644 index 0000000..7e12e27 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java @@ -0,0 +1,70 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A no-op split. + */ +public class MarketoReportingSplit extends InputSplit implements Writable { + private String beginDate; + private String endDate; + + public MarketoReportingSplit() { + } + + public MarketoReportingSplit(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + beginDate = dataInput.readUTF(); + endDate = dataInput.readUTF(); + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(beginDate); + dataOutput.writeUTF(endDate); + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } + + public String getBeginDate() { + return beginDate; + } + + public String getEndDate() { + return endDate; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java new file mode 100644 index 0000000..13193f0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +/** + * Represents report type. + */ +public enum ReportType { + LEADS("leads"), + ACTIVITIES("activities"); + + private String type; + + ReportType(String type) { + this.type = type; + } + + public static ReportType fromString(String reportType) { + for (ReportType rt : ReportType.values()) { + if (rt.type.equals(reportType)) { + return rt; + } + } + throw new IllegalArgumentException("Unknown report type: " + reportType); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java new file mode 100644 index 0000000..82ef757 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java @@ -0,0 +1,1208 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.asset.Email; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailCCField; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailField; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailTemplateResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.File; +import io.cdap.plugin.marketo.common.api.entities.asset.FileFolder; +import io.cdap.plugin.marketo.common.api.entities.asset.FileResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Folder; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderDescriptor; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Form; +import io.cdap.plugin.marketo.common.api.entities.asset.FormField; +import io.cdap.plugin.marketo.common.api.entities.asset.FormFieldsResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.FormKnownVisitorDTO; +import io.cdap.plugin.marketo.common.api.entities.asset.FormResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.FormThankYouPageDTO; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPage; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageTemplateResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Program; +import io.cdap.plugin.marketo.common.api.entities.asset.ProgramResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Recurrence; +import io.cdap.plugin.marketo.common.api.entities.asset.Segmentation; +import io.cdap.plugin.marketo.common.api.entities.asset.SegmentationResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartCampaign; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartCampaignsResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartList; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartListsResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Snippet; +import io.cdap.plugin.marketo.common.api.entities.asset.SnippetsResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.StaticList; +import io.cdap.plugin.marketo.common.api.entities.asset.StaticListsResponse; +import io.cdap.plugin.marketo.common.api.entities.asset.Tag; +import io.cdap.plugin.marketo.common.api.entities.asset.TagsResponse; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * Schema helper for entity plugin. + */ +@SuppressWarnings("DuplicatedCode") +public final class EntityHelper { + private static final List ENTITIES_WITHOUT_PAGING = ImmutableList.of( + EntityType.SEGMENTATION + ); + + public static boolean supportPaging(EntityType entityType) { + return !ENTITIES_WITHOUT_PAGING.contains(entityType); + } + + public static Iterator iteratorForEntityType(Marketo marketo, EntityType entityType, int offset, + int maxResults) { + switch (entityType) { + case EMAIL: { + return marketo.iteratePage( + "/rest/asset/v1/emails.json", + EmailResponse.class, + EmailResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case EMAILTEMPLATE: { + return marketo.iteratePage( + "/rest/asset/v1/emailTemplates.json", + EmailTemplateResponse.class, + EmailTemplateResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FILE: { + return marketo.iteratePage( + "/rest/asset/v1/files.json", + FileResponse.class, + FileResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FOLDER: { + return marketo.iteratePage( + "/rest/asset/v1/folders.json", + FolderResponse.class, + FolderResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FORMFIELD: { + return marketo.iteratePage( + "/rest/asset/v1/form/fields.json", + FormFieldsResponse.class, + FormFieldsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FORM: { + return marketo.iteratePage( + "/rest/asset/v1/forms.json", + FormResponse.class, + FormResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case LANDINGPAGE: { + return marketo.iteratePage( + "/rest/asset/v1/landingPages.json", + LandingPageResponse.class, + LandingPageResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case LANDINGPAGETEMPLATE: { + return marketo.iteratePage( + "/rest/asset/v1/landingPageTemplates.json", + LandingPageTemplateResponse.class, + LandingPageTemplateResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case PROGRAM: { + return marketo.iteratePage( + "/rest/asset/v1/programs.json", + ProgramResponse.class, + ProgramResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SEGMENTATION: { + return marketo.iteratePage( + "/rest/asset/v1/segmentation.json", + SegmentationResponse.class, + SegmentationResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SMARTCAMPAIGN: { + return marketo.iteratePage( + "/rest/asset/v1/smartCampaigns.json", + SmartCampaignsResponse.class, + SmartCampaignsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SMARTLIST: { + return marketo.iteratePage( + "/rest/asset/v1/smartLists.json", + SmartListsResponse.class, + SmartListsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SNIPPET: { + return marketo.iteratePage( + "/rest/asset/v1/snippets.json", + SnippetsResponse.class, + SnippetsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case STATICLIST: { + return marketo.iteratePage( + "/rest/asset/v1/staticLists.json", + StaticListsResponse.class, + StaticListsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case TAG: { + return marketo.iteratePage( + "/rest/asset/v1/tagTypes.json", + TagsResponse.class, + TagsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + default: { + throw new IllegalArgumentException("Unknown entity name:" + entityType.getValue()); + } + } + } + + public static Schema schemaForEntityName(String entityName) { + switch (entityName) { + case "Email": { + return getEmailSchema(); + } + case "EmailCCField": { + return getEmailCCFieldSchema(); + } + case "EmailField": { + return getEmailFieldSchema(); + } + case "EmailTemplate": { + return getEmailTemplateSchema(); + } + case "File": { + return getFileSchema(); + } + case "FileFolder": { + return getFileFolderSchema(); + } + case "Folder": { + return getFolderSchema(); + } + case "FolderDescriptor": { + return getFolderDescriptorSchema(); + } + case "Form": { + return getFormSchema(); + } + case "FormField": { + return getFormFieldSchema(); + } + case "FormKnownVisitorDTO": { + return getFormKnownVisitorDTOSchema(); + } + case "FormThankYouPageDTO": { + return getFormThankYouPageDTOSchema(); + } + case "LandingPage": { + return getLandingPageSchema(); + } + case "LandingPageTemplate": { + return getLandingPageTemplateSchema(); + } + case "Program": { + return getProgramSchema(); + } + case "Recurrence": { + return getRecurrenceSchema(); + } + case "Segmentation": { + return getSegmentationSchema(); + } + case "SmartCampaign": { + return getSmartCampaignSchema(); + } + case "SmartList": { + return getSmartListSchema(); + } + case "Snippet": { + return getSnippetSchema(); + } + case "StaticList": { + return getStaticListSchema(); + } + case "Tag": { + return getTagSchema(); + } + default: { + throw new IllegalArgumentException("Unknown entity name:" + entityName); + } + } + } + + public static Schema getEmailSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("operational", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("publishToMSI", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("subject", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("textOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("webView", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("autoCopyToText", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("ccFields", + Schema.nullableOf(Schema.arrayOf(EntityHelper.getEmailCCFieldSchema())))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailCCFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("attributeId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("objectName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("displayName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("apiName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailTemplateSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFileSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFileFolderSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("mimeType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("size", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFileFolderSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFolderSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("accessZoneId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folderType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("isArchive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("parent", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("path", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFolderDescriptorSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folderName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("buttonLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("buttonLocation", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fontFamily", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fontSize", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("knownVisitor", Schema.nullableOf(EntityHelper.getFormKnownVisitorDTOSchema()))); + fields.add(Schema.Field.of("labelPosition", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("language", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("locale", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("progressiveProfiling", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("thankYouList", + Schema.nullableOf(Schema.arrayOf(EntityHelper.getFormThankYouPageDTOSchema())))); + fields.add(Schema.Field.of("theme", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("waitingLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("dataType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("defaultValue", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fieldMaskValues", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fieldWidth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("initiallyChecked", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isLabelToRight", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isMultiselect", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isRequired", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("labelWidth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("maxLength", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("maximumNumber", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("minimumNumber", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("picklistValues", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("placeholderText", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("validationMessage", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("visibleRows", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormKnownVisitorDTOSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormThankYouPageDTOSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("isDefault", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("followupType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("followupValue", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("operator", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("subjectField", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("values", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getLandingPageSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("customHeadHTML", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("facebookOgTags", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("formPrefill", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("keywords", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("mobileEnabled", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("robots", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getLandingPageTemplateSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("enableMunchkin", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("templateType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getProgramSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("channel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getRecurrenceSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("startAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("endAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("intervalType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("interval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekdayOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("weekdayMask", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + fields.add(Schema.Field.of("dayOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("dayOfWeek", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSegmentationSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSmartCampaignSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isActive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isRequestable", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("recurrence", Schema.nullableOf(EntityHelper.getRecurrenceSchema()))); + fields.add(Schema.Field.of("qualificationRuleType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("qualificationRuleInterval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("qualificationRuleUnit", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("maxMembers", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("isCommunicationLimitEnabled", + Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("smartListId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("flowId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSmartListSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSnippetSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getStaticListSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getTagSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("applicableProgramTypes", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("required", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("tagType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static StructuredRecord structuredRecordFromEntity(String entityName, Object entity, + Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + switch (entityName) { + case "Email": { + Email email = (Email) entity; + if (entity == null) { + break; + } + builder.set("createdAt", email.getCreatedAt()); + builder.set("description", email.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + email.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("fromEmail", EntityHelper.structuredRecordFromEntity( + "EmailField", + email.getFromEmail(), + schema.getField("fromEmail").getSchema().getNonNullable())); + builder.set("fromName", EntityHelper.structuredRecordFromEntity( + "EmailField", + email.getFromName(), + schema.getField("fromName").getSchema().getNonNullable())); + builder.set("id", email.getId()); + builder.set("name", email.getName()); + builder.set("operational", email.getOperational()); + builder.set("publishToMSI", email.getPublishToMSI()); + builder.set("replyEmail", EntityHelper.structuredRecordFromEntity( + "EmailField", + email.getReplyEmail(), + schema.getField("replyEmail").getSchema().getNonNullable())); + builder.set("status", email.getStatus()); + builder.set("subject", EntityHelper.structuredRecordFromEntity( + "EmailField", + email.getSubject(), + schema.getField("subject").getSchema().getNonNullable())); + builder.set("template", email.getTemplate()); + builder.set("textOnly", email.getTextOnly()); + builder.set("updatedAt", email.getUpdatedAt()); + builder.set("url", email.getUrl()); + builder.set("version", email.getVersion()); + builder.set("webView", email.getWebView()); + builder.set("workspace", email.getWorkspace()); + builder.set("autoCopyToText", email.getAutoCopyToText()); + builder.set( + "ccFields", + email.getCcFields().stream() + .map(ent -> EntityHelper.structuredRecordFromEntity( + "EmailCCField", + ent, + schema.getField("ccFields").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + break; + } + case "EmailCCField": { + EmailCCField emailccfield = (EmailCCField) entity; + if (entity == null) { + break; + } + builder.set("attributeId", emailccfield.getAttributeId()); + builder.set("objectName", emailccfield.getObjectName()); + builder.set("displayName", emailccfield.getDisplayName()); + builder.set("apiName", emailccfield.getApiName()); + break; + } + case "EmailField": { + EmailField emailfield = (EmailField) entity; + if (entity == null) { + break; + } + builder.set("type", emailfield.getType()); + builder.set("value", emailfield.getValue()); + break; + } + case "EmailTemplate": { + EmailTemplate emailtemplate = (EmailTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", emailtemplate.getCreatedAt()); + builder.set("description", emailtemplate.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + emailtemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", emailtemplate.getId()); + builder.set("name", emailtemplate.getName()); + builder.set("status", emailtemplate.getStatus()); + builder.set("updatedAt", emailtemplate.getUpdatedAt()); + builder.set("url", emailtemplate.getUrl()); + builder.set("version", emailtemplate.getVersion()); + builder.set("workspace", emailtemplate.getWorkspace()); + break; + } + case "File": { + File file = (File) entity; + if (entity == null) { + break; + } + builder.set("createdAt", file.getCreatedAt()); + builder.set("description", file.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FileFolder", + file.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", file.getId()); + builder.set("mimeType", file.getMimeType()); + builder.set("name", file.getName()); + builder.set("size", file.getSize()); + builder.set("updatedAt", file.getUpdatedAt()); + builder.set("url", file.getUrl()); + break; + } + case "FileFolder": { + FileFolder filefolder = (FileFolder) entity; + if (entity == null) { + break; + } + builder.set("id", filefolder.getId()); + builder.set("name", filefolder.getName()); + builder.set("type", filefolder.getType()); + break; + } + case "Folder": { + Folder folder = (Folder) entity; + if (entity == null) { + break; + } + builder.set("accessZoneId", folder.getAccessZoneId()); + builder.set("createdAt", folder.getCreatedAt()); + builder.set("description", folder.getDescription()); + builder.set("folderId", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getFolderId(), + schema.getField("folderId").getSchema().getNonNullable())); + builder.set("folderType", folder.getFolderType()); + builder.set("id", folder.getId()); + builder.set("isArchive", folder.getArchive()); + builder.set("isSystem", folder.getSystem()); + builder.set("name", folder.getName()); + builder.set("parent", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getParent(), + schema.getField("parent").getSchema().getNonNullable())); + builder.set("path", folder.getPath()); + builder.set("updatedAt", folder.getUpdatedAt()); + builder.set("url", folder.getUrl()); + builder.set("workspace", folder.getWorkspace()); + break; + } + case "FolderDescriptor": { + FolderDescriptor folderdescriptor = (FolderDescriptor) entity; + if (entity == null) { + break; + } + builder.set("id", folderdescriptor.getId()); + builder.set("type", folderdescriptor.getType()); + builder.set("folderName", folderdescriptor.getFolderName()); + break; + } + case "Form": { + Form form = (Form) entity; + if (entity == null) { + break; + } + builder.set("buttonLabel", form.getButtonLabel()); + builder.set("buttonLocation", form.getButtonLocation()); + builder.set("createdAt", form.getCreatedAt()); + builder.set("description", form.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + form.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("fontFamily", form.getFontFamily()); + builder.set("fontSize", form.getFontSize()); + builder.set("id", form.getId()); + builder.set("knownVisitor", EntityHelper.structuredRecordFromEntity( + "FormKnownVisitorDTO", + form.getKnownVisitor(), + schema.getField("knownVisitor").getSchema().getNonNullable())); + builder.set("labelPosition", form.getLabelPosition()); + builder.set("language", form.getLanguage()); + builder.set("locale", form.getLocale()); + builder.set("name", form.getName()); + builder.set("progressiveProfiling", form.getProgressiveProfiling()); + builder.set("status", form.getStatus()); + builder.set( + "thankYouList", + form.getThankYouList().stream() + .map(ent -> EntityHelper.structuredRecordFromEntity( + "FormThankYouPageDTO", + ent, + schema.getField("thankYouList").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + builder.set("theme", form.getTheme()); + builder.set("updatedAt", form.getUpdatedAt()); + builder.set("url", form.getUrl()); + builder.set("waitingLabel", form.getWaitingLabel()); + break; + } + case "FormField": { + FormField formfield = (FormField) entity; + if (entity == null) { + break; + } + builder.set("dataType", formfield.getDataType()); + builder.set("defaultValue", formfield.getDefaultValue()); + builder.set("description", formfield.getDescription()); + builder.set("fieldMaskValues", formfield.getFieldMaskValues()); + builder.set("fieldWidth", formfield.getFieldWidth()); + builder.set("id", formfield.getId()); + builder.set("initiallyChecked", formfield.getInitiallyChecked()); + builder.set("isLabelToRight", formfield.getLabelToRight()); + builder.set("isMultiselect", formfield.getMultiselect()); + builder.set("isRequired", formfield.getRequired()); + builder.set("labelWidth", formfield.getLabelWidth()); + builder.set("maxLength", formfield.getMaxLength()); + builder.set("maximumNumber", formfield.getMaximumNumber()); + builder.set("minimumNumber", formfield.getMinimumNumber()); + builder.set("picklistValues", formfield.getPicklistValues()); + builder.set("placeholderText", formfield.getPlaceholderText()); + builder.set("validationMessage", formfield.getValidationMessage()); + builder.set("visibleRows", formfield.getVisibleRows()); + break; + } + case "FormKnownVisitorDTO": { + FormKnownVisitorDTO formknownvisitordto = (FormKnownVisitorDTO) entity; + if (entity == null) { + break; + } + builder.set("template", formknownvisitordto.getTemplate()); + builder.set("type", formknownvisitordto.getType()); + break; + } + case "FormThankYouPageDTO": { + FormThankYouPageDTO formthankyoupagedto = (FormThankYouPageDTO) entity; + if (entity == null) { + break; + } + builder.set("isDefault", formthankyoupagedto.getDefault()); + builder.set("followupType", formthankyoupagedto.getFollowupType()); + builder.set("followupValue", formthankyoupagedto.getFollowupValue()); + builder.set("operator", formthankyoupagedto.getOperator()); + builder.set("subjectField", formthankyoupagedto.getSubjectField()); + builder.set("values", formthankyoupagedto.getValues()); + break; + } + case "LandingPage": { + LandingPage landingpage = (LandingPage) entity; + if (entity == null) { + break; + } + builder.set("url", landingpage.getUrl()); + builder.set("computedUrl", landingpage.getComputedUrl()); + builder.set("createdAt", landingpage.getCreatedAt()); + builder.set("customHeadHTML", landingpage.getCustomHeadHTML()); + builder.set("description", landingpage.getDescription()); + builder.set("facebookOgTags", landingpage.getFacebookOgTags()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpage.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("formPrefill", landingpage.getFormPrefill()); + builder.set("id", landingpage.getId()); + builder.set("keywords", landingpage.getKeywords()); + builder.set("mobileEnabled", landingpage.getMobileEnabled()); + builder.set("name", landingpage.getName()); + builder.set("robots", landingpage.getRobots()); + builder.set("status", landingpage.getStatus()); + builder.set("template", landingpage.getTemplate()); + builder.set("title", landingpage.getTitle()); + builder.set("updatedAt", landingpage.getUpdatedAt()); + builder.set("workspace", landingpage.getWorkspace()); + break; + } + case "LandingPageTemplate": { + LandingPageTemplate landingpagetemplate = (LandingPageTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", landingpagetemplate.getCreatedAt()); + builder.set("description", landingpagetemplate.getDescription()); + builder.set("enableMunchkin", landingpagetemplate.getEnableMunchkin()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpagetemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", landingpagetemplate.getId()); + builder.set("name", landingpagetemplate.getName()); + builder.set("status", landingpagetemplate.getStatus()); + builder.set("templateType", landingpagetemplate.getTemplateType()); + builder.set("updatedAt", landingpagetemplate.getUpdatedAt()); + builder.set("url", landingpagetemplate.getUrl()); + builder.set("workspace", landingpagetemplate.getWorkspace()); + break; + } + case "Program": { + Program program = (Program) entity; + if (entity == null) { + break; + } + builder.set("channel", program.getChannel()); + builder.set("createdAt", program.getCreatedAt()); + builder.set("description", program.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + program.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", program.getId()); + builder.set("name", program.getName()); + builder.set("sfdcId", program.getSfdcId()); + builder.set("sfdcName", program.getSfdcName()); + builder.set("status", program.getStatus()); + builder.set("type", program.getType()); + builder.set("updatedAt", program.getUpdatedAt()); + builder.set("url", program.getUrl()); + builder.set("workspace", program.getWorkspace()); + break; + } + case "Recurrence": { + Recurrence recurrence = (Recurrence) entity; + if (entity == null) { + break; + } + builder.set("startAt", recurrence.getStartAt()); + builder.set("endAt", recurrence.getEndAt()); + builder.set("intervalType", recurrence.getIntervalType()); + builder.set("interval", recurrence.getInterval()); + builder.set("weekdayOnly", recurrence.getWeekdayOnly()); + builder.set("weekdayMask", recurrence.getWeekdayMask()); + builder.set("dayOfMonth", recurrence.getDayOfMonth()); + builder.set("dayOfWeek", recurrence.getDayOfWeek()); + builder.set("weekOfMonth", recurrence.getWeekOfMonth()); + break; + } + case "Segmentation": { + Segmentation segmentation = (Segmentation) entity; + if (entity == null) { + break; + } + builder.set("createdAt", segmentation.getCreatedAt()); + builder.set("description", segmentation.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + segmentation.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", segmentation.getId()); + builder.set("name", segmentation.getName()); + builder.set("status", segmentation.getStatus()); + builder.set("updatedAt", segmentation.getUpdatedAt()); + builder.set("url", segmentation.getUrl()); + builder.set("workspace", segmentation.getWorkspace()); + break; + } + case "SmartCampaign": { + SmartCampaign smartcampaign = (SmartCampaign) entity; + if (entity == null) { + break; + } + builder.set("id", smartcampaign.getId()); + builder.set("name", smartcampaign.getName()); + builder.set("description", smartcampaign.getDescription()); + builder.set("type", smartcampaign.getType()); + builder.set("isSystem", smartcampaign.getSystem()); + builder.set("isActive", smartcampaign.getActive()); + builder.set("isRequestable", smartcampaign.getRequestable()); + builder.set("recurrence", EntityHelper.structuredRecordFromEntity( + "Recurrence", + smartcampaign.getRecurrence(), + schema.getField("recurrence").getSchema().getNonNullable())); + builder.set("qualificationRuleType", smartcampaign.getQualificationRuleType()); + builder.set("qualificationRuleInterval", smartcampaign.getQualificationRuleInterval()); + builder.set("qualificationRuleUnit", smartcampaign.getQualificationRuleUnit()); + builder.set("maxMembers", smartcampaign.getMaxMembers()); + builder.set("isCommunicationLimitEnabled", smartcampaign.getCommunicationLimitEnabled()); + builder.set("smartListId", smartcampaign.getSmartListId()); + builder.set("flowId", smartcampaign.getFlowId()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartcampaign.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("createdAt", smartcampaign.getCreatedAt()); + builder.set("updatedAt", smartcampaign.getUpdatedAt()); + builder.set("workspace", smartcampaign.getWorkspace()); + builder.set("status", smartcampaign.getStatus()); + break; + } + case "SmartList": { + SmartList smartlist = (SmartList) entity; + if (entity == null) { + break; + } + builder.set("id", smartlist.getId()); + builder.set("name", smartlist.getName()); + builder.set("description", smartlist.getDescription()); + builder.set("createdAt", smartlist.getCreatedAt()); + builder.set("updatedAt", smartlist.getUpdatedAt()); + builder.set("url", smartlist.getUrl()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("workspace", smartlist.getWorkspace()); + break; + } + case "Snippet": { + Snippet snippet = (Snippet) entity; + if (entity == null) { + break; + } + builder.set("createdAt", snippet.getCreatedAt()); + builder.set("description", snippet.getDescription()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + snippet.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", snippet.getId()); + builder.set("name", snippet.getName()); + builder.set("status", snippet.getStatus()); + builder.set("updatedAt", snippet.getUpdatedAt()); + builder.set("url", snippet.getUrl()); + builder.set("workspace", snippet.getWorkspace()); + break; + } + case "StaticList": { + StaticList staticlist = (StaticList) entity; + if (entity == null) { + break; + } + builder.set("id", staticlist.getId()); + builder.set("name", staticlist.getName()); + builder.set("description", staticlist.getDescription()); + builder.set("createdAt", staticlist.getCreatedAt()); + builder.set("updatedAt", staticlist.getUpdatedAt()); + builder.set("url", staticlist.getUrl()); + builder.set("folder", EntityHelper.structuredRecordFromEntity( + "FolderDescriptor", + staticlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("workspace", staticlist.getWorkspace()); + builder.set("computedUrl", staticlist.getComputedUrl()); + break; + } + case "Tag": { + Tag tag = (Tag) entity; + if (entity == null) { + break; + } + builder.set("applicableProgramTypes", tag.getApplicableProgramTypes()); + builder.set("required", tag.getRequired()); + builder.set("tagType", tag.getTagType()); + break; + } + default: { + throw new IllegalArgumentException("Unknown entity name:" + entityName); + } + } + return builder.build(); + } + + /** + * Entity type. + */ + public enum EntityType { + EMAIL("Email"), + + EMAILTEMPLATE("EmailTemplate"), + + FILE("File"), + + FOLDER("Folder"), + + FORM("Form"), + + FORMFIELD("FormField"), + + LANDINGPAGE("LandingPage"), + + LANDINGPAGETEMPLATE("LandingPageTemplate"), + + PROGRAM("Program"), + + SEGMENTATION("Segmentation"), + + SMARTCAMPAIGN("SmartCampaign"), + + SMARTLIST("SmartList"), + + SNIPPET("Snippet"), + + STATICLIST("StaticList"), + + TAG("Tag"); + + private final String value; + + EntityType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static EntityType fromString(String value) { + for (EntityType entityType : EntityType.values()) { + if (entityType.value.equals(value)) { + return entityType; + } + } + throw new IllegalArgumentException("Unknown entity type type: " + value); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java new file mode 100644 index 0000000..0b1ba00 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class MarketoEntityFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + + MarketoEntityFormatProvider(MarketoEntitySourceConfig config) { + this.conf = Collections.unmodifiableMap(new HashMap() {{ + put(PROPERTY_CONFIG_JSON, gson.toJson(config)); + }}); + } + + @Override + public String getInputFormatClassName() { + return MarketoEntityInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java new file mode 100644 index 0000000..6f67747 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * InputFormat for mapreduce job. + */ +public class MarketoEntityInputFormat extends InputFormat { + private static final Gson GSON = new Gson(); + private static final Logger LOG = LoggerFactory.getLogger(MarketoEntityInputFormat.class); + + @Override + public List getSplits(JobContext jobContext) { + Configuration conf = jobContext.getConfiguration(); + MarketoEntitySourceConfig config = GSON.fromJson( + conf.get(MarketoEntityFormatProvider.PROPERTY_CONFIG_JSON), MarketoEntitySourceConfig.class); + + if (EntityHelper.supportPaging(config.getEntityType())) { + return IntStream.range(0, config.getSplitsCount()) + .mapToObj(splitId -> new MarketoEntitySplit(splitId, config.getResultsPerPage(), config.getSplitsCount())) + .collect(Collectors.toList()); + } else { + LOG.info("Entity '{}' does not support paging, single split used", config.getEntityType().getValue()); + return ImmutableList.of(new MarketoEntitySplit(0, 200, 1)); + } + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + MarketoEntitySplit split = (MarketoEntitySplit) inputSplit; + return new MarketoEntityRecordReader(split.getSplitId(), split.getResultsPerPage(), split.getTotalSplits()); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java new file mode 100644 index 0000000..6cb2e10 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.hadoop.io.NullWritable; + +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Plugin that reads entities from Marketo api. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(MarketoEntityPlugin.NAME) +@Description("Reads Entities from Marketo.") +public class MarketoEntityPlugin extends BatchSource { + public static final String NAME = "MarketoEntityPlugin"; + + private final MarketoEntitySourceConfig config; + + public MarketoEntityPlugin(MarketoEntitySourceConfig config) { + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + validateConfiguration(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Marketo entities", + Objects.requireNonNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new MarketoEntityFormatProvider(config))); + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + emitter.emit(input.getValue()); + } + + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java new file mode 100644 index 0000000..5bd98bb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java @@ -0,0 +1,119 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.format.StructuredRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Iterator; + +/** + * RecordReader implementation, which reads events from Marketo api. + */ +public class MarketoEntityRecordReader extends RecordReader { + private static final Logger LOG = LoggerFactory.getLogger(MarketoEntityRecordReader.class); + private static final Gson GSON = new GsonBuilder().create(); + private Iterator iterator = null; + private Object current = null; + private MarketoEntitySourceConfig config; + private int splitId; + private int resultsPerPage; + private int totalSplits; + private int currentOffset = -1; + private boolean needSwapPage = false; + private boolean supportPaging = true; + + public MarketoEntityRecordReader(int splitId, int resultsPerPage, int totalSplits) { + this.splitId = splitId; + this.resultsPerPage = resultsPerPage; + this.totalSplits = totalSplits; + this.currentOffset = splitId * resultsPerPage; + } + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + deserializeConfiguration(taskAttemptContext.getConfiguration()); + iterator = EntityHelper.iteratorForEntityType(config.getMarketo(), config.getEntityType(), + currentOffset, resultsPerPage); + LOG.info("Split '{}' fetched data by offset '{}'", splitId, currentOffset); + } + + @Override + public boolean nextKeyValue() { + if (iterator.hasNext()) { + current = iterator.next(); + // this page is not empty, try next page as well + needSwapPage = true; + return true; + } else if (swapPage()) { + // page swapped, try to get new item + return nextKeyValue(); + } else { + return false; + } + } + + private boolean swapPage() { + if (needSwapPage && supportPaging) { + needSwapPage = false; + currentOffset += totalSplits * resultsPerPage; + iterator = EntityHelper.iteratorForEntityType(config.getMarketo(), config.getEntityType(), currentOffset, + resultsPerPage); + LOG.info("Split '{}' fetched data by offset '{}'", splitId, currentOffset); + return true; + } + return false; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public StructuredRecord getCurrentValue() { + return EntityHelper.structuredRecordFromEntity( + config.getEntityType().getValue(), + current, + config.getSchema() + ); + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void close() { + } + + private void deserializeConfiguration(Configuration conf) { + String configJson = conf.get(MarketoEntityFormatProvider.PROPERTY_CONFIG_JSON); + config = GSON.fromJson(configJson, MarketoEntitySourceConfig.class); + supportPaging = EntityHelper.supportPaging(config.getEntityType()); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java new file mode 100644 index 0000000..5e7a353 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java @@ -0,0 +1,160 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Provides all required configuration for reading Marketo entities. + */ +public class MarketoEntitySourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_CLIENT_ID = "clientId"; + public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; + public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; + public static final String PROPERTY_ENTITY_TYPE = "entityType"; + public static final String PROPERTY_SPLITS_COUNT = "splitsCount"; + public static final String PROPERTY_RESULTS_PER_PAGE = "resultsPerPage"; + + @Name(PROPERTY_CLIENT_ID) + @Description("Marketo Client ID.") + @Macro + protected String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Description("Marketo Client secret.") + @Macro + protected String clientSecret; + + @Name(PROPERTY_REST_API_ENDPOINT) + @Description("REST API endpoint URL.") + @Macro + protected String restApiEndpoint; + + @Name(PROPERTY_ENTITY_TYPE) + @Description("Type of entity.") + @Macro + protected String entityType; + + @Name(PROPERTY_SPLITS_COUNT) + @Description("Number of splits to fetch data.") + @Macro + protected int splitsCount; + + @Name(PROPERTY_RESULTS_PER_PAGE) + @Description("Max results per page.") + @Macro + protected int resultsPerPage; + + private transient Schema schema = null; + private transient Marketo marketo = null; + + public MarketoEntitySourceConfig(String referenceName) { + super(referenceName); + } + + public EntityHelper.EntityType getEntityType() { + return EntityHelper.EntityType.fromString(entityType); + } + + public Schema getSchema() { + if (schema == null) { + schema = EntityHelper.schemaForEntityName(getEntityType().getValue()); + } + return schema; + } + + public Marketo getMarketo() { + if (marketo == null) { + marketo = new Marketo(getRestApiEndpoint(), getClientId(), getClientSecret()); + } + return marketo; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestApiEndpoint() { + return restApiEndpoint; + } + + public int getSplitsCount() { + return splitsCount; + } + + public int getResultsPerPage() { + return resultsPerPage; + } + + void validate(FailureCollector failureCollector) { + IdUtils.validateReferenceName(referenceName, failureCollector); + validateEntityType(failureCollector); + validateMarketoEndpoint(failureCollector); + validateSecrets(failureCollector); + } + + void validateEntityType(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_ENTITY_TYPE)) { + try { + getEntityType(); + } catch (IllegalArgumentException ex) { + failureCollector.addFailure(String.format("Incorrect entity type '%s'.", entityType), + "Set entity type to valid.") + .withConfigProperty(PROPERTY_ENTITY_TYPE); + } + } + } + + void validateSecrets(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { + failureCollector.addFailure("Client ID is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { + failureCollector.addFailure("Client Secret is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + } + + void validateMarketoEndpoint(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REST_API_ENDPOINT)) { + try { + new URL(getRestApiEndpoint()); + } catch (MalformedURLException e) { + failureCollector + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), null) + .withConfigProperty(PROPERTY_REST_API_ENDPOINT); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java new file mode 100644 index 0000000..044465d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A no-op split. + */ +public class MarketoEntitySplit extends InputSplit implements Writable { + private int splitId; + private int resultsPerPage; + private int totalSplits; + + public MarketoEntitySplit() { + } + + public MarketoEntitySplit(int splitId, int resultsPerPage, int totalSplits) { + this.splitId = splitId; + this.resultsPerPage = resultsPerPage; + this.totalSplits = totalSplits; + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + splitId = dataInput.readInt(); + resultsPerPage = dataInput.readInt(); + totalSplits = dataInput.readInt(); + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeInt(splitId); + dataOutput.writeInt(resultsPerPage); + dataOutput.writeInt(totalSplits); + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } + + public int getSplitId() { + return splitId; + } + + public int getResultsPerPage() { + return resultsPerPage; + } + + public int getTotalSplits() { + return totalSplits; + } +} diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java new file mode 100644 index 0000000..80c7071 --- /dev/null +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java @@ -0,0 +1,291 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import io.cdap.plugin.marketo.common.api.entities.Warning; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class MarketoHttpTest { + private static final Gson GSON = new Gson(); + + @Rule + public WireMockRule wireMockRule = new WireMockRule( + WireMockConfiguration.wireMockConfig().dynamicPort() + ); + + public static class StubResponse extends BaseResponse { + StubResponse(boolean success, List errors, List warnings) { + setSuccess(success); + setErrors(errors); + setWarnings(warnings); + } + } + + public static class PageResponse extends BaseResponse { + private List results; + + PageResponse(boolean moreResults, String nextPageToken, String... items) { + setSuccess(true); + setMoreResult(moreResults); + results = Arrays.asList(items); + setNextPageToken(nextPageToken); + } + + public List getResults() { + return results; + } + } + + @Test + public void testPaging() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page1", "1", "2", "3")) + ) + ) + .willSetStateTo("page1") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page1") + .withQueryParam("nextPageToken", WireMock.equalTo("page1")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page2", "4", "5", "6")) + ) + ) + .willSetStateTo("page2") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page2") + .withQueryParam("nextPageToken", WireMock.equalTo("page2")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(false, null, "7", "8", "9")) + ) + ) + .willSetStateTo("page3") + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + + List results = StreamSupport.stream(Spliterators.spliteratorUnknownSize( + m.iteratePage("/rest/v1/paged.json", PageResponse.class, PageResponse::getResults), + Spliterator.ORDERED), false).sorted().collect(Collectors.toList()); + + Assert.assertArrayEquals(new String[]{"1", "2", "3", "4", "5", "6", "7", "8", "9"}, results.toArray()); + } + + @Test + public void testToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .withQueryParam("grant_type", WireMock.equalTo("client_credentials")) + .withQueryParam("client_id", WireMock.equalTo("clientNiceId")) + .withQueryParam("client_secret", WireMock.equalTo("clientNiceSecret")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + Assert.assertEquals("niceToken", m.getCurrentToken().getAccessToken()); + } + + @Test + public void testTokenRefresh() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList( + new Error(602, "Access token expired")), + Collections.emptyList())) + ) + ) + .willSetStateTo("refreshed") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs("refreshed") + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/stub.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/identity/oauth/token"))); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/rest/v1/stub.json"))); + } + + @Test + public void testPost() { + setupToken(); + + WireMock.stubFor( + WireMock.post(WireMock.urlPathMatching("/rest/v1/post.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedPost("/rest/v1/post.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class), "body", String::toString); + + WireMock.verify( + WireMock.postRequestedFor(WireMock.urlPathEqualTo("/rest/v1/post.json")) + .withRequestBody(WireMock.equalTo("body")) + ); + } + + @Test + public void testMessages() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/justWarnings.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Arrays.asList( + new Warning(700, "Reversed agent 007"), + new Warning(777, "Result of 1000 - 333") + )))) + ) + ); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/errors.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList(new Error(123, "No way")), + Collections.emptyList()))) + ) + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/justWarnings.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + + try { + m.validatedGet("/rest/v1/errors.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("123")); + Assert.assertTrue(ex.getMessage().contains("No way")); + } + } + + @Test + public void testBuildUri() { + setupToken(); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + String uriWithToken = m.buildUri("/hello", Collections.emptyMap()).toString(); + Assert.assertTrue(uriWithToken.contains("access_token")); + String uriWithoutToken = m.buildUri("/hello", ImmutableMap.of("param", "value"), false).toString(); + Assert.assertFalse(uriWithoutToken.contains("access_token")); + Assert.assertTrue(uriWithoutToken.contains("param=value")); + } + + @Test + public void testHttpError() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/fail.json")) + .willReturn( + WireMock.aResponse().withStatus(500).withBody("GJ server.") + ) + ); + + try { + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/fail.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("GJ server")); + } + } + + @Test + public void invalidEndpoint() { + try { + new MarketoHttp("%^%^&%^", "clientNiceId", "clientNiceSecret"); + Assert.fail("This call expected to fail."); + } catch (IllegalArgumentException ex) { + Assert.assertEquals("'%^%^&%^/identity/oauth/token' is invalid URI", ex.getMessage()); + } + } + + void setupToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + } + + String getApiUrl() { + return String.format("http://localhost:%d", wireMockRule.port()); + } +} diff --git a/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java new file mode 100644 index 0000000..16205fb --- /dev/null +++ b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.collect.ImmutableList; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.entities.asset.Email; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailCCField; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderDescriptor; +import io.cdap.plugin.marketo.common.api.entities.asset.Program; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class EntityHelperTest { + @Test + public void testNestedRecords() { + Schema programSchema = EntityHelper.getProgramSchema(); + Program program = Program.builder().folder(new FolderDescriptor("1234", "Hello", null)) + .build(); + StructuredRecord programRecord = EntityHelper.structuredRecordFromEntity("Program", program, + programSchema); + + Assert.assertEquals(programRecord.get("folder").get("id"), "1234"); + + Schema emailSchema = EntityHelper.getEmailSchema(); + Email email = Email.builder().ccFields(ImmutableList.of( + new EmailCCField("attr1", "cc1", "cc 1", "cc1"), + new EmailCCField("attr2", "cc2", "cc 2", "cc2") + )).build(); + StructuredRecord emailRecord = EntityHelper.structuredRecordFromEntity("Email", email, + emailSchema); + + List ccRecords = emailRecord.>get("ccFields"); + Assert.assertEquals(2, ccRecords.size()); + Assert.assertEquals(ccRecords.get(0).get("attributeId"), "attr1"); + } +} diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..600350b --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + diff --git a/widgets/MarketoEntityPlugin-batchsource.json b/widgets/MarketoEntityPlugin-batchsource.json new file mode 100644 index 0000000..b7e4940 --- /dev/null +++ b/widgets/MarketoEntityPlugin-batchsource.json @@ -0,0 +1,94 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Marketo Entity", + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + }, + { + "widget-type": "textbox", + "label": "Rest API endpoint", + "name": "restApiEndpoint" + }, + { + "widget-type": "select", + "label": "Entity Type", + "name": "entityType", + "widget-attributes": { + "default": "File", + "values": [ + "Email", + "EmailTemplate", + "File", + "Folder", + "Form", + "FormField", + "LandingPage", + "LandingPageTemplate", + "Program", + "Segmentation", + "SmartCampaign", + "SmartList", + "Snippet", + "StaticList", + "Tag" + ] + } + } + ] + }, + { + "label": "Authentication", + "properties": [ + { + "widget-type": "textbox", + "label": "Client ID", + "name": "clientId" + }, + { + "widget-type": "password", + "label": "Client Secret", + "name": "clientSecret" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "number", + "label": "Splits Count", + "name": "splitsCount", + "widget-attributes": { + "default": 2, + "min": 1, + "max": 5 + } + }, + { + "widget-type": "number", + "label": "Max Results Per Page", + "name": "resultsPerPage", + "widget-attributes": { + "default": 200, + "min": 20, + "max": 200 + } + } + ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": {} + } + ] +} \ No newline at end of file diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json new file mode 100644 index 0000000..e5390ba --- /dev/null +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -0,0 +1,77 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Marketo Reporting", + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + }, + { + "widget-type": "textbox", + "label": "Rest API endpoint", + "name": "restApiEndpoint" + } + ] + }, + { + "label": "Authentication", + "properties": [ + { + "widget-type": "textbox", + "label": "Client ID", + "name": "clientId" + }, + { + "widget-type": "password", + "label": "Client Secret", + "name": "clientSecret" + } + ] + }, + { + "label": "Report", + "properties": [ + { + "widget-type": "select", + "label": "Report Type", + "name": "reportType", + "widget-attributes": { + "default": "leads", + "values": [ + "leads", + "activities" + ] + } + }, + { + "widget-type": "textbox", + "label": "Start Date", + "name": "startDate", + "widget-attributes": { + "placeholder": "Start date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } + }, + { + "widget-type": "textbox", + "label": "End Date", + "name": "endDate", + "widget-attributes": { + "placeholder": "End date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } + } + ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": {} + } + ] +} \ No newline at end of file