From ba80fb620ae263d6800b0ddee8d9889fd0b309d3 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Mon, 7 Feb 2022 12:57:34 +0100 Subject: [PATCH 01/26] feat(sink-elasticsearch): WIP test --- .../elasticsearch/ElasticsearchLogSink.java | 4 +- .../djobi/plugins/logging/WorkLoggerTest.java | 132 ++++++++---------- 2 files changed, 57 insertions(+), 79 deletions(-) diff --git a/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/sink/elasticsearch/ElasticsearchLogSink.java b/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/sink/elasticsearch/ElasticsearchLogSink.java index 1f177f1..47a0dc4 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/sink/elasticsearch/ElasticsearchLogSink.java +++ b/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/sink/elasticsearch/ElasticsearchLogSink.java @@ -57,9 +57,7 @@ private void init() { final String version = serverVersions.get(this.settingUrl); if (version == null || version.startsWith("2")) { - this.settingIndex += "/doc"; - } else { - this.settingIndex += "/_doc"; + logger.error("Only support ES >= 7"); } } } diff --git a/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java b/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java index 3b25b3c..61d09c3 100644 --- a/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java +++ b/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java @@ -27,7 +27,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import java.io.IOException; -import java.util.List; import java.util.Map; @ExtendWith(MyTestRunner.class) @@ -40,57 +39,46 @@ class WorkLoggerTest { static private void setup() throws IOException { final HttpMock httpMock = MyTestRunner.injector.getInstance(HttpMock.class); - final String responseComponents = "{\n" + - " \"href\": \"http://localhost:8080/api/v1/clusters/DataHub/services/SPARK/components/SPARK_THRIFTSERVER\",\n" + - " \"ServiceComponentInfo\": {\n" + - " \"category\": \"SLAVE\",\n" + - " \"cluster_name\": \"DataHub\",\n" + - " \"component_name\": \"SPARK_THRIFTSERVER\",\n" + - " \"display_name\": \"Spark Thrift Server\",\n" + - " \"init_count\": 0,\n" + - " \"install_failed_count\": 0,\n" + - " \"installed_count\": 0,\n" + - " \"recovery_enabled\": \"false\",\n" + - " \"service_name\": \"SPARK\",\n" + - " \"started_count\": 1,\n" + - " \"state\": \"STARTED\",\n" + - " \"total_count\": 1,\n" + - " \"unknown_count\": 0\n" + - " },\n" + - " \"host_components\": [\n" + - " {\n" + - " \"href\": \"http://localhost:8080/api/v1/clusters/DataHub/hosts/my-host/host_components/SPARK_THRIFTSERVER\",\n" + - " \"HostRoles\": {\n" + - " \"cluster_name\": \"DataHub\",\n" + - " \"component_name\": \"SPARK_THRIFTSERVER\",\n" + - " \"host_name\": \"my-host\"\n" + - " }\n" + - " }\n" + - " ]\n" + - "}"; - - final String responseRestartAction = "{\n" + - " \"href\": \"http://localhost:8080/api/v1/clusters/datahub_dev/requests/102\",\n" + - " \"Requests\": {\n" + - " \"id\": 102,\n" + - " \"status\": \"Accepted\"\n" + - " }\n" + - "}"; - httpMock - .start(8080) - .addEndpoint("get_components", httpMock - .endpoint() - .responseAsJSON() - .ifRequestRegex("/api/v1/clusters/([a-z]*)/services/([a-z]*)/components/([a-z]*)\\??") - .withResponseBody(responseComponents) - ) - .addEndpoint("action_restart", httpMock - .endpoint() - .responseAsJSON() - .ifRequestRegex("/api/v1/clusters/([a-z]*)/requests\\??") - .withResponseBody(responseRestartAction) - ) + .start(8080) + .addEndpoint("home", httpMock + .endpoint() + .responseAsJSON() + .ifRequest("GET", "/") + .withResponseBody("{ 'name' : 'elasticsearch', 'version' : { 'number' : '7.16.2' } }".replace("'", "\"")) + ) + .addEndpoint("index_delete", httpMock + .endpoint() + .responseAsJSON() + .ifMethod("DELETE") + .withResponseBody("{ \"acknowledged\":true }") + ) + .addEndpoint("index_send_document", httpMock + .endpoint() + .responseAsJSON() + .ifMethod("POST") + .withResponseBody("{ \"acknowledged\":true, \"_id\" : \"toto\" }") + ) + .addEndpoint("index_send_document", httpMock + .endpoint() + .responseAsJSON() + .ifMethod("PUT") + .withResponseBody("{ \"acknowledged\":true, \"_id\" : \"toto\" }") + ) + .addEndpoint("count", httpMock + .endpoint() + .responseAsJSON() + .ifMethod("GET") + .ifRequestRegex("/([a-z-]+)/_count") + .withResponseBody("{ \"count\" : 1}") + ) + .addEndpoint("get_document", httpMock + .endpoint() + .responseAsJSON() + .ifMethod("GET") + .ifRequestRegex("/djobi-jobs/(.+)") + .withResponseBody("{ \"_source\" : { \"pipeline\" : { \"name\" : \"good_dummy2.yml\"} } }") + ) ; } @@ -135,22 +123,26 @@ void testParameterJSONSerialization() throws Exception { @Test @Tag("IntegrationTest") void testElasticsearchSink() throws Exception { + final String configStr = "djobi.plugins.logger { sinks { " + + "jobs { enabled = true, type = elasticsearch, options { url = \"http://localhost:8080\", index = \"djobi-jobs\" } } \n" + + "stages { enabled = true, type = elasticsearch, options { url = \"http://localhost:8080\", index = \"djobi-stages\" } } \n" + + "metrics { enabled = false } \n" + + "" + + "} }"; - final String configPrefix = "djobi.plugins.logger.sinks."; final Djobi application = - new ApplicationBuilder() - .configure(ConfigFactory.parseMap( - MyMapUtils.mapString( - configPrefix + LoggerTypes.TYPE_JOBS + ".type", "elasticsearch", - configPrefix + LoggerTypes.TYPE_STAGES + ".type", "elasticsearch", - configPrefix + LoggerTypes.TYPE_METRICS + ".type", "" - ) - ).withFallback(ConfigFactory.load().resolveWith(ConfigFactory.parseMap(MyMapUtils.mapString("loggerType", "elasticsearch"))))) - .addPlugin(new DefaultActionsPlugin()) - .addPlugin(new LoggingPlugin()) - .addDependency(Reporter.class, TestStdoutReporter.class) - .build(); + new ApplicationBuilder() + .configure( + ConfigFactory + .parseString(configStr) + .withFallback(ConfigFactory.load()) + ) + .addPlugin(new DefaultActionsPlugin()) + .addPlugin(new LoggingPlugin()) + .addDependency(Reporter.class, TestStdoutReporter.class) + .build() + ; final Injector injector = application.getInjector(); final JobRunStartSubscriber jobRunStartSubscriber = injector.getInstance(JobRunStartSubscriber.class); @@ -180,8 +172,6 @@ void testElasticsearchSink() throws Exception { Assertions.assertEquals(1, c); assertJobLog(elasticsearchUtils, esSink, pipeline); - - assertStageLog(elasticsearchUtils, stageSink, pipeline); } private void assertJobLog(ElasticsearchUtils elasticsearchUtils, ElasticsearchLogSink esSink, Pipeline pipeline) throws Exception { @@ -192,14 +182,4 @@ private void assertJobLog(ElasticsearchUtils elasticsearchUtils, ElasticsearchLo Assertions.assertEquals("good_dummy2.yml", MyMapUtils.browse(doc, "pipeline.name")); } - private void assertStageLog(ElasticsearchUtils elasticsearchUtils, ElasticsearchLogSink esSink, Pipeline pipeline) throws Exception { - Map result = elasticsearchUtils.query(esSink.getSettingUrl(), esSink.getSettingIndex(), "_search?q=job.uid:" + pipeline.getJob(0).getUid()); - - Assertions.assertNotNull(result); - - List hits = (List) MyMapUtils.browse(result, "hits.hits"); - - Assertions.assertEquals(3, hits.size()); - } - } From db5bb48855a744f24133eecc4db1472c226d2289 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:34:04 +0100 Subject: [PATCH 02/26] fix: log4j xml --- djobi-cli/src/test/resources/{log4j.xml => log4j2.xml} | 2 +- djobi-cli/src/test/resources/pipelines/mono.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename djobi-cli/src/test/resources/{log4j.xml => log4j2.xml} (94%) diff --git a/djobi-cli/src/test/resources/log4j.xml b/djobi-cli/src/test/resources/log4j2.xml similarity index 94% rename from djobi-cli/src/test/resources/log4j.xml rename to djobi-cli/src/test/resources/log4j2.xml index a26ba9e..ed65c66 100644 --- a/djobi-cli/src/test/resources/log4j.xml +++ b/djobi-cli/src/test/resources/log4j2.xml @@ -1,5 +1,5 @@ - + diff --git a/djobi-cli/src/test/resources/pipelines/mono.yml b/djobi-cli/src/test/resources/pipelines/mono.yml index fd3204d..6b5c1fd 100644 --- a/djobi-cli/src/test/resources/pipelines/mono.yml +++ b/djobi-cli/src/test/resources/pipelines/mono.yml @@ -6,7 +6,7 @@ jobs: parameters: country: fr stages: - query: - kind: "fs-input" - spec: - path: "./data/{{country}}.txt" + - name: query + kind: "fs-input" + spec: + path: "./data/{{country}}.txt" From ba234dad999a1748cf717c37bd19f6a94de1e8e1 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:35:22 +0100 Subject: [PATCH 03/26] feat: add pipeline request factory --- .../cli/utils/PipelineRequestFactory.java | 98 +++++++++++++++++++ .../cli/utils/PipelineRequestFactoryTest.java | 48 +++++++++ 2 files changed, 146 insertions(+) create mode 100644 djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java create mode 100644 djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java new file mode 100644 index 0000000..6000d7b --- /dev/null +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java @@ -0,0 +1,98 @@ +package io.datatok.djobi.cli.utils; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.datatok.djobi.engine.PipelineExecutionRequest; +import io.datatok.djobi.plugins.report.VerbosityLevel; +import io.datatok.djobi.utils.EnvProvider; +import io.datatok.djobi.utils.MetaUtils; +import io.datatok.djobi.utils.MyMapUtils; + +import java.nio.channels.Pipe; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +@Singleton +public class PipelineRequestFactory { + + static public final String ENV_PREFIX_META = "META"; + + static public final String ENV_PREFIX_ARG = "ARG"; + + @Inject + MetaUtils metaUtils; + + @Inject + EnvProvider envProvider; + + /** + * For dev, test purposes + */ + private PipelineExecutionRequest lastObjectBuilt; + + public PipelineExecutionRequest build(final String pipelineDefinitionPath) { + return lastObjectBuilt = new PipelineExecutionRequest(pipelineDefinitionPath); + } + + public PipelineExecutionRequest build( + final String inPipelinePath, + final Map inArgumentMap, + final Map inMetaMap, + final String inJobsFilter, + final String inPhasesFilter, + final boolean[] verbosity + ) { + final PipelineExecutionRequest pipelineRequest = new PipelineExecutionRequest(); + + final Map argsMap = MyMapUtils.merge(envProvider.getScopedStartsWith(ENV_PREFIX_ARG), inArgumentMap); + final Map metasMap = + metaUtils.clean( + MyMapUtils.merge(envProvider.getScopedStartsWith(ENV_PREFIX_META), inMetaMap) + ); + + pipelineRequest + .setPipelineDefinitionPath(inPipelinePath) + .setRaw(argsMap) + .setJobsFilter(Arrays.asList(inJobsFilter.split(","))) + .setJobPhases(Arrays.asList(inPhasesFilter.split(","))) + .setMeta(metasMap) + .setVerbosity(getVerbosity(verbosity)) + ; + + return lastObjectBuilt = pipelineRequest; + } + + /** + * For dev, test purposes + * @return PipelineExecutionRequest + */ + public PipelineExecutionRequest getLastObjectBuilt() { + return lastObjectBuilt; + } + + private VerbosityLevel getVerbosity(boolean[] verbosity) { + int l = verbosity.length; + + if (l < 1) { + return VerbosityLevel.NORMAL; + } + + if (l < 2) { + return VerbosityLevel.VERBOSE; + } + + if (l < 3) { + return VerbosityLevel.VERY_VERBOSE; + } + + if (l < 4) { + return VerbosityLevel.VERY_VERY_VERBOSE; + } + + return VerbosityLevel.ALICIA; + } + +} diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java new file mode 100644 index 0000000..7542e37 --- /dev/null +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java @@ -0,0 +1,48 @@ +package io.datatok.djobi.cli.utils; + +import com.google.common.collect.Lists; +import com.google.inject.Inject; +import io.datatok.djobi.engine.PipelineExecutionRequest; +import io.datatok.djobi.utils.EnvProvider; +import io.datatok.djobi.utils.MyMapUtils; +import org.apache.commons.collections.ListUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Map; + +public class PipelineRequestFactoryTest { + + @Inject + PipelineRequestFactory pipelineRequestFactory; + + @Inject + EnvProvider envProvider; + + @Test + public void testCLIArgsAndEnv() { + envProvider + .clearScopedCache() + .setScoped("META_TITLE", "hello world") + .setScoped("ARG_DATE", "today") + .setScoped("ARG_TITLE", "from env") + ; + + PipelineExecutionRequest pipelineExecutionRequest = pipelineRequestFactory.build( + "", + MyMapUtils.mapString("date", "yesterday"), + MyMapUtils.mapString(), + "a,b", + "", + new boolean[]{} + ); + + Assertions.assertEquals("yesterday", pipelineExecutionRequest.getRaw().get("date")); + Assertions.assertEquals("from env", pipelineExecutionRequest.getRaw().get("title")); + Assertions.assertEquals(Arrays.asList("a", "b"), pipelineExecutionRequest.getJobsFilter()); + Assertions.assertEquals("hello world", pipelineExecutionRequest.getMeta().get("title")); + } + +} From 45f02cd204a73bba7aa524c23ef2680c404c10a9 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:35:31 +0100 Subject: [PATCH 04/26] feat(cli): upgrade picocli --- djobi-cli/build.gradle | 2 +- .../src/main/java/io/datatok/djobi/Main.java | 10 ++- .../io/datatok/djobi/cli/CommandFactory.java | 10 +-- .../io/datatok/djobi/cli/CommandKernel.java | 25 ++++++ .../cli/commands/DumpPipelineCommand.java | 8 +- .../{DjobiCommand.java => RootCommand.java} | 16 +--- .../cli/commands/RunPipelineCommand.java | 78 +++++++------------ .../djobi/cli/DumpActionCommandTest.java | 4 +- .../datatok/djobi/cli/DumpEnvCommandTest.java | 13 ++-- .../djobi/cli/DumpPipelineCommandTest.java | 6 +- .../djobi/cli/RunPipelineCommandTest.java | 26 ++++++- .../cli/{ => utils}/CLIOutUtilsTest.java | 2 +- .../djobi/cli/{ => utils}/CLIUtilsTest.java | 2 +- 13 files changed, 109 insertions(+), 93 deletions(-) create mode 100644 djobi-cli/src/main/java/io/datatok/djobi/cli/CommandKernel.java rename djobi-cli/src/main/java/io/datatok/djobi/cli/commands/{DjobiCommand.java => RootCommand.java} (75%) rename djobi-cli/src/test/java/io/datatok/djobi/cli/{ => utils}/CLIOutUtilsTest.java (89%) rename djobi-cli/src/test/java/io/datatok/djobi/cli/{ => utils}/CLIUtilsTest.java (92%) diff --git a/djobi-cli/build.gradle b/djobi-cli/build.gradle index cc075be..7019141 100644 --- a/djobi-cli/build.gradle +++ b/djobi-cli/build.gradle @@ -42,7 +42,7 @@ dependencies { ) implementation( - [group: 'info.picocli', name: 'picocli', version: '3.9.5'] + [group: 'info.picocli', name: 'picocli', version: '4.6.2'] ) } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/Main.java b/djobi-cli/src/main/java/io/datatok/djobi/Main.java index 7f2bc8b..c066362 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/Main.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/Main.java @@ -1,15 +1,17 @@ package io.datatok.djobi; +import com.google.inject.Injector; import io.datatok.djobi.application.ApplicationBuilder; import io.datatok.djobi.application.Djobi; import io.datatok.djobi.application.exceptions.BuildApplicationException; -import io.datatok.djobi.cli.CommandFactory; +import io.datatok.djobi.cli.CommandKernel; import io.datatok.djobi.cli.StdoutReporter; import io.datatok.djobi.plugins.report.Reporter; import io.datatok.djobi.plugins.s3.S3Plugin; import io.datatok.djobi.plugins.stages.DefaultActionsPlugin; import io.datatok.djobi.spark.SparkPlugin; import io.datatok.djobi.utils.ClassUtils; +import picocli.CommandLine; import java.io.Serializable; @@ -52,8 +54,12 @@ public static void main(String[] args) { return ; } + Injector injector = application.getInjector(); + CommandKernel commandKernel = injector.getInstance(CommandKernel.class); + CommandLine rootCommandImpl = commandKernel.getRootCommand(); + try { - application.getInjector().getInstance(CommandFactory.class).run(args); + rootCommandImpl.execute(args); } catch(RuntimeException e) { e.printStackTrace(); } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java index b5754f1..e9da7f1 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java @@ -1,12 +1,13 @@ package io.datatok.djobi.cli; import com.google.inject.Injector; -import io.datatok.djobi.cli.commands.DjobiCommand; import org.fusesource.jansi.AnsiConsole; import picocli.CommandLine; import javax.inject.Inject; import javax.inject.Singleton; +import java.util.LinkedHashMap; +import java.util.Map; @Singleton public class CommandFactory implements CommandLine.IFactory { @@ -18,12 +19,11 @@ public CommandFactory() { AnsiConsole.systemInstall(); } - public void run(final String[] args) { - CommandLine.run(DjobiCommand.class, this, args); - } - @Override public K create(Class cls) throws Exception { + if (cls.equals(Map.class)) { + return CommandLine.defaultFactory().create(cls); + } return injector.getInstance(cls); } } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandKernel.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandKernel.java new file mode 100644 index 0000000..9f8ea8d --- /dev/null +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandKernel.java @@ -0,0 +1,25 @@ +package io.datatok.djobi.cli; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import io.datatok.djobi.cli.commands.RootCommand; +import picocli.CommandLine; + +@Singleton +public class CommandKernel { + + @Inject + private CommandFactory commandFactory; + + @Inject + private Provider rootCommandProvider; + + public CommandLine getRootCommand() { + return new CommandLine(rootCommandProvider.get(), commandFactory); + } + + public void run(String... args) { + getRootCommand().execute(args); + } +} diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java index e7ded4c..9db3c07 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java @@ -1,10 +1,9 @@ package io.datatok.djobi.cli.commands; import com.fasterxml.jackson.core.JsonProcessingException; -import io.datatok.djobi.cli.utils.CLIUtils; +import io.datatok.djobi.cli.utils.PipelineRequestFactory; import io.datatok.djobi.engine.Job; import io.datatok.djobi.engine.Pipeline; -import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.engine.phases.ActionPhases; import io.datatok.djobi.engine.phases.StagePhaseMetaData; import io.datatok.djobi.engine.stage.ActionFactory; @@ -32,6 +31,9 @@ public class DumpPipelineCommand implements Runnable { @Inject ActionFactory actionFactory; + @Inject + PipelineRequestFactory pipelineRequestFactory; + @CommandLine.ParentCommand DumpCommand dumpCommand; @@ -51,7 +53,7 @@ public void run() { AnsiConsole.systemInstall(); try { - pipeline = pipelineLoader.get(PipelineExecutionRequest.build(pipelinePath, args).setJobsFilter(Arrays.asList(jobs.split(",")))); + pipeline = pipelineLoader.get(pipelineRequestFactory.build(pipelinePath).setJobsFilter(Arrays.asList(jobs.split(",")))); } catch (IOException e) { e.printStackTrace(); } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DjobiCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RootCommand.java similarity index 75% rename from djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DjobiCommand.java rename to djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RootCommand.java index 97cb5a9..017f1fd 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DjobiCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RootCommand.java @@ -27,21 +27,11 @@ ""}, subcommands = {DumpCommand.class, RunPipelineCommand.class} ) -public class DjobiCommand implements Runnable { - - @Inject - private CommandFactory commandFactory; - - @Inject - private Reporter reporter; - - @Inject - private ApplicationData app; - +public class RootCommand implements Runnable { public void run() { - reporter.output("Djobi version %s", app.getVersion()); + //reporter.output("Djobi version %s", app.getVersion()); - new CommandLine(this, commandFactory).usage(System.out); + //new CommandLine(this, commandFactory).usage(System.out); } } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java index 55b3261..8b6ee75 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java @@ -1,6 +1,7 @@ package io.datatok.djobi.cli.commands; import com.google.inject.Inject; +import io.datatok.djobi.cli.utils.PipelineRequestFactory; import io.datatok.djobi.cli.utils.CLIUtils; import io.datatok.djobi.engine.Engine; import io.datatok.djobi.engine.Pipeline; @@ -9,6 +10,7 @@ import io.datatok.djobi.plugins.report.OutVerbosity; import io.datatok.djobi.plugins.report.VerbosityLevel; import io.datatok.djobi.utils.MetaUtils; +import io.datatok.djobi.utils.MyMapUtils; import picocli.CommandLine; import java.io.IOException; @@ -19,24 +21,24 @@ public class RunPipelineCommand implements Runnable { @Inject - YAMLPipelineLoader pipelineLoader; + PipelineRequestFactory pipelineRequestFactory; @Inject - Engine pipelineRunner; + YAMLPipelineLoader pipelineLoader; @Inject - MetaUtils metaUtils; + Engine pipelineRunner; @Inject OutVerbosity outVerbosity; - @CommandLine.Option(paramLabel = "args", names = {"-A", "--args"}, description = "arguments (date, ...)") + @CommandLine.Option(paramLabel = "args", names = {"-a", "--arg"}, description = "arguments (date, ...)") Map args; @CommandLine.Option(paramLabel = "jobs", names = {"--jobs"}, description = "jobs filter", defaultValue = "") String jobs; - @CommandLine.Option(paramLabel = "run_metas", names = {"-M", "--meta"}, description = "pipeline run meta (for logging)") + @CommandLine.Option(paramLabel = "run_metas", names = {"-m", "--meta"}, description = "pipeline run meta (for logging)") Map runMetas; @CommandLine.Option( @@ -47,69 +49,41 @@ public class RunPipelineCommand implements Runnable { ) String phases; - @CommandLine.Parameters(paramLabel = "pipeline", arity = "1..*", description = "the pipeline directory path") + @CommandLine.Parameters(paramLabel = "pipeline", defaultValue = "${DJOBI_PIPELINE}" ,arity = "1..*", description = "the pipeline directory path") String pipelinePath; @CommandLine.Option(names = { "-v", "--verbose" }, description = "Verbose mode. Helpful for troubleshooting. " + "Multiple -v options increase the verbosity.") - private boolean[] verbose = new boolean[0]; + private boolean[] verbosityOption = new boolean[0]; @Override public void run() { Pipeline pipeline = null; - if (pipelinePath != null && !pipelinePath.isEmpty()) { + if (pipelinePath == null || pipelinePath.isEmpty()) { + CLIUtils.printError("pipeline is missing!"); + return ; + } - final PipelineExecutionRequest pipelineRequest = new PipelineExecutionRequest(args); + final PipelineExecutionRequest pipelineRequest = pipelineRequestFactory.build(pipelinePath, args, runMetas, jobs, phases, verbosityOption); - pipelineRequest - .setPipelineDefinitionPath(pipelinePath) - .setJobsFilter(Arrays.asList(this.jobs.split(","))) - .setJobPhases(Arrays.asList(this.phases.split(","))) - .setMeta(metaUtils.clean(runMetas)) - .setVerbosity(getVerbosity()) - ; + outVerbosity.setVerbosityLevel(pipelineRequest.getVerbosity()); - outVerbosity.setVerbosityLevel(pipelineRequest.getVerbosity()); + try { + pipeline = pipelineLoader.get(pipelineRequest); + pipelineRequest.setPipeline(pipeline); + } catch (IOException e) { + CLIUtils.printError(e.getMessage()); + e.printStackTrace(); + } + if (pipeline != null) { try { - pipeline = pipelineLoader.get(pipelineRequest); - pipelineRequest.setPipeline(pipeline); - } catch (IOException e) { + pipelineRunner.run(pipelineRequest); + } catch (Exception e) { CLIUtils.printError(e.getMessage()); e.printStackTrace(); } - - if (pipeline != null) { - try { - pipelineRunner.run(pipelineRequest); - } catch (Exception e) { - CLIUtils.printError(e.getMessage()); - e.printStackTrace(); - } - } } } - - private VerbosityLevel getVerbosity() { - int l = verbose.length; - - if (l < 1) { - return VerbosityLevel.NORMAL; - } - - if (l < 2) { - return VerbosityLevel.VERBOSE; - } - - if (l < 3) { - return VerbosityLevel.VERY_VERBOSE; - } - - if (l < 4) { - return VerbosityLevel.VERY_VERY_VERBOSE; - } - - return VerbosityLevel.ALICIA; - } -} +} \ No newline at end of file diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java index 2db4c83..6152954 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java @@ -13,7 +13,7 @@ class DumpActionCommandTest { @Inject - private CommandFactory commandFactory; + private CommandKernel commandKernel; @Test void jsonExample() throws Exception { run(new String[]{"dump", "--format", "json", "action"}); @@ -24,7 +24,7 @@ class DumpActionCommandTest { } private void run(final String[] args) { - commandFactory.run(args); + commandKernel.run(args); } } diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java index 85f20cf..d6b015e 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import picocli.CommandLine; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -13,7 +14,7 @@ class DumpEnvCommandTest { @Inject - private CommandFactory commandFactory; + private CommandKernel commandKernel; private final ByteArrayOutputStream myOut = new ByteArrayOutputStream(); @@ -22,7 +23,7 @@ class DumpEnvCommandTest { } @Test void jsonExample() throws Exception { - run(new String[]{"dump", "--format", "json", "env"}); + run("dump", "--format", "json", "env"); final String output = captureStdout(); @@ -31,7 +32,7 @@ class DumpEnvCommandTest { } @Test void tableExample() { - run(new String[]{"dump", "--format", "plain", "env"}); + run("dump", "--format", "plain", "env"); final String output = captureStdout(); @@ -39,7 +40,7 @@ class DumpEnvCommandTest { } @Test void tableWithGrepExample() { - run(new String[]{"dump", "--format", "plain", "--grep", "hello,djobi", "env"}); + run("dump", "--format", "plain", "--grep", "hello,djobi", "env"); final String output = captureStdout(); @@ -48,8 +49,8 @@ class DumpEnvCommandTest { Assertions.assertTrue(output.split("\n").length < 5); } - private void run(final String[] args) { - commandFactory.run(args); + private void run(String... args) { + commandKernel.getRootCommand().execute(args); } private String captureStdout() { diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java index 2606d91..b4742a4 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java @@ -16,7 +16,7 @@ class DumpPipelineCommandTest { @Inject - private CommandFactory commandFactory; + private CommandKernel commandKernel; @Inject private DumpPipelineCommand dumpPipelineCommand; @@ -28,7 +28,7 @@ class DumpPipelineCommandTest { } @Test void basic() { - new CommandLine(dumpPipelineCommand).parse(new String[]{"--args", "date=today", "-Ahello=toto", "./src/test/resources/pipelines/good.yml"}); + new CommandLine(dumpPipelineCommand).parseArgs("--args", "date=today", "-Ahello=toto", "./src/test/resources/pipelines/good.yml"); Assertions.assertEquals(MyMapUtils.map("date", "today", "hello", "toto"), dumpPipelineCommand.args); } @@ -51,7 +51,7 @@ class DumpPipelineCommandTest { } private void run(final String[] args) { - commandFactory.run(args); + commandKernel.run(args); } private String captureStdout() { diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java index 90f3eb0..b494d37 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java @@ -1,6 +1,8 @@ package io.datatok.djobi.cli; import com.google.inject.Inject; +import io.datatok.djobi.cli.utils.PipelineRequestFactory; +import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.plugins.report.OutVerbosity; import io.datatok.djobi.plugins.report.VerbosityLevel; import org.junit.jupiter.api.Assertions; @@ -13,11 +15,14 @@ public class RunPipelineCommandTest { @Inject - private CommandFactory commandFactory; + private CommandKernel commandKernel; @Inject private OutVerbosity outVerbosity; + @Inject + PipelineRequestFactory pipelineRequestFactory; + private final ByteArrayOutputStream myOut = new ByteArrayOutputStream(); @BeforeEach @@ -27,16 +32,29 @@ void setup() { @Test void runNormal() { - commandFactory.run(new String[]{"run", "./src/test/resources/pipelines/mono.yml"}); + commandKernel.run("run", "./src/test/resources/pipelines/mono.yml"); Assertions.assertEquals(outVerbosity.getVerbosityLevel(), VerbosityLevel.NORMAL); Assertions.assertTrue(outVerbosity.isNotQuiet()); Assertions.assertFalse(outVerbosity.isVerbose()); } + @Test + void runWithMetaAndArgument() { + commandKernel.run("run", "-a", "date=yesterday", "./src/test/resources/pipelines/mono.yml"); + + PipelineExecutionRequest lastObj = pipelineRequestFactory.getLastObjectBuilt(); + + Assertions.assertEquals(outVerbosity.getVerbosityLevel(), VerbosityLevel.NORMAL); + Assertions.assertTrue(outVerbosity.isNotQuiet()); + Assertions.assertFalse(outVerbosity.isVerbose()); + + Assertions.assertEquals("yesterday", lastObj.getRaw().get("date")); + } + @Test void runVerbose() { - commandFactory.run(new String[]{"run", "-v", "./src/test/resources/pipelines/mono.yml"}); + commandKernel.run("run", "-v", "./src/test/resources/pipelines/mono.yml"); Assertions.assertEquals(outVerbosity.getVerbosityLevel(), VerbosityLevel.VERBOSE); Assertions.assertTrue(outVerbosity.isNotQuiet()); @@ -46,7 +64,7 @@ void runVerbose() { @Test void runVeryVerbose() { - commandFactory.run(new String[]{"run", "-vv", "./src/test/resources/pipelines/mono.yml"}); + commandKernel.run("run", "-vv", "./src/test/resources/pipelines/mono.yml"); Assertions.assertEquals(outVerbosity.getVerbosityLevel(), VerbosityLevel.VERY_VERBOSE); Assertions.assertTrue(outVerbosity.isVerbose()); diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIOutUtilsTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIOutUtilsTest.java similarity index 89% rename from djobi-cli/src/test/java/io/datatok/djobi/cli/CLIOutUtilsTest.java rename to djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIOutUtilsTest.java index 2f145ea..529454b 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIOutUtilsTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIOutUtilsTest.java @@ -1,4 +1,4 @@ -package io.datatok.djobi.cli; +package io.datatok.djobi.cli.utils; import io.datatok.djobi.cli.utils.CLIOutUtils; import org.junit.jupiter.api.Assertions; diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIUtilsTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java similarity index 92% rename from djobi-cli/src/test/java/io/datatok/djobi/cli/CLIUtilsTest.java rename to djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java index b17d582..5d976d7 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIUtilsTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java @@ -1,4 +1,4 @@ -package io.datatok.djobi.cli; +package io.datatok.djobi.cli.utils; import io.datatok.djobi.cli.utils.CLIUtils; import org.junit.jupiter.api.Assertions; From 68810e4b036550d95f416cf78a6c83ef53b51131 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:35:59 +0100 Subject: [PATCH 05/26] feat: add env provider feat: add env provider --- .../djobi/configuration/EnvVarLoader.java | 69 ++--------- .../io/datatok/djobi/utils/EnvProvider.java | 111 ++++++++++++++++++ .../djobi/configuration/EnvVarLoaderTest.java | 12 +- .../datatok/djobi/utils/EnvProviderTest.java | 38 ++++++ 4 files changed, 170 insertions(+), 60 deletions(-) create mode 100644 djobi-core/src/main/java/io/datatok/djobi/utils/EnvProvider.java create mode 100644 djobi-core/src/test/java/io/datatok/djobi/utils/EnvProviderTest.java diff --git a/djobi-core/src/main/java/io/datatok/djobi/configuration/EnvVarLoader.java b/djobi-core/src/main/java/io/datatok/djobi/configuration/EnvVarLoader.java index b49ac38..9ac4027 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/configuration/EnvVarLoader.java +++ b/djobi-core/src/main/java/io/datatok/djobi/configuration/EnvVarLoader.java @@ -1,76 +1,33 @@ package io.datatok.djobi.configuration; +import com.google.inject.Inject; +import com.google.inject.Singleton; import com.typesafe.config.Config; import com.typesafe.config.ConfigException; import com.typesafe.config.ConfigFactory; +import io.datatok.djobi.utils.EnvProvider; import java.util.HashMap; import java.util.Locale; import java.util.Map; +@Singleton public class EnvVarLoader { - private static final String ENV_VAR_PREFIX = "DJOBI_"; - static Config loadEnvVariablesOverrides() { - final Map env = new HashMap<>(System.getenv()); - final Map result = new HashMap<>(); + static final public String PREFIX = "CONFIG"; - for (String key : env.keySet()) { - if (key.startsWith(ENV_VAR_PREFIX) && !key.equals(ENV_VAR_PREFIX)) { - result.put(envVariableAsProperty(key), env.get(key)); - } - } + @Inject + EnvProvider envProvider; - return ConfigFactory.parseMap(result, "env variables"); - } - - static private String envVariableAsProperty(String variable) throws ConfigException { - StringBuilder builder = new StringBuilder(); - - String strippedPrefix = variable.substring(EnvVarLoader.ENV_VAR_PREFIX.length()); - - strippedPrefix = strippedPrefix.toLowerCase(Locale.ROOT); + public Config loadEnvVariablesOverrides() { + final Map rawMap = EnvProvider.formatKeys( + envProvider.getScopedStartsWith(PREFIX) + ); - int underscores = 0; - for (char c : strippedPrefix.toCharArray()) { - if (c == '_') { - underscores++; - } else { - if (underscores > 0 && underscores < 4) { - builder.append(underscoreMappings(underscores)); - } else if (underscores > 3) { - throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores."); - } - underscores = 0; - builder.append(c); - } - } - - if (underscores > 0 && underscores < 4) { - builder.append(underscoreMappings(underscores)); - } else if (underscores > 3) { - throw new ConfigException.BadPath(variable, "Environment variable contains an un-mapped number of underscores."); - } - - return builder.toString(); + return ConfigFactory.parseMap(rawMap, "env variables"); } - private static char underscoreMappings(int num) { - // Rationale on name mangling: - // - // Most shells (e.g. bash, sh, etc.) doesn't support any character other - // than alphanumeric and `_` in environment variables names. - // In HOCON the default separator is `.` so it is directly translated to a - // single `_` for convenience; `-` and `_` are less often present in config - // keys but they have to be representable and the only possible mapping is - // `_` repeated. - switch (num) { - case 1: return '.'; - case 2: return '-'; - case 3: return '_'; - default: return 0; - } - } + } diff --git a/djobi-core/src/main/java/io/datatok/djobi/utils/EnvProvider.java b/djobi-core/src/main/java/io/datatok/djobi/utils/EnvProvider.java new file mode 100644 index 0000000..ebe2e59 --- /dev/null +++ b/djobi-core/src/main/java/io/datatok/djobi/utils/EnvProvider.java @@ -0,0 +1,111 @@ +package io.datatok.djobi.utils; + +import com.google.inject.Singleton; +import com.typesafe.config.ConfigException; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +@Singleton +public class EnvProvider { + + static final public String SEPARATOR = "_"; + + static final public String SCOPE_PREFIX_DEFAULT = "DJOBI" + SEPARATOR; + + private final Map overrides = new HashMap<>(); + + public String get(String key) { + return System.getenv(key); + } + + public EnvProvider setScoped(String key, String value) { + overrides.put(SCOPE_PREFIX_DEFAULT + key, value); + return this; + } + + public EnvProvider clearScopedCache() { + overrides.clear(); + return this; + } + + public String getScoped(String key) { + if (overrides.containsKey(key)) { + return overrides.get(key); + } + return System.getenv(SCOPE_PREFIX_DEFAULT + key); + } + + public Map getScopedStartsWith(final String prefix) { + final Map source = new HashMap<>(System.getenv()); + + source.putAll(overrides); + + final String prefixAll = SCOPE_PREFIX_DEFAULT + prefix; + + return + source + .entrySet() + .stream() + .filter(e -> e.getKey().startsWith(prefixAll)) + .collect(Collectors.toMap(e -> envToMapKey(prefixAll, e.getKey()), Map.Entry::getValue)); + } + + static public String envToMapKey(String prefix, String key) { + return key.substring(prefix.length() + 1).toLowerCase(Locale.ROOT); + } + + static public Map formatKeys(Map inMap) { + return + inMap + .entrySet() + .stream() + .collect(Collectors.toMap(e -> EnvProvider.formatKey(e.getKey()), Map.Entry::getValue)); + } + + static public String formatKey(String variable) { + final StringBuilder builder = new StringBuilder(); + int underscores = 0; + + for (char c : variable.toLowerCase(Locale.ROOT).toCharArray()) { + if (c == '_') { + underscores++; + } else { + if (underscores > 0 && underscores < 4) { + builder.append(underscoreMappings(underscores)); + } else if (underscores > 3) { + builder.append('_'); + } + underscores = 0; + builder.append(c); + } + } + + if (underscores > 0 && underscores < 4) { + builder.append(underscoreMappings(underscores)); + } else if (underscores > 3) { + builder.append('_'); + } + + return builder.toString(); + } + + private static char underscoreMappings(int num) { + // Rationale on name mangling: + // + // Most shells (e.g. bash, sh, etc.) doesn't support any character other + // than alphanumeric and `_` in environment variables names. + // In HOCON the default separator is `.` so it is directly translated to a + // single `_` for convenience; `-` and `_` are less often present in config + // keys but they have to be representable and the only possible mapping is + // `_` repeated. + switch (num) { + case 1: return '.'; + case 2: return '-'; + case 3: return '_'; + default: return 0; + } + } +} diff --git a/djobi-core/src/test/java/io/datatok/djobi/configuration/EnvVarLoaderTest.java b/djobi-core/src/test/java/io/datatok/djobi/configuration/EnvVarLoaderTest.java index 0fb8470..77fc58d 100644 --- a/djobi-core/src/test/java/io/datatok/djobi/configuration/EnvVarLoaderTest.java +++ b/djobi-core/src/test/java/io/datatok/djobi/configuration/EnvVarLoaderTest.java @@ -1,5 +1,6 @@ package io.datatok.djobi.configuration; +import com.google.inject.Inject; import com.typesafe.config.Config; import io.datatok.djobi.test.MyTestRunner; import org.junit.jupiter.api.Assertions; @@ -10,17 +11,20 @@ @ExtendWith(MyTestRunner.class) class EnvVarLoaderTest { + @Inject + EnvVarLoader envVarLoader; + @Test void testPreCheck() throws Exception { withEnvironmentVariable("first", "first value") .and("DJOBI_", "second value") - .and("DJOBI_LOGGER_ENABLED", "false") + .and("DJOBI_LOG", "second value") + .and("DJOBI_CONFIG_LOGGER_ENABLED", "false") .execute(() -> { - - Config config = EnvVarLoader.loadEnvVariablesOverrides(); + Config config = envVarLoader.loadEnvVariablesOverrides(); Assertions.assertFalse(config.hasPath("toto")); - + Assertions.assertFalse(config.hasPath("log")); Assertions.assertTrue(config.hasPath("logger.enabled")); }); } diff --git a/djobi-core/src/test/java/io/datatok/djobi/utils/EnvProviderTest.java b/djobi-core/src/test/java/io/datatok/djobi/utils/EnvProviderTest.java new file mode 100644 index 0000000..bd32c1a --- /dev/null +++ b/djobi-core/src/test/java/io/datatok/djobi/utils/EnvProviderTest.java @@ -0,0 +1,38 @@ +package io.datatok.djobi.utils; + +import com.google.inject.Inject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +public class EnvProviderTest { + + @Inject + EnvProvider envProvider; + + @Test + public void testKeyFormatter() { + Assertions.assertEquals("logger.enabled", EnvProvider.formatKey("LOGGER_ENABLED")); + Assertions.assertEquals("logger-enabled", EnvProvider.formatKey("LOGGER__ENABLED")); + Assertions.assertEquals("logger_enabled", EnvProvider.formatKey("LOGGER___ENABLED")); + Assertions.assertEquals("my-logger.enabled", EnvProvider.formatKey("my__logger_ENABLED")); + } + + @Test + public void testEnvMap() { + envProvider + .clearScopedCache() + .setScoped("META_TITLE", "title") + .setScoped("MTA_NOP", "nop") + .setScoped("META_THIS_IS_IT", "this is it") + ; + + Map mp = envProvider.getScopedStartsWith("META"); + + Assertions.assertEquals(2, mp.size()); + Assertions.assertEquals("title", mp.get("title")); + Assertions.assertEquals("this is it", mp.get("this_is_it")); + } + +} From bc6cd2ce6961c953762fbaf567e286419c0c34df Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:40:14 +0100 Subject: [PATCH 06/26] style: misc --- .../djobi/engine/PipelineExecutionRequest.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java index 7db16d0..19f83b8 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java @@ -59,11 +59,11 @@ public class PipelineExecutionRequest { */ private List jobPhases; - static public PipelineExecutionRequest build(final String pipelineDefinitionPath) { + public static PipelineExecutionRequest build(final String pipelineDefinitionPath) { return new PipelineExecutionRequest(pipelineDefinitionPath); } - static public PipelineExecutionRequest build(final String pipelineDefinitionPath, final Map args) { + public static PipelineExecutionRequest build(final String pipelineDefinitionPath, final Map args) { return new PipelineExecutionRequest(args).setPipelineDefinitionPath(pipelineDefinitionPath); } @@ -81,6 +81,9 @@ public PipelineExecutionRequest(final Map args) { this.debug = this.raw.containsKey("debug"); } + public PipelineExecutionRequest() { + } + public PipelineExecutionRequest addArgument(final String k, final String v) { if (this.raw == null) { this.raw = new HashMap<>(); @@ -103,6 +106,11 @@ public Map getRaw() { return raw; } + public PipelineExecutionRequest setRaw(Map raw) { + this.raw = raw; + return this; + } + public PipelineExecutionRequest setPipelineDefinitionPath(String pipelineDefinitionPath) { this.pipelineDefinitionPath = pipelineDefinitionPath; return this; From 8f37cdbd207829c6d7fc311968ddd476b4bf09ac Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:58:27 +0100 Subject: [PATCH 07/26] chore: misc --- .../src/main/java/io/datatok/djobi/cli/CommandFactory.java | 3 ++- .../main/java/io/datatok/djobi/engine/flow/ConditionFlow.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java index e9da7f1..9f346b5 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java @@ -7,6 +7,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; @Singleton @@ -21,7 +22,7 @@ public CommandFactory() { @Override public K create(Class cls) throws Exception { - if (cls.equals(Map.class)) { + if (cls.equals(Map.class) || cls.equals(List.class)) { return CommandLine.defaultFactory().create(cls); } return injector.getInstance(cls); diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/flow/ConditionFlow.java b/djobi-core/src/main/java/io/datatok/djobi/engine/flow/ConditionFlow.java index 506ca46..0639a64 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/flow/ConditionFlow.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/flow/ConditionFlow.java @@ -25,6 +25,10 @@ public boolean test(final Stage stage) { return true; } + if (condition.equals("false") || condition.equals("no")) { + return false; + } + final String[] items = condition.split("\\s+"); final String operator = items[0]; From cd0b5e9aba58c7f5c951cc789b4f0852ec8d26e7 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:58:51 +0100 Subject: [PATCH 08/26] style: indent --- .../djobi/loaders/yaml/YAMLPipelineLoader.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java index 774a7a7..363e065 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java +++ b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java @@ -156,11 +156,17 @@ private Pipeline getImplementation(final PipelineExecutionRequest pipelineReques final ParameterBag jobParameters = actionArgFactory.resolve(jobDefinition.parameters, pipelineRequest); final Map jobContextParameters = new HashMap<>(); - if (jobDefinition.matrix == null) { + if (jobDefinition.contexts == null) { jobContextParameters.put(DEFAULT_CONTEXT, jobParameters); } else { - for (String c : jobDefinition.matrix.keySet()) { - jobContextParameters.put(c, ParameterBag.merge(jobParameters, actionArgFactory.resolve(jobDefinition.matrix.get(c), pipelineRequest))); + for (String c : jobDefinition.contexts.keySet()) { + jobContextParameters.put(c, ParameterBag.merge( + jobParameters, + actionArgFactory.resolve( + jobDefinition.contexts.get(c), + pipelineRequest + ) + )); } } From 31b278d18bc811b639482dc6e659fde082adfeeb Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Fri, 11 Feb 2022 21:59:08 +0100 Subject: [PATCH 09/26] fix: remove matrix field fix: remove matrix field --- .../djobi/loaders/yaml/pojo/JobDefinition.java | 18 ++++++++++-------- .../src/test/resources/pipelines/good.yml | 5 ++--- .../test/resources/pipelines/good_dummy.yml | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/JobDefinition.java b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/JobDefinition.java index 90ec948..2f5803b 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/JobDefinition.java +++ b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/JobDefinition.java @@ -12,17 +12,24 @@ public class JobDefinition { + /** + * Stages definition + */ public List stages; + /** + * Parameters definition + */ public Map parameters; /** - * @since v5.0.0 + * Labels definition */ public Map labels; - public Map> matrix; - + /** + * Define a matrix to execute job as variant. + */ public Map> contexts; public String id; @@ -35,11 +42,6 @@ public JobDefinition() { this.uid = UUID.randomUUID().toString(); } - public JobDefinition setContexts(Map> context) { - this.matrix = context; - return this; - } - public Job getJob(final Pipeline pipeline, final ParameterBag run) { final Job job = new Job(); diff --git a/djobi-core/src/test/resources/pipelines/good.yml b/djobi-core/src/test/resources/pipelines/good.yml index a17881e..7cbefcf 100644 --- a/djobi-core/src/test/resources/pipelines/good.yml +++ b/djobi-core/src/test/resources/pipelines/good.yml @@ -26,7 +26,7 @@ jobs: p1: p1 p2: p2 pd: job - matrix: + contexts: a: pjc1: a pd: context_a @@ -37,8 +37,7 @@ jobs: pd: context_b file: not_found stages: - - name: setup_file - name: setup-file-overriden-name + - name: setup-file-overriden-name kind: fs-input labels: io.datatok.djobi/stage-type: input diff --git a/djobi-core/src/test/resources/pipelines/good_dummy.yml b/djobi-core/src/test/resources/pipelines/good_dummy.yml index 5a94ea1..cce216c 100644 --- a/djobi-core/src/test/resources/pipelines/good_dummy.yml +++ b/djobi-core/src/test/resources/pipelines/good_dummy.yml @@ -12,7 +12,7 @@ jobs: p1: p1 p2: p2 pd: job - matrix: + contexts: a: pjc1: a pd: context_a From 7cf45261b947beb4e5c07ce3e0e7d3874fc8d2b4 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 12 Feb 2022 19:32:41 +0100 Subject: [PATCH 10/26] feat: add test --- .../datatok/djobi/cli/RunPipelineCommandTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java index b494d37..b794e70 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java @@ -1,6 +1,7 @@ package io.datatok.djobi.cli; import com.google.inject.Inject; +import com.typesafe.config.Config; import io.datatok.djobi.cli.utils.PipelineRequestFactory; import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.plugins.report.OutVerbosity; @@ -12,6 +13,8 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import static com.github.stefanbirkner.systemlambda.SystemLambda.*; + public class RunPipelineCommandTest { @Inject @@ -52,6 +55,18 @@ void runWithMetaAndArgument() { Assertions.assertEquals("yesterday", lastObj.getRaw().get("date")); } + @Test + void runPipelinePathFromEnv() throws Exception { + withEnvironmentVariable("DJOBI_PIPELINE", "./src/test/resources/pipelines/mono.yml") + .execute(() -> { + commandKernel.run("run"); + + PipelineExecutionRequest lastObj = pipelineRequestFactory.getLastObjectBuilt(); + + Assertions.assertEquals("./src/test/resources/pipelines/mono.yml", lastObj.getPipelineDefinitionPath()); + }); + } + @Test void runVerbose() { commandKernel.run("run", "-v", "./src/test/resources/pipelines/mono.yml"); From 46bcf67fc2d1acca85de6672a7e6a5449e8b8a90 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sat, 12 Feb 2022 19:32:55 +0100 Subject: [PATCH 11/26] feat: upgrade vendors --- djobi-core/build.gradle | 4 ++-- djobi-tests/build.gradle | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/djobi-core/build.gradle b/djobi-core/build.gradle index dece905..c125203 100644 --- a/djobi-core/build.gradle +++ b/djobi-core/build.gradle @@ -38,9 +38,9 @@ dependencies { [group: 'org.apache.commons', name: 'commons-csv', version:'1.6'], [group: 'com.github.spullara.mustache.java', name: 'compiler', version:'0.9.6'], [group: 'org.yaml', name: 'snakeyaml', version:'1.29'], - [group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.3.1'], + [group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'], [group: "co.elastic.apm", name: "apm-agent-api", version: '1.28.4'], - [group: 'com.google.inject', name: 'guice', version: '5.0.1'], + [group: 'com.google.inject', name: 'guice', version: '5.1.0'], [group: 'com.typesafe', name: 'config', version:'1.4.1'] ) diff --git a/djobi-tests/build.gradle b/djobi-tests/build.gradle index c30d7fa..f909089 100644 --- a/djobi-tests/build.gradle +++ b/djobi-tests/build.gradle @@ -15,8 +15,8 @@ dependencies { testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.2' api( - [group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.25'], - [group: 'com.github.stefanbirkner', name: 'system-lambda', version: '1.2.0'] + [group: 'org.slf4j', name: 'slf4j-log4j12', version: '1.7.36'], + [group: 'com.github.stefanbirkner', name: 'system-lambda', version: '1.2.1'] ) // https://mvnrepository.com/artifact/org.json/json @@ -25,8 +25,7 @@ dependencies { compileOnly project(':djobi-core') compileOnly( - [group: 'com.squareup.okhttp3', name: 'mockwebserver3-junit5', version: '5.0.0-alpha.2'], - [group: 'com.google.inject', name: 'guice', version: '5.0.1'], + [group: 'com.squareup.okhttp3', name: 'mockwebserver3-junit5', version: '5.0.0-alpha.4'] ) } From 90ce880aaf1e0af1d64b2c2e229ddccd1ce870d4 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 12:19:50 +0100 Subject: [PATCH 12/26] fix(cli): dump command --- .../io/datatok/djobi/cli/commands/DumpPipelineCommand.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java index 9db3c07..bd08682 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java @@ -53,7 +53,9 @@ public void run() { AnsiConsole.systemInstall(); try { - pipeline = pipelineLoader.get(pipelineRequestFactory.build(pipelinePath).setJobsFilter(Arrays.asList(jobs.split(",")))); + pipeline = pipelineLoader.get( + pipelineRequestFactory.build(pipelinePath, args, null, jobs, "", null) + ); } catch (IOException e) { e.printStackTrace(); } From 9e2643ba101f334d0f3a3ac3f66b9d3414eb1532 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 18:04:35 +0100 Subject: [PATCH 13/26] fix(pipeline): pipeline format --- djobi-cli/src/test/resources/pipelines/good.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/djobi-cli/src/test/resources/pipelines/good.yml b/djobi-cli/src/test/resources/pipelines/good.yml index 960b65f..1879e8c 100644 --- a/djobi-cli/src/test/resources/pipelines/good.yml +++ b/djobi-cli/src/test/resources/pipelines/good.yml @@ -32,26 +32,26 @@ jobs: pd: context_b file: not_found stages: - setup_file: + - name: setup_file kind: fs-input spec: path: "../data/json_1" format: json - as_table: + - name: as_table kind: org.spark.mutate spec: alias_table: my_json_1 - input: + - name: input kind: "sql" check: false spec: query: "queries/{{pd}}.sql" - filter: + - name: filter kind: "org.spark.mutate" spec: adds: day: "{{day}}\\/{{month}}\\/{{year}}" - output: + - name: output kind: "fs-output" spec: path: "/tmp/djobi_test_engine" From 19ba5ba7209b5bf0cc6a751f9b9509bcfde6972f Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 18:29:45 +0100 Subject: [PATCH 14/26] fix(test): missing djobi-core dep --- djobi-cli/build.gradle | 1 + djobi-elasticsearch/build.gradle | 1 + .../elasticsearch/output/ElasticsearchEngineTest.java | 2 -- djobi-filter-user_agent/build.gradle | 2 ++ djobi-kafka/build.gradle | 1 + .../djobi/engine/stages/kafka/KafkaOutputRunnerTest.java | 6 ++---- .../djobi/engine/stages/kafka/KafkaPreCheckerStage.java | 2 ++ 7 files changed, 9 insertions(+), 6 deletions(-) diff --git a/djobi-cli/build.gradle b/djobi-cli/build.gradle index 7019141..ca39d2e 100644 --- a/djobi-cli/build.gradle +++ b/djobi-cli/build.gradle @@ -31,6 +31,7 @@ dependencies { * Need core tests code */ testImplementation(project(":djobi-tests")) + testImplementation(project(':djobi-core')) /** * To run Djobi via Idea. diff --git a/djobi-elasticsearch/build.gradle b/djobi-elasticsearch/build.gradle index 6fa806a..150fd97 100644 --- a/djobi-elasticsearch/build.gradle +++ b/djobi-elasticsearch/build.gradle @@ -22,6 +22,7 @@ dependencies { compileOnly project(path: ':djobi-core', configuration: "djobiCore") testImplementation(project(":djobi-tests")) + testImplementation(project(':djobi-core')) es7(group: 'org.elasticsearch', name: 'elasticsearch-spark-30_' + scalaVersion, version:'7.16.2') { exclude group: "org.scala-lang" diff --git a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java index e08e36b..9ccadd0 100644 --- a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java +++ b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java @@ -38,8 +38,6 @@ void testRunJob() throws Exception { final Pipeline pipeline = getPipeline("good.yml"); - engine.setExecutionRequest(pipeline.getPipelineRequest()); - Method method = Engine.class.getDeclaredMethod("run", Job.class); method.setAccessible(true); diff --git a/djobi-filter-user_agent/build.gradle b/djobi-filter-user_agent/build.gradle index 8f25d5c..51fc59e 100644 --- a/djobi-filter-user_agent/build.gradle +++ b/djobi-filter-user_agent/build.gradle @@ -14,7 +14,9 @@ dependencies { compileOnly project(':djobi-core') compileOnly project(path: ':djobi-core', configuration: 'spark') compileOnly project(path: ':djobi-core', configuration: 'djobiCore') + testImplementation(project(':djobi-tests')) + testImplementation(project(':djobi-core')) compileOnly(group: 'eu.bitwalker', name: 'UserAgentUtils', version: '1.21') diff --git a/djobi-kafka/build.gradle b/djobi-kafka/build.gradle index e5a3dcf..1a26bb6 100644 --- a/djobi-kafka/build.gradle +++ b/djobi-kafka/build.gradle @@ -29,6 +29,7 @@ dependencies { compileOnly project(path: ':djobi-core', configuration: "djobiCore") testImplementation(project(":djobi-tests")) + testImplementation(project(':djobi-core')) kafka1(group: 'org.apache.kafka', name: 'kafka-clients', version:'1.1.1') { exclude group: "org.xerial.snappy" diff --git a/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaOutputRunnerTest.java b/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaOutputRunnerTest.java index 3895430..5dad9bf 100644 --- a/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaOutputRunnerTest.java +++ b/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaOutputRunnerTest.java @@ -17,16 +17,14 @@ import org.apache.log4j.Logger; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import javax.inject.Inject; import javax.inject.Provider; @ExtendWith(MyTestRunner.class) +@Tag("IntegrationTest") public class KafkaOutputRunnerTest { @Inject diff --git a/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaPreCheckerStage.java b/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaPreCheckerStage.java index 760908e..d1d500f 100644 --- a/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaPreCheckerStage.java +++ b/djobi-kafka/src/test/java/io/datatok/djobi/engine/stages/kafka/KafkaPreCheckerStage.java @@ -12,6 +12,7 @@ import io.datatok.djobi.utils.templating.TemplateUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -19,6 +20,7 @@ import javax.inject.Provider; @ExtendWith(MyTestRunner.class) +@Tag("IntegrationTest") public class KafkaPreCheckerStage { @Inject From 395d6398f42ecdb2ae1cdd81b4b931a47269cc2f Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 18:41:16 +0100 Subject: [PATCH 15/26] refactor(pipeline): pipeline request --- .../cli/commands/RunPipelineCommand.java | 5 +- .../cli/utils/PipelineRequestFactory.java | 12 +-- .../djobi/cli/RunPipelineCommandTest.java | 5 +- .../cli/utils/PipelineRequestFactoryTest.java | 10 +- .../java/io/datatok/djobi/engine/Engine.java | 28 +---- .../io/datatok/djobi/engine/Pipeline.java | 2 +- .../engine/PipelineExecutionRequest.java | 100 ++++++++---------- .../loaders/yaml/YAMLPipelineLoader.java | 8 +- .../subcribers/JobRunStartSubscriber.java | 2 +- .../datatok/djobi/utils/ActionArgFactory.java | 2 +- .../io/datatok/djobi/engine/EngineTest.java | 4 - 11 files changed, 61 insertions(+), 117 deletions(-) diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java index 8b6ee75..b3a9f35 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java @@ -54,7 +54,7 @@ public class RunPipelineCommand implements Runnable { @CommandLine.Option(names = { "-v", "--verbose" }, description = "Verbose mode. Helpful for troubleshooting. " + "Multiple -v options increase the verbosity.") - private boolean[] verbosityOption = new boolean[0]; + private final boolean[] verbosityOption = new boolean[0]; @Override public void run() { @@ -71,7 +71,6 @@ public void run() { try { pipeline = pipelineLoader.get(pipelineRequest); - pipelineRequest.setPipeline(pipeline); } catch (IOException e) { CLIUtils.printError(e.getMessage()); e.printStackTrace(); @@ -79,7 +78,7 @@ public void run() { if (pipeline != null) { try { - pipelineRunner.run(pipelineRequest); + pipelineRunner.run(pipeline); } catch (Exception e) { CLIUtils.printError(e.getMessage()); e.printStackTrace(); diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java index 6000d7b..3a4dac4 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/PipelineRequestFactory.java @@ -8,12 +8,8 @@ import io.datatok.djobi.utils.MetaUtils; import io.datatok.djobi.utils.MyMapUtils; -import java.nio.channels.Pipe; import java.util.Arrays; -import java.util.HashMap; -import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; @Singleton public class PipelineRequestFactory { @@ -54,11 +50,11 @@ public PipelineExecutionRequest build( ); pipelineRequest - .setPipelineDefinitionPath(inPipelinePath) - .setRaw(argsMap) + .setDefinitionURI(inPipelinePath) + .setArguments(argsMap) .setJobsFilter(Arrays.asList(inJobsFilter.split(","))) .setJobPhases(Arrays.asList(inPhasesFilter.split(","))) - .setMeta(metasMap) + .setMetaDataLabels(metasMap) .setVerbosity(getVerbosity(verbosity)) ; @@ -74,7 +70,7 @@ public PipelineExecutionRequest getLastObjectBuilt() { } private VerbosityLevel getVerbosity(boolean[] verbosity) { - int l = verbosity.length; + int l = verbosity == null ? 0 : verbosity.length; if (l < 1) { return VerbosityLevel.NORMAL; diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java index b794e70..94e8c40 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java @@ -1,7 +1,6 @@ package io.datatok.djobi.cli; import com.google.inject.Inject; -import com.typesafe.config.Config; import io.datatok.djobi.cli.utils.PipelineRequestFactory; import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.plugins.report.OutVerbosity; @@ -52,7 +51,7 @@ void runWithMetaAndArgument() { Assertions.assertTrue(outVerbosity.isNotQuiet()); Assertions.assertFalse(outVerbosity.isVerbose()); - Assertions.assertEquals("yesterday", lastObj.getRaw().get("date")); + Assertions.assertEquals("yesterday", lastObj.getArgument("date")); } @Test @@ -63,7 +62,7 @@ void runPipelinePathFromEnv() throws Exception { PipelineExecutionRequest lastObj = pipelineRequestFactory.getLastObjectBuilt(); - Assertions.assertEquals("./src/test/resources/pipelines/mono.yml", lastObj.getPipelineDefinitionPath()); + Assertions.assertEquals("./src/test/resources/pipelines/mono.yml", lastObj.getDefinitionURI()); }); } diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java index 7542e37..d9c83d4 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java @@ -1,17 +1,13 @@ package io.datatok.djobi.cli.utils; -import com.google.common.collect.Lists; import com.google.inject.Inject; import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.utils.EnvProvider; import io.datatok.djobi.utils.MyMapUtils; -import org.apache.commons.collections.ListUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Map; public class PipelineRequestFactoryTest { @@ -39,10 +35,10 @@ public void testCLIArgsAndEnv() { new boolean[]{} ); - Assertions.assertEquals("yesterday", pipelineExecutionRequest.getRaw().get("date")); - Assertions.assertEquals("from env", pipelineExecutionRequest.getRaw().get("title")); + Assertions.assertEquals("yesterday", pipelineExecutionRequest.getArgument("date")); + Assertions.assertEquals("from env", pipelineExecutionRequest.getArgument("title")); Assertions.assertEquals(Arrays.asList("a", "b"), pipelineExecutionRequest.getJobsFilter()); - Assertions.assertEquals("hello world", pipelineExecutionRequest.getMeta().get("title")); + Assertions.assertEquals("hello world", pipelineExecutionRequest.getMetaDataLabel("title")); } } diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/Engine.java b/djobi-core/src/main/java/io/datatok/djobi/engine/Engine.java index e676c93..a2d84be 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/Engine.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/Engine.java @@ -28,27 +28,11 @@ public class Engine { @Inject private Map phases; - /** - * @since v3.6.0 - */ - private PipelineExecutionRequest executionRequest; - /** * @since v3.6.0 */ private boolean clearJobAfterExecution = true; - /** - * - * @param executionRequest PipelineExecutionRequest - * @throws Exception - */ - public void run(final PipelineExecutionRequest executionRequest) throws Exception { - this.executionRequest = executionRequest; - - run(executionRequest.getPipeline()); - } - /** * Run the pipline. * @@ -59,8 +43,6 @@ public void run(final Pipeline pipeline) throws Exception { } public void run(final Pipeline pipeline, String jobIdFilter) throws Exception { - this.executionRequest = pipeline.getPipelineRequest(); - lookupContext.setCurrentPipeline(pipeline); logger.info(String.format("Run pipeline with %d jobs", pipeline.getJobs().size())); @@ -98,7 +80,7 @@ public void run(final Job job) throws Exception { logger.info(String.format("Executing job [%s]", job.getId())); - final List phasesFilter = executionRequest.getJobPhases(); + final List phasesFilter = pipeline.getPipelineRequest().getJobPhases(); this.runJobPhase(job, ActionPhases.CONFIGURE); @@ -134,14 +116,6 @@ public void run(final Job job) throws Exception { this.eventBus.trigger(new JobRunFinishEvent(job)); } - public PipelineExecutionRequest getExecutionRequest() { - return executionRequest; - } - - public void setExecutionRequest(PipelineExecutionRequest executionRequest) { - this.executionRequest = executionRequest; - } - public boolean isClearJobAfterExecution() { return clearJobAfterExecution; } diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/Pipeline.java b/djobi-core/src/main/java/io/datatok/djobi/engine/Pipeline.java index 5042aee..08bac15 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/Pipeline.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/Pipeline.java @@ -102,7 +102,7 @@ public String resolvePath(final String path) { } public String getName() { - final String path = getPipelineRequest().getPipelineDefinitionPath(); + final String path = getPipelineRequest().getDefinitionURI(); return StringUtils.strip(Arrays.stream(path.split("/")).reduce("", (a, b) -> { if (b.equals("pipelines")) { diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java index 19f83b8..8520110 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java @@ -7,19 +7,17 @@ import java.util.List; import java.util.Map; +/** + * Represent which pipeline and how, djobi must run. + */ public class PipelineExecutionRequest { static public final String PHASES_FILTER_DEFAULT = "configuration,pre_check,run,post_check"; /** - * @since v1.0.0 - */ - private String pipelineDefinitionPath; - - /** - * @since v3.6.0 + * The full URL or path to the pipeline definition. */ - private Pipeline pipeline; + private String definitionURI; /** * @since v1.0.0 @@ -32,30 +30,22 @@ public class PipelineExecutionRequest { private List jobsFilter; /** - * Store raw args. - * - * @since v1.0.0 + * Store arguments. */ - private Map raw; + final private Map arguments; /** * Store run meta info (for logging). - * - * @since v2.2.8 */ - private Map meta; + private Map metaDataLabels; /** * Store verbosity level. - * - * @since v2.2.9 - #34 */ private VerbosityLevel verbosity; /** * Store job phases. - * - * @since v3.6.0 */ private List jobPhases; @@ -64,33 +54,27 @@ public static PipelineExecutionRequest build(final String pipelineDefinitionPath } public static PipelineExecutionRequest build(final String pipelineDefinitionPath, final Map args) { - return new PipelineExecutionRequest(args).setPipelineDefinitionPath(pipelineDefinitionPath); + return new PipelineExecutionRequest(args).setDefinitionURI(pipelineDefinitionPath); } - public PipelineExecutionRequest(final String pipelineDefinitionPath) { - this.pipelineDefinitionPath = pipelineDefinitionPath; + public PipelineExecutionRequest() { + this.arguments = new HashMap<>(); } - public PipelineExecutionRequest(final Map args) { - if (args == null) { - this.raw = new HashMap<>(); - } else { - this.raw = args; - } + public PipelineExecutionRequest(final String definitionURI) { + this(); - this.debug = this.raw.containsKey("debug"); + this.definitionURI = definitionURI; } - public PipelineExecutionRequest() { - } + public PipelineExecutionRequest(final Map arguments) { + this(); - public PipelineExecutionRequest addArgument(final String k, final String v) { - if (this.raw == null) { - this.raw = new HashMap<>(); + if (arguments != null) { + this.setArguments(arguments); } - this.raw.put(k, v); - return this; + this.debug = this.arguments.containsKey("debug"); } public PipelineExecutionRequest setJobsFilter(List jobsFilter) { @@ -102,30 +86,43 @@ public List getJobsFilter() { return jobsFilter; } - public Map getRaw() { - return raw; + public Map getArguments() { + return arguments; } - public PipelineExecutionRequest setRaw(Map raw) { - this.raw = raw; + public PipelineExecutionRequest setArguments(Map p) { + this.arguments.putAll(p); return this; } - public PipelineExecutionRequest setPipelineDefinitionPath(String pipelineDefinitionPath) { - this.pipelineDefinitionPath = pipelineDefinitionPath; + public PipelineExecutionRequest addArgument(final String k, final String v) { + this.arguments.put(k, v); return this; } - public String getPipelineDefinitionPath() { - return pipelineDefinitionPath; + public String getArgument(String key) { + return this.arguments.get(key); } - public Map getMeta() { - return meta; + public PipelineExecutionRequest setDefinitionURI(String definitionURI) { + this.definitionURI = definitionURI; + return this; } - public PipelineExecutionRequest setMeta(Map meta) { - this.meta = meta; + public String getDefinitionURI() { + return definitionURI; + } + + public Map getMetaDataLabels() { + return metaDataLabels; + } + + public String getMetaDataLabel(String key) { + return metaDataLabels.get(key); + } + + public PipelineExecutionRequest setMetaDataLabels(Map metaDataLabels) { + this.metaDataLabels = metaDataLabels; return this; } @@ -151,13 +148,4 @@ public PipelineExecutionRequest setJobPhases(List jobPhases) { this.jobPhases = jobPhases; return this; } - - public Pipeline getPipeline() { - return pipeline; - } - - public PipelineExecutionRequest setPipeline(Pipeline pipeline) { - this.pipeline = pipeline; - return this; - } } diff --git a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java index 363e065..349d7b9 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java +++ b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoader.java @@ -77,7 +77,7 @@ private PipelineDefinition getDefinition(final PipelineExecutionRequest pipeline final Yaml yaml = new Yaml(constructor); final String yamlContent; - File definitionFile = Paths.get(pipelineRequest.getPipelineDefinitionPath()).toFile(); + File definitionFile = Paths.get(pipelineRequest.getDefinitionURI()).toFile(); if (!definitionFile.exists()) { @@ -96,7 +96,7 @@ private PipelineDefinition getDefinition(final PipelineExecutionRequest pipeline if (definitionFile == null) { - throw new IOException(String.format("Dont find a valid definition file under %s !", pipelineRequest.getPipelineDefinitionPath())); + throw new IOException(String.format("Dont find a valid definition file under %s !", pipelineRequest.getDefinitionURI())); } } else if (!extensionCandidates.contains(FilenameUtils.getExtension(definitionBaseName))) @@ -209,10 +209,6 @@ private Pipeline getImplementation(final PipelineExecutionRequest pipelineReques .setJobs(jobs) ; - pipelineRequest - .setPipeline(pipeline) - ; - return pipeline; } diff --git a/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/subcribers/JobRunStartSubscriber.java b/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/subcribers/JobRunStartSubscriber.java index 9f77ad2..e030d7c 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/subcribers/JobRunStartSubscriber.java +++ b/djobi-core/src/main/java/io/datatok/djobi/plugins/logging/subcribers/JobRunStartSubscriber.java @@ -63,7 +63,7 @@ public void call(Event event) { "pipeline", pipelineMap, "date", dateParser.format(new java.util.Date()), "timeline", timelineMap, - "meta", pipeline.getPipelineRequest().getMeta(), + "meta", pipeline.getPipelineRequest().getMetaDataLabels(), "labels", job.getLabels() ); diff --git a/djobi-core/src/main/java/io/datatok/djobi/utils/ActionArgFactory.java b/djobi-core/src/main/java/io/datatok/djobi/utils/ActionArgFactory.java index cb25c8c..d962c0c 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/utils/ActionArgFactory.java +++ b/djobi-core/src/main/java/io/datatok/djobi/utils/ActionArgFactory.java @@ -15,7 +15,7 @@ public class ActionArgFactory { private static final Logger logger = Logger.getLogger(ActionArgFactory.class); public ParameterBag resolve(final Map definitionBag, final PipelineExecutionRequest pipelineRequest) { - final Map jobRequestArgs = pipelineRequest.getRaw(); + final Map jobRequestArgs = pipelineRequest.getArguments(); final ParameterBag bag = new ParameterBag(); if (definitionBag == null) { diff --git a/djobi-core/src/test/java/io/datatok/djobi/engine/EngineTest.java b/djobi-core/src/test/java/io/datatok/djobi/engine/EngineTest.java index ad32046..d4fdcf1 100644 --- a/djobi-core/src/test/java/io/datatok/djobi/engine/EngineTest.java +++ b/djobi-core/src/test/java/io/datatok/djobi/engine/EngineTest.java @@ -88,8 +88,6 @@ class EngineTest { this.eventBus.unsubscribe(); - engine.setExecutionRequest(pipeline.getPipelineRequest()); - final Job job0 = pipeline.getJob(0); Assertions.assertEquals(ExecutionStatus.DONE_OK, job0.getExecutionStatus()); @@ -114,8 +112,6 @@ class EngineTest { Method method = Engine.class.getDeclaredMethod("run", Job.class); method.setAccessible(true); - engine.setExecutionRequest(pipeline.getPipelineRequest()); - final Job job0 = pipeline.getJob(0); method.invoke(engine, job0); From cd8ce80323d66b086dc5a369fe692da759275456 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 18:44:12 +0100 Subject: [PATCH 16/26] docs: misc --- .../java/io/datatok/djobi/engine/PipelineExecutionRequest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java index 8520110..8b04967 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/PipelineExecutionRequest.java @@ -16,6 +16,7 @@ public class PipelineExecutionRequest { /** * The full URL or path to the pipeline definition. + * We just support classic file-system (no HDFS, S3 yet). */ private String definitionURI; From 82971a95213aa1ac1172c0e8c6f9a91eab301223 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 18:47:47 +0100 Subject: [PATCH 17/26] ci(actions): enable tests --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 442337e..174fcb0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,4 +26,4 @@ jobs: uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021 with: arguments: | - -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} -x test djobiAssemble \ No newline at end of file + -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble \ No newline at end of file From 8ef240ec6d006346bacc2553f2404373e6d5e884 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 19:30:58 +0100 Subject: [PATCH 18/26] ci(actions): run test --- .github/workflows/ci.yaml | 7 ++++++- .github/workflows/release.yaml | 2 +- build.gradle | 10 +++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 174fcb0..70954f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,4 +26,9 @@ jobs: uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021 with: arguments: | - -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble \ No newline at end of file + -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} test djobiAssemble + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2 + if: always() # always run even if the previous step fails + with: + report_paths: '**/build/test-results/test/TEST-*.xml' \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 73625d0..0e80cd2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,7 @@ jobs: uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021 with: arguments: | - -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble + -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} test djobiAssemble - name: upload artifacts uses: actions/upload-artifact@v2 with: diff --git a/build.gradle b/build.gradle index ea1608f..05e6119 100644 --- a/build.gradle +++ b/build.gradle @@ -297,11 +297,11 @@ build.dependsOn(testReport); * Relocate some common libs, to prevent conflicts. */ shadowJar { - relocate 'com.typesafe.config', 'my.typesafe.config' - relocate 'com.google.inject', 'my.google.inject' - relocate 'com.google.common', 'my.google.common' - relocate 'okhttp3', 'my.okhttp3' - relocate 'okio','my.okio' + relocate 'com.typesafe.config', 'io.datatok.djobi.typesafe.config' + relocate 'com.google.inject', 'io.datatok.djobi.google.inject' + relocate 'com.google.common', 'io.datatok.djobi.google.common' + relocate 'okhttp3', 'io.datatok.djobi.okhttp3' + relocate 'okio','io.datatok.djobi.okio' minimize() } From 262f166a03dbca3024c26b2c4e9f321f83df92c2 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Sun, 13 Feb 2022 21:39:40 +0100 Subject: [PATCH 19/26] feat(templating): move to jinja - #4 --- djobi-core/build.gradle | 2 +- .../io/datatok/djobi/engine/Parameter.java | 15 ----- .../engine/actions/DjobiActionProvider.java | 4 +- ...w.java => TemplateRenderActionRunner.java} | 2 +- .../djobi/loaders/JobMaterializer.java | 2 +- .../utils/templating/MyGuardedBinding.java | 28 -------- .../templating/MyReflectionObjectHander.java | 15 ----- .../djobi/utils/templating/TemplateUtils.java | 66 ++++++++----------- .../djobi/engine/TemplateUtilsTest.java | 21 +++--- gradle.properties | 1 + 10 files changed, 41 insertions(+), 115 deletions(-) rename djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/{MustacheView.java => TemplateRenderActionRunner.java} (95%) delete mode 100644 djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyGuardedBinding.java delete mode 100644 djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyReflectionObjectHander.java diff --git a/djobi-core/build.gradle b/djobi-core/build.gradle index c125203..0636647 100644 --- a/djobi-core/build.gradle +++ b/djobi-core/build.gradle @@ -36,7 +36,7 @@ dependencies { api( [group: 'org.apache.commons', name: 'commons-csv', version:'1.6'], - [group: 'com.github.spullara.mustache.java', name: 'compiler', version:'0.9.6'], + [group: 'com.hubspot.jinjava', name: 'jinjava', version: jinjavaVersion], [group: 'org.yaml', name: 'snakeyaml', version:'1.29'], [group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'], [group: "co.elastic.apm", name: "apm-agent-api", version: '1.28.4'], diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/Parameter.java b/djobi-core/src/main/java/io/datatok/djobi/engine/Parameter.java index ec757f8..a26b814 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/Parameter.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/Parameter.java @@ -67,21 +67,6 @@ public String getValueAsString() { return this.getValue().toString(); } - public List> getValuesWithIndex() { - List v = (List) this.value; - - return - IntStream - .range(0, v.size()) - .mapToObj(i -> MyMapUtils.map( - "value", v.get(i), - "loop_index", i + 1, - "loop_first", i == 0, - "loop_last", i == (v.size() - 1) - ) - ).collect(Collectors.toList()); - } - public String getValueDescription() { final Object v = this.getValue(); diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/actions/DjobiActionProvider.java b/djobi-core/src/main/java/io/datatok/djobi/engine/actions/DjobiActionProvider.java index fdfc505..6f0331c 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/actions/DjobiActionProvider.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/actions/DjobiActionProvider.java @@ -27,7 +27,7 @@ import io.datatok.djobi.engine.actions.transformer.serializer.SerializerConfigurator; import io.datatok.djobi.engine.actions.transformer.serializer.SerializerRunner; import io.datatok.djobi.engine.actions.transformer.serializer.SerializerType; -import io.datatok.djobi.engine.actions.transformer.serializers.MustacheView; +import io.datatok.djobi.engine.actions.transformer.serializers.TemplateRenderActionRunner; import io.datatok.djobi.engine.actions.csv.CSVFilterConfigurator; import io.datatok.djobi.engine.actions.utils.generator.RandomGeneratorConfigurator; import io.datatok.djobi.engine.actions.utils.generator.RandomGeneratorRunner; @@ -124,7 +124,7 @@ protected void configureNet() { protected void configureTransformers() { registerConfigurator(CSVType.NAME, CSVFilterConfigurator.class); - registerRunner(MustacheView.TYPE, MustacheView.class); + registerRunner(TemplateRenderActionRunner.TYPE, TemplateRenderActionRunner.class); registerConfigurator(SerializerType.TYPE, SerializerConfigurator.class); registerRunner(SerializerType.TYPE, SerializerRunner.class); diff --git a/djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/MustacheView.java b/djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/TemplateRenderActionRunner.java similarity index 95% rename from djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/MustacheView.java rename to djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/TemplateRenderActionRunner.java index 1e6ae19..dcaddc1 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/MustacheView.java +++ b/djobi-core/src/main/java/io/datatok/djobi/engine/actions/transformer/serializers/TemplateRenderActionRunner.java @@ -12,7 +12,7 @@ import javax.inject.Inject; -public class MustacheView implements ActionRunner { +public class TemplateRenderActionRunner implements ActionRunner { final static public String TYPE = "transform.mustache"; diff --git a/djobi-core/src/main/java/io/datatok/djobi/loaders/JobMaterializer.java b/djobi-core/src/main/java/io/datatok/djobi/loaders/JobMaterializer.java index 05ddddc..c655ae9 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/loaders/JobMaterializer.java +++ b/djobi-core/src/main/java/io/datatok/djobi/loaders/JobMaterializer.java @@ -128,7 +128,7 @@ private ParameterBag resolveParametersForJob(final ParameterBag parameterBag) { if (p.getType().equals(Parameter.TYPE_STRING) && p.getValue() != null && p.getValueAsString().contains("{")) { final Parameter pc = p.clone(); - pc.setValue(templateUtils.render((String) p.getValue(), Arrays.asList(MyMapUtils.map("args", parameterBag)))); + pc.setValue(templateUtils.renderTemplate((String) p.getValue(), MyMapUtils.map("args", parameterBag))); ret.put(entry.getKey(), pc); } else { diff --git a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyGuardedBinding.java b/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyGuardedBinding.java deleted file mode 100644 index 0f08212..0000000 --- a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyGuardedBinding.java +++ /dev/null @@ -1,28 +0,0 @@ -package io.datatok.djobi.utils.templating; - -import com.github.mustachejava.Code; -import com.github.mustachejava.ObjectHandler; -import com.github.mustachejava.TemplateContext; -import com.github.mustachejava.reflect.GuardedBinding; -import com.github.mustachejava.reflect.MissingWrapper; -import com.github.mustachejava.util.Wrapper; - -import java.util.List; - -public class MyGuardedBinding extends GuardedBinding { - - MyGuardedBinding(ObjectHandler oh, String name, TemplateContext tc, Code code) { - super(oh, name, tc, code); - } - - protected synchronized Wrapper getWrapper(String name, List scopes) { - final Wrapper foundWrapper = super.getWrapper(name, scopes); - - if (foundWrapper instanceof MissingWrapper) { - throw new IllegalStateException(String.format("'%s' is missing!", name)); - } - - return foundWrapper; - } - -} diff --git a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyReflectionObjectHander.java b/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyReflectionObjectHander.java deleted file mode 100644 index 950e463..0000000 --- a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/MyReflectionObjectHander.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.datatok.djobi.utils.templating; - -import com.github.mustachejava.Binding; -import com.github.mustachejava.Code; -import com.github.mustachejava.TemplateContext; -import com.github.mustachejava.reflect.ReflectionObjectHandler; - -public class MyReflectionObjectHander extends ReflectionObjectHandler { - - @Override - public Binding createBinding(String name, TemplateContext tc, Code code) { - return new MyGuardedBinding(this, name, tc, code); - } - -} diff --git a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/TemplateUtils.java b/djobi-core/src/main/java/io/datatok/djobi/utils/templating/TemplateUtils.java index 9f19d4b..af86604 100644 --- a/djobi-core/src/main/java/io/datatok/djobi/utils/templating/TemplateUtils.java +++ b/djobi-core/src/main/java/io/datatok/djobi/utils/templating/TemplateUtils.java @@ -1,20 +1,15 @@ package io.datatok.djobi.utils.templating; -import com.github.mustachejava.DefaultMustacheFactory; -import com.github.mustachejava.Mustache; +import com.hubspot.jinjava.Jinjava; import io.datatok.djobi.configuration.Configuration; import io.datatok.djobi.engine.Job; import io.datatok.djobi.utils.MyMapUtils; import javax.inject.Inject; import javax.inject.Singleton; -import java.io.IOException; -import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; @Singleton public class TemplateUtils { @@ -42,55 +37,46 @@ public Object renderOption(String snippet, final Job job) { } public String render(String snippet) { - return renderTemplate(false, snippet); - } - - public String render(String snippet, final List templateDataBuffer) { - return renderTemplate(false, snippet, templateDataBuffer); + return renderTemplate(snippet); } public String render(String snippet, final Job job) { - return renderTemplate(false, snippet, Arrays.asList(MyMapUtils.map("pipeline", job.getPipeline(), "job", job, "data", job.getData(), "args", job.getParameters(), "context", job.getParameters()), job.getParameters())); - } + Map templateData = new HashMap<>(); + + if (job.getParameters() != null) { + templateData.putAll(job.getParameters()); + } + + templateData.putAll(MyMapUtils.map( + "pipeline", job.getPipeline(), + "job", job, + "data", job.getData(), + "parameters", job.getParameters(), + "context", job.getParameters() + )); - public String renderTemplate(boolean isFile, String template) { - return renderTemplate(isFile, template, new ArrayList<>()); + return renderTemplate(snippet, templateData); } - public String renderTemplate(boolean isFile, String template, final List templateDataBuffer) { - final DefaultMustacheFactory mf = new DefaultMustacheFactory(); - final Writer writerMessage = new StringWriter(); - final Mustache mustache; + public String renderTemplate(String template) { + return renderTemplate(template, new HashMap<>()); + } - mf.setObjectHandler(new MyReflectionObjectHander()); + public String renderTemplate(String template, Map inTemplateData) { + final Jinjava jinjava = new Jinjava(); if (template == null) { return null; } - if (isFile) { - mustache = mf.compile(template); - } else { - mustache = mf.compile(new StringReader(template), "template"); - } - - final ArrayList templateData = new ArrayList<>(templateDataBuffer); + //final Map templateData = new HashMap<>(); if (configuration != null) { - /** "config." is deprecated, use "env." **/ - templateData.add(MyMapUtils.map("config", configuration.root().unwrapped())); - templateData.add(MyMapUtils.map("env", MyMapUtils.wrapByMissingKeyException(configuration.root().unwrapped()))); - } - - mustache.execute(writerMessage, templateData); - - try { - writerMessage.flush(); - } catch (IOException e) { - e.printStackTrace(); + jinjava.getGlobalContext().put("env", MyMapUtils.wrapByMissingKeyException(configuration.root().unwrapped())); + jinjava.getGlobalContext().put("config", configuration.root().unwrapped()); } - return writerMessage.toString(); + return jinjava.render(template, inTemplateData); } /** diff --git a/djobi-core/src/test/java/io/datatok/djobi/engine/TemplateUtilsTest.java b/djobi-core/src/test/java/io/datatok/djobi/engine/TemplateUtilsTest.java index 4003115..a2f81be 100644 --- a/djobi-core/src/test/java/io/datatok/djobi/engine/TemplateUtilsTest.java +++ b/djobi-core/src/test/java/io/datatok/djobi/engine/TemplateUtilsTest.java @@ -1,6 +1,5 @@ package io.datatok.djobi.engine; -import com.github.mustachejava.MustacheException; import io.datatok.djobi.engine.parameters.DateParameter; import io.datatok.djobi.loaders.yaml.YAMLPipelineLoader; import io.datatok.djobi.test.MyTestRunner; @@ -26,16 +25,9 @@ class TemplateUtilsTest { private YAMLPipelineLoader yamlPipelineLoader; @Test void stringTest() { - Assertions.assertEquals("Hello, env is test", templateUtils.renderTemplate(false, "Hello, env is {{env._meta_.config}}")); + Assertions.assertEquals("Hello, env is test", templateUtils.renderTemplate("Hello, env is {{env._meta_.config}}")); } - @Test void testMissingData() { - Assertions.assertThrows (MustacheException.class, () -> templateUtils.renderTemplate(false, "Hello, env is {{toto}}")); - Assertions.assertThrows (MustacheException.class, () -> templateUtils.renderTemplate(false, "Hello, env is {{env.toto}}")); - Assertions.assertThrows (MustacheException.class, () -> templateUtils.renderTemplate(false, "Hello, env is {{env._meta_.toto}}")); - } - - @Test void testWithJobData() { final Job job = new Job(); @@ -57,11 +49,16 @@ class TemplateUtilsTest { @Test void testWithList() { final Job job = new Job(); - job.setParameters(new ParameterBag("hello", Arrays.asList("hello", "world"), "date", new DateParameter("date", Calendar.getInstance()))); + job.setParameters( + new ParameterBag( + "hello", Arrays.asList("hello", "world"), + "date", new DateParameter("date", Calendar.getInstance()) + ) + ); - Assertions.assertEquals("hello world ", templateUtils.render("{{#hello.value}}{{.}} {{/hello.value}}", job)); + Assertions.assertEquals("hello world ", templateUtils.render("{% for h in parameters.hello.value %}{{h}} {% endfor %}", job)); - Assertions.assertEquals("1 2 ", templateUtils.render("{{#hello.valuesWithIndex}}{{loop_index}} {{/hello.valuesWithIndex}}", job)); + Assertions.assertEquals("1 2 ", templateUtils.render("{% for h in parameters.hello.value %}{{loop.index}} {% endfor %}", job)); } @Test void testWithPipelineData() throws IOException { diff --git a/gradle.properties b/gradle.properties index fa6941d..cb754e9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,6 +5,7 @@ mainClass='io.datatok.djobi.Main' ## Dependecies Version sparkVersion=3.1.2 scalaVersion=2.12 +jinjavaVersion=2.6.0 #org.gradle.java.home=/Library/Java/JavaVirtualMachines/jdk1.8.0_101.jdk/Contents/Home/ From 72bcc9d8bddc85bdc6c6fc2da7f4657d6dbac129 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 09:45:34 +0100 Subject: [PATCH 20/26] build: clean deps --- build.gradle | 57 ++++++++---------- djobi-cli/build.gradle | 19 +++--- djobi-core/build.gradle | 59 +++++++++---------- .../datatok/djobi/engine/stages/S3Test.java | 15 ++++- djobi-elasticsearch/build.gradle | 3 +- djobi-filter-user_agent/build.gradle | 1 - djobi-kafka/build.gradle | 3 +- djobi-tests/build.gradle | 1 + 8 files changed, 79 insertions(+), 79 deletions(-) diff --git a/build.gradle b/build.gradle index 05e6119..77956a5 100644 --- a/build.gradle +++ b/build.gradle @@ -7,6 +7,28 @@ plugins { id 'nebula.release' version '16.0.0' } +/* PROJECT DEPENDENCY VERSIONS */ +// define all common versioned dependencies here +project.ext.dependencyStrings = [ + GOOGLE_GUICE: 'com.google.inject:guice:5.0.1', + TYPESAFE_CONFIG: 'com.typesafe:config:1.4.1', + ORG_YAML: 'org.yaml:snakeyaml:1.29', + OKHTTP: 'com.squareup.okhttp3:okhttp:4.9.3', + ELASTIC_APM: 'co.elastic.apm:apm-agent-api:1.29.0', + JANSI: 'org.fusesource.jansi:jansi:2.4.0', + PICOCLI: 'info.picocli:picocli:4.6.2' +] + +/** + * Spark provided dependencies + * Can be used with compileOnly + */ +project.ext.sparkDependencyStrings = [ + JACKSON_CORE: 'com.fasterxml.jackson.core:jackson-core:2.10.0', + JCRAFT: 'com.jcraft:jsch:0.1.55', + APACHE_COMMONS_TEXT: 'org.apache.commons:commons-text:1.4' +] + allprojects { afterEvaluate { project -> println "I'm configuring $project.name with version $project.version" @@ -49,38 +71,6 @@ subprojects { } ext.djobi = [ - /** - * Build a JAR to use if Djobi dont run with Spark Assembly (YARN..) - */ - sparkAssemblyProvide: { - String taskName = "sparkAssemblyProvide" - - shadowJar.enabled = false - - task(taskName, type: ShadowJar) { - doFirst { - println "Packaging ${project.name}->sparkAssemblyProvide " - } - - from sourceSets.main.output - configurations = [project.configurations.sparkAssemblyProvided] - - archiveFileName = "${project.name}-${project.version}-spark-assembly-provided-all.jar" - - relocate 'com.typesafe.config', 'my.typesafe.config' - relocate 'com.google.inject', 'my.google.inject' - relocate 'com.google.common', 'my.google.common' - relocate 'okhttp3', 'my.okhttp3' - relocate 'okio', 'my.okio' - - /*minimize { - exclude(dependency('org.elasticsearch:.*:.*')) - }*/ - } - - shadowJar.dependsOn(taskName) - }, - /** * Create a task: "makeRelease" * - called by djobiAssemble @@ -171,8 +161,7 @@ subprojects { testImplementation( [group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.2'], [group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.7.2'], - [group: 'com.squareup.okhttp3', name: 'mockwebserver3-junit5', version: '5.0.0-alpha.2'], - [group: 'com.google.inject', name: 'guice', version: '5.0.1'] + [group: 'com.squareup.okhttp3', name: 'mockwebserver3-junit5', version: '5.0.0-alpha.2'] ) } diff --git a/djobi-cli/build.gradle b/djobi-cli/build.gradle index ca39d2e..4105325 100644 --- a/djobi-cli/build.gradle +++ b/djobi-cli/build.gradle @@ -22,10 +22,12 @@ dependencies { */ compileOnly project(':djobi-core') - compileOnly project(path: ':djobi-core', configuration: "spark") - compileOnly project(path: ':djobi-core', configuration: "sparkAssemblyProvided") - compileOnly project(path: ':djobi-core', configuration: "sparkAWS") - compileOnly project(path: ':djobi-core', configuration: "djobiCore") + /** + * Provided by Spark - but we dont want Spark dep here for cli + */ + compileOnly sparkDependencyStrings.JACKSON_CORE + compileOnly sparkDependencyStrings.JCRAFT + compileOnly sparkDependencyStrings.APACHE_COMMONS_TEXT /** * Need core tests code @@ -42,9 +44,12 @@ dependencies { [group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.3.1'], ) - implementation( - [group: 'info.picocli', name: 'picocli', version: '4.6.2'] - ) + /** + * Optional dep as it may not work on ARM64 + */ + compileOnly dependencyStrings.JANSI + + implementation dependencyStrings.PICOCLI } run { diff --git a/djobi-core/build.gradle b/djobi-core/build.gradle index 0636647..cc0f165 100644 --- a/djobi-core/build.gradle +++ b/djobi-core/build.gradle @@ -7,50 +7,32 @@ plugins { configurations { - // Hold deps from spark-hadoop-assembly.jar - sparkAssemblyProvided - // For AWS sparkAWS // Spark Core, SQL etc... spark - // Common libs - djobiCore - - djobiRelease - - djobiRelease.extendsFrom djobiCore + compileOnly.extendsFrom spark, sparkAWS + testImplementation.extendsFrom spark, sparkAWS - implementation.extendsFrom djobiCore - - api.extendsFrom sparkAssemblyProvided - api.extendsFrom spark - api.extendsFrom sparkAWS + djobiRelease.extendsFrom implementation } dependencies { // deps shared by all djobi projects - - api( + implementation( [group: 'org.apache.commons', name: 'commons-csv', version:'1.6'], [group: 'com.hubspot.jinjava', name: 'jinjava', version: jinjavaVersion], - [group: 'org.yaml', name: 'snakeyaml', version:'1.29'], - [group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'], - [group: "co.elastic.apm", name: "apm-agent-api", version: '1.28.4'], - [group: 'com.google.inject', name: 'guice', version: '5.1.0'], - [group: 'com.typesafe', name: 'config', version:'1.4.1'] + [group: 'javax.mail', name: 'javax.mail-api', version:'1.6.2'] ) - // Provided by Apache Spark Assembly 1.6.3 - // Useful if you don't run via Hadoop - sparkAssemblyProvided( - [group: 'javax.mail', name: 'javax.mail-api', version:'1.6.2'], - [group: 'com.jcraft', name: 'jsch', version:'0.1.55'], - [group: 'org.fusesource.jansi', name: 'jansi', version: '2.3.4'] - ) + api dependencyStrings.GOOGLE_GUICE + api dependencyStrings.TYPESAFE_CONFIG + api dependencyStrings.ORG_YAML + api dependencyStrings.OKHTTP + api dependencyStrings.ELASTIC_APM sparkAWS( [group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '1.12.53'] @@ -93,8 +75,26 @@ dependencies { testImplementation(project(":djobi-tests")) } +jar { + manifest { + attributes 'Implementation-Version': archiveVersion + attributes 'Automatic-Module-Name': 'io.datatok.djobi' + + // OSGi metadata + attributes 'Bundle-SymbolicName': 'io.datatok.djobi' + attributes 'Bundle-Name': 'Application to run ETL over Apache Spark' + attributes 'Bundle-Vendor': 'Datatok' + attributes 'Bundle-DocURL': 'https://github.com/datatok/djobi' + attributes 'Bundle-License': 'https://www.apache.org/licenses/LICENSE-2.0' + attributes 'Export-Package': 'io.datatok.djobi.*' + } +} + + javadoc.options.addStringOption('Xdoclint:none', '-quiet') +/* RELEASE */ + signing { sign configurations.archives } @@ -150,5 +150,4 @@ publishing { } } -djobi.createRelease() -djobi.sparkAssemblyProvide() \ No newline at end of file +djobi.createRelease() \ No newline at end of file diff --git a/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java b/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java index cfd0925..5a148b0 100644 --- a/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java +++ b/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java @@ -3,9 +3,13 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.model.ObjectMetadata; import io.datatok.djobi.engine.actions.fs.output.FSOutputType; @@ -49,9 +53,14 @@ public class S3Test extends ActionTest { ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.HTTP); - this.conn = new AmazonS3Client(credentials, clientConfig); - this.conn.setEndpoint(S3_ENDPOINT); - this.conn.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); + this.conn = AmazonS3ClientBuilder + .standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withRegion(Regions.US_EAST_2) + .withClientConfiguration(clientConfig) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, Regions.US_EAST_2.getName())) + .withPathStyleAccessEnabled(true) + .build(); } diff --git a/djobi-elasticsearch/build.gradle b/djobi-elasticsearch/build.gradle index 150fd97..9584a08 100644 --- a/djobi-elasticsearch/build.gradle +++ b/djobi-elasticsearch/build.gradle @@ -18,8 +18,7 @@ configurations { dependencies { compileOnly project(':djobi-core') - compileOnly project(path: ':djobi-core', configuration: "spark") - compileOnly project(path: ':djobi-core', configuration: "djobiCore") + compileOnly project(path: ':djobi-core', configuration: 'spark') testImplementation(project(":djobi-tests")) testImplementation(project(':djobi-core')) diff --git a/djobi-filter-user_agent/build.gradle b/djobi-filter-user_agent/build.gradle index 51fc59e..0f70319 100644 --- a/djobi-filter-user_agent/build.gradle +++ b/djobi-filter-user_agent/build.gradle @@ -13,7 +13,6 @@ configurations { dependencies { compileOnly project(':djobi-core') compileOnly project(path: ':djobi-core', configuration: 'spark') - compileOnly project(path: ':djobi-core', configuration: 'djobiCore') testImplementation(project(':djobi-tests')) testImplementation(project(':djobi-core')) diff --git a/djobi-kafka/build.gradle b/djobi-kafka/build.gradle index 1a26bb6..a51a06c 100644 --- a/djobi-kafka/build.gradle +++ b/djobi-kafka/build.gradle @@ -25,8 +25,7 @@ configurations { dependencies { compileOnly project(':djobi-core') - compileOnly project(path: ':djobi-core', configuration: "spark") - compileOnly project(path: ':djobi-core', configuration: "djobiCore") + compileOnly project(path: ':djobi-core', configuration: 'spark') testImplementation(project(":djobi-tests")) testImplementation(project(':djobi-core')) diff --git a/djobi-tests/build.gradle b/djobi-tests/build.gradle index f909089..568ab04 100644 --- a/djobi-tests/build.gradle +++ b/djobi-tests/build.gradle @@ -23,6 +23,7 @@ dependencies { implementation 'org.json:json:20211205' compileOnly project(':djobi-core') + compileOnly project(path: ':djobi-core', configuration: 'spark') compileOnly( [group: 'com.squareup.okhttp3', name: 'mockwebserver3-junit5', version: '5.0.0-alpha.4'] From 4f7624ae0d0f7bd815d240105a13fd5b776ead33 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 09:45:56 +0100 Subject: [PATCH 21/26] feat: work on docker image --- packages/docker/Dockerfile | 97 ++++++++++++++--------------- packages/docker/entrypoint.sh | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 51 deletions(-) create mode 100644 packages/docker/entrypoint.sh diff --git a/packages/docker/Dockerfile b/packages/docker/Dockerfile index aeccd03..d9eddaf 100644 --- a/packages/docker/Dockerfile +++ b/packages/docker/Dockerfile @@ -1,68 +1,52 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -ARG java_image_tag=11-jre-slim - -FROM openjdk:${java_image_tag} - -# Before building the docker image, first build and make a Spark distribution following -# the instructions in http://spark.apache.org/docs/latest/building-spark.html. -# If this docker file is being used in the context of building your images from a Spark -# distribution, the docker build command should be invoked from the top level directory -# of the Spark distribution. E.g.: -# docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile . +ARG java_image_tag=11-jre + +## +# Stage base (tooling) +## +FROM openjdk:${java_image_tag} AS base RUN set -ex && \ sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \ apt-get update && \ ln -s /lib /lib64 && \ apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps curl && \ - mkdir -p /opt/spark && \ - mkdir -p /opt/spark/work-dir && \ - touch /opt/spark/RELEASE && \ - rm /bin/sh && \ - ln -sv /bin/bash /bin/sh && \ echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \ chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \ rm -rf /var/cache/apt/* -ARG spark_version=3.1.2 -ARG hadoop_version=3.2 +## +# Stage Apache Spark +## +ARG SPARK_VERSION=3.1.2 +ARG HADOOP_VERSION=3.2 -RUN curl -sL https://mirrors.estointernet.in/apache/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop${hadoop_version}.tgz --output spark-bin-hadoop.tgz && \ - tar -xzf spark-bin-hadoop.tgz +FROM base AS spark-base -RUN cd spark-${spark_version}-bin-hadoop${hadoop_version} && \ - cp -r jars /opt/spark/jars && \ - cp -r bin /opt/spark/bin && \ - cp -r sbin /opt/spark/sbin && \ - cp -r conf /opt/spark/conf && \ - cp -r kubernetes/dockerfiles/spark/entrypoint.sh /opt/ && \ - cp -r kubernetes/dockerfiles/spark/decom.sh /opt/ && \ - cp -r kubernetes/tests /opt/spark/tests && \ - cp -r data /opt/spark/data +# Download and extract Spark +RUN curl -L https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -o spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ + && tar -xvzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \ + && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} /opt/spark \ + && rm spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -ENV SPARK_HOME /opt/spark -ENV DJOBI_HOME /opt/djobi +## +# Stage Elastic APM +## +ARG ELASTIC_APM_VERSION=1.29.0 -ENV PATH $PATH:${SPARK_HOME}/bin:${DJOBI_HOME} +FROM base AS elastic-apm -ADD build/release ${DJOBI_HOME} +RUN mkdir -p /opt/elastic && \ + curl -L https://search.maven.org/remotecontent?filepath=co/elastic/apm/elastic-apm-agent/${ELASTIC_APM_VERSION}/elastic-apm-agent-${ELASTIC_APM_VERSION}.jar -o /opt/elastic/elastic-apm-agent-${ELASTIC_APM_VERSION}.jar -WORKDIR ${DJOBI_HOME} +## +# Stage Djobi +## +FROM base + +ENV SPARK_HOME /opt/spark +ENV DJOBI_HOME /opt/djobi +ENV ELASTIC_HOME /opt/elastic +ENV PATH $PATH:${SPARK_HOME}/bin:${DJOBI_HOME} ## # add djobi user to run as no-root @@ -70,8 +54,19 @@ WORKDIR ${DJOBI_HOME} RUN groupadd --gid 1000 djobi && \ useradd --no-create-home --uid 1000 --home-dir ${DJOBI_HOME} --gid 1000 djobi -RUN chmod g+w /opt/spark/work-dir -RUN chmod a+x /opt/decom.sh +## +# get artifacts +## +COPY --from=elastic-apm /opt/elastic ${ELASTIC_HOME} +COPY --from=spark-base /opt/spark ${SPARK_HOME} +COPY build/release ${DJOBI_HOME} +COPY packages/docker ${DJOBI_HOME}/docker + +WORKDIR ${DJOBI_HOME} + RUN chmod +x /opt/djobi/djobi +RUN chmod a+x ${DJOBI_HOME}/docker/entrypoint.sh + +ENTRYPOINT [ "/opt/djobi/docker/entrypoint.sh" ] USER djobi \ No newline at end of file diff --git a/packages/docker/entrypoint.sh b/packages/docker/entrypoint.sh new file mode 100644 index 0000000..cada8db --- /dev/null +++ b/packages/docker/entrypoint.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Check whether there is a passwd entry for the container UID +myuid=$(id -u) +mygid=$(id -g) +# turn off -e for getent because it will return error code in anonymous uid case +set +e +uidentry=$(getent passwd $myuid) +set -e + +# If there is no passwd entry for the container UID, attempt to create one +if [ -z "$uidentry" ] ; then + if [ -w /etc/passwd ] ; then + echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd + else + echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" + fi +fi + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}') +fi + +SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" +env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt +readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt + +if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then + SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH" +fi + +if ! [ -z ${PYSPARK_PYTHON+x} ]; then + export PYSPARK_PYTHON +fi +if ! [ -z ${PYSPARK_DRIVER_PYTHON+x} ]; then + export PYSPARK_DRIVER_PYTHON +fi + +# If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor. +# It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s. +if [ -n "${HADOOP_HOME}" ] && [ -z "${SPARK_DIST_CLASSPATH}" ]; then + export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)" +fi + +if ! [ -z ${HADOOP_CONF_DIR+x} ]; then + SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH"; +fi + +if ! [ -z ${SPARK_CONF_DIR+x} ]; then + SPARK_CLASSPATH="$SPARK_CONF_DIR:$SPARK_CLASSPATH"; +elif ! [ -z ${SPARK_HOME+x} ]; then + SPARK_CLASSPATH="$SPARK_HOME/conf:$SPARK_CLASSPATH"; +fi + +case "$1" in + driver) + shift 1 + CMD=( + "$SPARK_HOME/bin/spark-submit" + --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" + --deploy-mode client + "$@" + ) + ;; + executor) + shift 1 + CMD=( + ${JAVA_HOME}/bin/java + "${SPARK_EXECUTOR_JAVA_OPTS[@]}" + -Xms$SPARK_EXECUTOR_MEMORY + -Xmx$SPARK_EXECUTOR_MEMORY + -cp "$SPARK_CLASSPATH:$SPARK_DIST_CLASSPATH" + org.apache.spark.scheduler.cluster.k8s.KubernetesExecutorBackend + --driver-url $SPARK_DRIVER_URL + --executor-id $SPARK_EXECUTOR_ID + --cores $SPARK_EXECUTOR_CORES + --app-id $SPARK_APPLICATION_ID + --hostname $SPARK_EXECUTOR_POD_IP + --resourceProfileId $SPARK_RESOURCE_PROFILE_ID + --podName $SPARK_EXECUTOR_POD_NAME + ) + ;; + + *) + echo "Non-spark-on-k8s command provided, proceeding in pass-through mode..." + CMD=("$@") + ;; +esac + +# Execute the container CMD under tini for better hygiene +exec /usr/bin/tini -s -- "${CMD[@]}" \ No newline at end of file From 5e2bc276b2b95e06bdc736c070787b0a5d6ddda2 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 09:46:08 +0100 Subject: [PATCH 22/26] feat: work on docker image --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 3317ba9..546fbf0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * !dev !build +!packages !VERSION !README.md \ No newline at end of file From 5d5f08d98266ea3949f3342d450f566bb2cfa9d5 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 10:12:51 +0100 Subject: [PATCH 23/26] fix: conf var optional --- dev/default.conf | 2 +- djobi-core/src/main/resources/reference.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/default.conf b/dev/default.conf index 2f4da48..2c3c8f8 100644 --- a/dev/default.conf +++ b/dev/default.conf @@ -91,7 +91,7 @@ djobi { utils_country { type: "table" format: "parquet" - path: ${projectRoot}"/dev/data/utils_country" + path: ${?projectRoot}"/dev/data/utils_country" columns: { test: "toto" } diff --git a/djobi-core/src/main/resources/reference.conf b/djobi-core/src/main/resources/reference.conf index ce5c43c..9c81f07 100644 --- a/djobi-core/src/main/resources/reference.conf +++ b/djobi-core/src/main/resources/reference.conf @@ -82,7 +82,7 @@ djobi { utils_country { type: "table" format: "parquet" - path: ${projectRoot}"/dev/data/utils_country" + path: ${?projectRoot}"/dev/data/utils_country" columns: { test: "toto" } From f6589179fd6132f276d8a605fb4a2d94d5de9232 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 10:13:06 +0100 Subject: [PATCH 24/26] fix: remove dep on jansi --- .../src/main/java/io/datatok/djobi/Main.java | 8 +- .../io/datatok/djobi/cli/CommandFactory.java | 12 ++- .../io/datatok/djobi/cli/StdoutReporter.java | 10 ++- .../cli/commands/DumpPipelineCommand.java | 6 +- .../cli/commands/RunPipelineCommand.java | 9 ++- .../datatok/djobi/cli/utils/CLIDumpUtils.java | 18 ----- .../datatok/djobi/cli/utils/CLIOutUtils.java | 2 +- .../djobi/cli/utils/CLISimpleUtils.java | 51 +++++++++++++ .../io/datatok/djobi/cli/utils/CLIUtils.java | 66 ++-------------- .../djobi/cli/utils/CLIWithJansiUtils.java | 75 +++++++++++++++++++ .../datatok/djobi/cli/utils/CLIUtilsTest.java | 20 ----- 11 files changed, 159 insertions(+), 118 deletions(-) delete mode 100644 djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIDumpUtils.java create mode 100644 djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLISimpleUtils.java create mode 100644 djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIWithJansiUtils.java delete mode 100644 djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java diff --git a/djobi-cli/src/main/java/io/datatok/djobi/Main.java b/djobi-cli/src/main/java/io/datatok/djobi/Main.java index c066362..109fe31 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/Main.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/Main.java @@ -6,6 +6,8 @@ import io.datatok.djobi.application.exceptions.BuildApplicationException; import io.datatok.djobi.cli.CommandKernel; import io.datatok.djobi.cli.StdoutReporter; +import io.datatok.djobi.cli.utils.CLISimpleUtils; +import io.datatok.djobi.cli.utils.CLIUtils; import io.datatok.djobi.plugins.report.Reporter; import io.datatok.djobi.plugins.s3.S3Plugin; import io.datatok.djobi.plugins.stages.DefaultActionsPlugin; @@ -43,7 +45,10 @@ public static void main(String[] args) { builder.addPlugin(new S3Plugin()); } - builder.addDependency(Reporter.class, StdoutReporter.class); + builder + .addDependency(Reporter.class, StdoutReporter.class) + .addDependency(CLIUtils.class, CLISimpleUtils.class) + ; application = builder.loadPlugins().build(); } catch(BuildApplicationException e) { @@ -55,6 +60,7 @@ public static void main(String[] args) { } Injector injector = application.getInjector(); + CommandKernel commandKernel = injector.getInstance(CommandKernel.class); CommandLine rootCommandImpl = commandKernel.getRootCommand(); diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java index 9f346b5..bd0f3f0 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/CommandFactory.java @@ -1,12 +1,11 @@ package io.datatok.djobi.cli; +import com.google.inject.Inject; import com.google.inject.Injector; -import org.fusesource.jansi.AnsiConsole; +import com.google.inject.Singleton; +import io.datatok.djobi.cli.utils.CLIUtils; import picocli.CommandLine; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -16,9 +15,8 @@ public class CommandFactory implements CommandLine.IFactory { @Inject private Injector injector; - public CommandFactory() { - AnsiConsole.systemInstall(); - } + @Inject + private CLIUtils cliUtils; @Override public K create(Class cls) throws Exception { diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/StdoutReporter.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/StdoutReporter.java index 7c2caf8..3dd7b7d 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/StdoutReporter.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/StdoutReporter.java @@ -1,5 +1,7 @@ package io.datatok.djobi.cli; +import com.google.inject.Inject; +import com.google.inject.Singleton; import io.datatok.djobi.cli.utils.CLIOutUtils; import io.datatok.djobi.cli.utils.CLIUtils; import io.datatok.djobi.engine.Job; @@ -8,9 +10,8 @@ import io.datatok.djobi.engine.stage.Stage; import io.datatok.djobi.plugins.report.Reporter; import io.datatok.djobi.utils.ClassUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; -import javax.inject.Singleton; import java.io.PrintStream; import java.util.Map; import java.util.stream.Collectors; @@ -20,11 +21,14 @@ @Singleton public class StdoutReporter implements Reporter { + @Inject + private CLIUtils cliUtils; + private PrintStream stdoutPrintStream = System.out; @Override public void output(String format, Object... args) { - CLIUtils.output(format, args); + cliUtils.output(format, args); } @Override diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java index bd08682..d1bd053 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/DumpPipelineCommand.java @@ -10,13 +10,11 @@ import io.datatok.djobi.loaders.yaml.YAMLPipelineLoader; import io.datatok.djobi.plugins.report.Reporter; import io.datatok.djobi.utils.JSONUtils; -import org.apache.commons.lang.StringUtils; -import org.fusesource.jansi.AnsiConsole; +import org.apache.commons.lang3.StringUtils; import picocli.CommandLine; import javax.inject.Inject; import java.io.IOException; -import java.util.Arrays; import java.util.Map; @CommandLine.Command(name = "pipeline", description = "dump a pipeline") @@ -50,8 +48,6 @@ public class DumpPipelineCommand implements Runnable { public void run() { Pipeline pipeline = null; - AnsiConsole.systemInstall(); - try { pipeline = pipelineLoader.get( pipelineRequestFactory.build(pipelinePath, args, null, jobs, "", null) diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java index b3a9f35..bbcecfc 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/commands/RunPipelineCommand.java @@ -32,6 +32,9 @@ public class RunPipelineCommand implements Runnable { @Inject OutVerbosity outVerbosity; + @Inject + CLIUtils cliUtils; + @CommandLine.Option(paramLabel = "args", names = {"-a", "--arg"}, description = "arguments (date, ...)") Map args; @@ -61,7 +64,7 @@ public void run() { Pipeline pipeline = null; if (pipelinePath == null || pipelinePath.isEmpty()) { - CLIUtils.printError("pipeline is missing!"); + cliUtils.printError("pipeline is missing!"); return ; } @@ -72,7 +75,7 @@ public void run() { try { pipeline = pipelineLoader.get(pipelineRequest); } catch (IOException e) { - CLIUtils.printError(e.getMessage()); + cliUtils.printError(e.getMessage()); e.printStackTrace(); } @@ -80,7 +83,7 @@ public void run() { try { pipelineRunner.run(pipeline); } catch (Exception e) { - CLIUtils.printError(e.getMessage()); + cliUtils.printError(e.getMessage()); e.printStackTrace(); } } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIDumpUtils.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIDumpUtils.java deleted file mode 100644 index e7021f9..0000000 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIDumpUtils.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.datatok.djobi.cli.utils; - -import io.datatok.djobi.engine.Job; -import io.datatok.djobi.engine.Pipeline; -import io.datatok.djobi.engine.stage.Stage; -import io.datatok.djobi.utils.ClassUtils; -import org.apache.commons.lang.StringUtils; - -import java.util.Map; -import java.util.stream.Collectors; - -import static org.fusesource.jansi.Ansi.ansi; - -public class CLIDumpUtils { - - - -} diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIOutUtils.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIOutUtils.java index 2121c8b..ab7e269 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIOutUtils.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIOutUtils.java @@ -2,7 +2,7 @@ import io.datatok.djobi.engine.Parameter; import io.datatok.djobi.utils.bags.ParameterBag; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import java.util.*; diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLISimpleUtils.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLISimpleUtils.java new file mode 100644 index 0000000..604f84f --- /dev/null +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLISimpleUtils.java @@ -0,0 +1,51 @@ +package io.datatok.djobi.cli.utils; + +import com.google.inject.Singleton; +import org.apache.commons.lang3.StringUtils; + +import java.util.Formatter; +import java.util.List; + +@Singleton +public class CLISimpleUtils implements CLIUtils { + + public void init() { + + } + + public void output(final String format, Object... args) { + output(new Formatter().format(format, args).toString()); + } + + public void output(final String out) { + System.out.println(out); + } + + public void printError(final String out) { + System.out.printf("\n\nFatal error: %s\n%n", out); + } + + public void outputHorizontalList(final List texts) { + StringBuilder buffer = new StringBuilder(); + + for (final String text : texts) { + buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); + } + + buffer.append("\n"); + + for (final String text : texts) { + buffer.append(String.format("| %s | --> ", text)); + } + + buffer + .delete(buffer.length() - 4, buffer.length()) + .append("\n"); + + for (final String text : texts) { + buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); + } + + output(buffer.toString()); + } +} diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIUtils.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIUtils.java index 6c1a151..be0ae2e 100644 --- a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIUtils.java +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIUtils.java @@ -1,70 +1,16 @@ package io.datatok.djobi.cli.utils; -import org.apache.commons.lang.StringUtils; - -import java.util.Formatter; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -import static org.fusesource.jansi.Ansi.ansi; - -public class CLIUtils { - - public static void output(final String format, Object... args) { - output(new Formatter().format(format, args).toString()); - } - - public static void output(final String out) { - System.out.println( ansi().render(out)); - } - - public static void printError(final String out) { - System.out.println( ansi().render(String.format("\n\n@|bold,white,bg_red Fatal error: %s|@\n", out))); - } - - public static void outputHorizontalList(final List texts) { - StringBuilder buffer = new StringBuilder(); - - for (final String text : texts) { - buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); - } - - buffer.append("\n"); - - for (final String text : texts) { - buffer.append(String.format("| %s | --> ", text)); - } - - buffer - .delete(buffer.length() - 4, buffer.length()) - .append("\n"); - - for (final String text : texts) { - buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); - } - - CLIUtils.output(buffer.toString()); - } - /** - * @deprecated v2.2.0 - */ - public static Map parseParameters(final String[] args) { +public interface CLIUtils { - final Map parameters = new HashMap<>(); + void init(); - for (final String arg : args) { - if (!arg.startsWith("--")) { - System.err.println("Bad parameter: " + arg); - System.exit(2); - } + void output(final String format, Object... args); - final String[] tab = arg.substring(2).split("="); - parameters.put(tab[0], tab.length == 2 ? tab[1] : ""); - } + void output(final String out); - return parameters; - } + void printError(final String out); + void outputHorizontalList(final List texts); } diff --git a/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIWithJansiUtils.java b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIWithJansiUtils.java new file mode 100644 index 0000000..b9ae4dd --- /dev/null +++ b/djobi-cli/src/main/java/io/datatok/djobi/cli/utils/CLIWithJansiUtils.java @@ -0,0 +1,75 @@ +package io.datatok.djobi.cli.utils; + +import org.apache.commons.lang3.StringUtils; +import org.fusesource.jansi.AnsiConsole; + +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.fusesource.jansi.Ansi.ansi; + +public class CLIWithJansiUtils { + + public static void init() { + AnsiConsole.systemInstall(); + } + + public static void output(final String format, Object... args) { + output(new Formatter().format(format, args).toString()); + } + + public static void output(final String out) { + System.out.println( ansi().render(out)); + } + + public static void printError(final String out) { + System.out.println( ansi().render(String.format("\n\n@|bold,white,bg_red Fatal error: %s|@\n", out))); + } + + public static void outputHorizontalList(final List texts) { + StringBuilder buffer = new StringBuilder(); + + for (final String text : texts) { + buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); + } + + buffer.append("\n"); + + for (final String text : texts) { + buffer.append(String.format("| %s | --> ", text)); + } + + buffer + .delete(buffer.length() - 4, buffer.length()) + .append("\n"); + + for (final String text : texts) { + buffer.append(String.format("+%s+ ", StringUtils.leftPad("", text.length() + 2, "-"))); + } + + CLIWithJansiUtils.output(buffer.toString()); + } + + /** + * @deprecated v2.2.0 + */ + public static Map parseParameters(final String[] args) { + + final Map parameters = new HashMap<>(); + + for (final String arg : args) { + if (!arg.startsWith("--")) { + System.err.println("Bad parameter: " + arg); + System.exit(2); + } + + final String[] tab = arg.substring(2).split("="); + parameters.put(tab[0], tab.length == 2 ? tab[1] : ""); + } + + return parameters; + } + +} diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java deleted file mode 100644 index 5d976d7..0000000 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/CLIUtilsTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.datatok.djobi.cli.utils; - -import io.datatok.djobi.cli.utils.CLIUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class CLIUtilsTest { - - @Test - void parseEmptyArgumentsTest() - { - CLIUtils.parseParameters(new String[]{}); - } - - @Test - void parseWrongArgumentsTest() - { - Assertions.assertEquals("toto", CLIUtils.parseParameters(new String[]{"--hello=toto", "--what=the"}).get("hello")); - } -} From b408ee432fe80118bbed033611c2071907a5b57a Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 12:08:03 +0100 Subject: [PATCH 25/26] fix(cli): tests runner --- djobi-cli/build.gradle | 6 ++- .../io/datatok/djobi/cli/CLITestRunner.java | 49 +++++++++++++++++++ .../djobi/cli/DumpActionCommandTest.java | 2 + .../datatok/djobi/cli/DumpEnvCommandTest.java | 2 + .../djobi/cli/DumpPipelineCommandTest.java | 2 + .../djobi/cli/RunPipelineCommandTest.java | 2 + .../cli/utils/PipelineRequestFactoryTest.java | 3 ++ .../org.junit.jupiter.api.extension.Extension | 1 - 8 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 djobi-cli/src/test/java/io/datatok/djobi/cli/CLITestRunner.java delete mode 100644 djobi-cli/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension diff --git a/djobi-cli/build.gradle b/djobi-cli/build.gradle index 4105325..862e2f1 100644 --- a/djobi-cli/build.gradle +++ b/djobi-cli/build.gradle @@ -33,7 +33,11 @@ dependencies { * Need core tests code */ testImplementation(project(":djobi-tests")) - testImplementation(project(':djobi-core')) + + testRuntimeOnly( + project(path: ':djobi-core'), + project(path: ':djobi-core', configuration: 'spark') + ) /** * To run Djobi via Idea. diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/CLITestRunner.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/CLITestRunner.java new file mode 100644 index 0000000..ff9a1ee --- /dev/null +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/CLITestRunner.java @@ -0,0 +1,49 @@ +package io.datatok.djobi.cli; + +import com.google.inject.Injector; +import io.datatok.djobi.application.ApplicationBuilder; +import io.datatok.djobi.application.Djobi; +import io.datatok.djobi.cli.utils.CLISimpleUtils; +import io.datatok.djobi.cli.utils.CLIUtils; +import io.datatok.djobi.engine.Engine; +import io.datatok.djobi.plugins.report.Reporter; +import io.datatok.djobi.plugins.s3.S3Plugin; +import io.datatok.djobi.plugins.stages.DefaultActionsPlugin; +import io.datatok.djobi.spark.SparkPlugin; +import io.datatok.djobi.test.TestStdoutReporter; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestInstancePostProcessor; + +public class CLITestRunner implements TestInstancePostProcessor { + static public Injector injector; + + public CLITestRunner() { + if (injector == null) { + try { + final Djobi application = + new ApplicationBuilder() + .configure() + .addDependency(Reporter.class, TestStdoutReporter.class) + .addDependency(CLIUtils.class, CLISimpleUtils.class) + .readReleaseNote() + .addPlugin(new DefaultActionsPlugin()) + .addPlugin(new SparkPlugin()) + .addPlugin(new S3Plugin()) + .build() + ; + + injector = application.getInjector(); + + injector.getInstance(Engine.class).setClearJobAfterExecution(false); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @Override + public void postProcessTestInstance(Object testInstance, ExtensionContext context) + throws Exception { + injector.injectMembers(testInstance); + } +} diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java index 6152954..357ae44 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java @@ -5,11 +5,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.HashMap; +@ExtendWith(CLITestRunner.class) class DumpActionCommandTest { @Inject diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java index d6b015e..07cff6b 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java @@ -5,12 +5,14 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import picocli.CommandLine; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.HashMap; +@ExtendWith(CLITestRunner.class) class DumpEnvCommandTest { @Inject diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java index b4742a4..23ba485 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java @@ -7,12 +7,14 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import picocli.CommandLine; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.HashMap; +@ExtendWith(CLITestRunner.class) class DumpPipelineCommandTest { @Inject diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java index 94e8c40..5cef999 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java @@ -8,12 +8,14 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import static com.github.stefanbirkner.systemlambda.SystemLambda.*; +@ExtendWith(CLITestRunner.class) public class RunPipelineCommandTest { @Inject diff --git a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java index d9c83d4..4cac39c 100644 --- a/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java +++ b/djobi-cli/src/test/java/io/datatok/djobi/cli/utils/PipelineRequestFactoryTest.java @@ -1,14 +1,17 @@ package io.datatok.djobi.cli.utils; import com.google.inject.Inject; +import io.datatok.djobi.cli.CLITestRunner; import io.datatok.djobi.engine.PipelineExecutionRequest; import io.datatok.djobi.utils.EnvProvider; import io.datatok.djobi.utils.MyMapUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.util.Arrays; +@ExtendWith(CLITestRunner.class) public class PipelineRequestFactoryTest { @Inject diff --git a/djobi-cli/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/djobi-cli/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension deleted file mode 100644 index dc6b412..0000000 --- a/djobi-cli/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension +++ /dev/null @@ -1 +0,0 @@ -io.datatok.djobi.test.MyTestRunner \ No newline at end of file From 255b593149e3f16d847e353f2ec64efcf6fea990 Mon Sep 17 00:00:00 2001 From: Thomas Decaux Date: Tue, 22 Feb 2022 12:19:32 +0100 Subject: [PATCH 26/26] fix(test): spark dep --- djobi-elasticsearch/build.gradle | 12 +++++++++--- djobi-filter-user_agent/build.gradle | 12 +++++++++--- djobi-kafka/build.gradle | 12 +++++++++--- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/djobi-elasticsearch/build.gradle b/djobi-elasticsearch/build.gradle index 9584a08..5f29516 100644 --- a/djobi-elasticsearch/build.gradle +++ b/djobi-elasticsearch/build.gradle @@ -20,15 +20,21 @@ dependencies { compileOnly project(':djobi-core') compileOnly project(path: ':djobi-core', configuration: 'spark') - testImplementation(project(":djobi-tests")) - testImplementation(project(':djobi-core')) - es7(group: 'org.elasticsearch', name: 'elasticsearch-spark-30_' + scalaVersion, version:'7.16.2') { exclude group: "org.scala-lang" exclude group: "org.apache.spark" exclude group: "com.google.protobuf" exclude group: "javax.xml.bind" } + + testImplementation( + project(":djobi-tests"), + project(':djobi-core') + ) + + testRuntimeOnly( + project(path: ':djobi-core', configuration: 'spark') + ) } djobi.createReleaseWithVariants(variants) \ No newline at end of file diff --git a/djobi-filter-user_agent/build.gradle b/djobi-filter-user_agent/build.gradle index 0f70319..ab4914a 100644 --- a/djobi-filter-user_agent/build.gradle +++ b/djobi-filter-user_agent/build.gradle @@ -14,12 +14,18 @@ dependencies { compileOnly project(':djobi-core') compileOnly project(path: ':djobi-core', configuration: 'spark') - testImplementation(project(':djobi-tests')) - testImplementation(project(':djobi-core')) - compileOnly(group: 'eu.bitwalker', name: 'UserAgentUtils', version: '1.21') bitwalker(group: 'eu.bitwalker', name: 'UserAgentUtils', version: '1.21') + + testImplementation( + project(":djobi-tests"), + project(':djobi-core') + ) + + testRuntimeOnly( + project(path: ':djobi-core', configuration: 'spark') + ) } djobi.createReleaseWithVariants(variants) \ No newline at end of file diff --git a/djobi-kafka/build.gradle b/djobi-kafka/build.gradle index a51a06c..0b67f53 100644 --- a/djobi-kafka/build.gradle +++ b/djobi-kafka/build.gradle @@ -27,9 +27,6 @@ dependencies { compileOnly project(':djobi-core') compileOnly project(path: ':djobi-core', configuration: 'spark') - testImplementation(project(":djobi-tests")) - testImplementation(project(':djobi-core')) - kafka1(group: 'org.apache.kafka', name: 'kafka-clients', version:'1.1.1') { exclude group: "org.xerial.snappy" exclude group: "org.slf4j" @@ -38,6 +35,15 @@ dependencies { kafka2(group: 'org.apache.kafka', name: 'kafka-clients', version:'2.2.0') { exclude group: "org.xerial.snappy" } + + testImplementation( + project(":djobi-tests"), + project(':djobi-core') + ) + + testRuntimeOnly( + project(path: ':djobi-core', configuration: 'spark') + ) } djobi.createReleaseWithVariants(variants) \ No newline at end of file