Merge branch 'main' into fix/outofmemory

.github/workflows/ci-pr.yml

@@ -30,7 +30,7 @@ jobs:
     # build
     - name: Build
       run: |
-        mvn -B -s .github/settings.xml $MAVEN_THREADS clean package
+        mvn -B -s .github/settings.xml $MAVEN_THREADS clean install
     # clean m2 cache
     - name: Clean cache
       run: |

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
     - name: Build and Publish
       run: |
         mvn clean        
-        jfrog rt mvn install -Pfull -B $MAVEN_THREADS
+        jfrog rt mvn install -Pfull,asciidoctordocs,restdocs -B $MAVEN_THREADS
         jfrog rt build-publish
     - name: Capture Test Results
       if: ${{ always() }}

build-docs.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
+SCDIR=$(realpath $SCDIR)
+pushd "$SCDIR" > /dev/null || exit
+    ./mvnw install -o -am -pl :spring-cloud-dataflow-classic-docs,:spring-cloud-dataflow-docs,:spring-cloud-skipper-server-core,:spring-cloud-skipper-docs -DskipTests
+    ./mvnw install -o -Pfull,asciidoctordocs,restdocs -pl :spring-cloud-dataflow-classic-docs,:spring-cloud-dataflow-docs
+    ./mvnw install -o -Pasciidoctordocs,restdocs -pl :spring-cloud-skipper-server-core,:spring-cloud-skipper-docs
+popd > /dev/null || exit

pom.xml

@@ -55,7 +55,10 @@
 		<module>spring-cloud-dataflow-container-registry</module>
 		<module>spring-cloud-dataflow-configuration-metadata</module>
 		<module>spring-cloud-dataflow-core-dsl</module>
+		<module>spring-cloud-dataflow-schema-core</module>
 		<module>spring-cloud-dataflow-core</module>
+		<module>spring-cloud-dataflow-schema</module>
+		<module>spring-cloud-dataflow-aggregate-task</module>
 		<module>spring-cloud-dataflow-server-core</module>
 		<module>spring-cloud-dataflow-rest-resource</module>
 		<module>spring-cloud-dataflow-audit</module>

spring-cloud-dataflow-aggregate-task/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.springframework.cloud</groupId>
+		<artifactId>spring-cloud-dataflow-parent</artifactId>
+		<version>2.11.0-SNAPSHOT</version>
+		<relativePath>../spring-cloud-dataflow-parent</relativePath>
+	</parent>
+	<artifactId>spring-cloud-dataflow-aggregate-task</artifactId>
+	<packaging>jar</packaging>
+	<properties>
+		<failIfNoTests>true</failIfNoTests>
+	</properties>
+	<dependencies>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-context</artifactId>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-task-batch</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-dataflow-core</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-dataflow-registry</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-dataflow-schema</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-annotations</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-test</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-engine</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-params</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.h2database</groupId>
+			<artifactId>h2</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<version>3.0.0</version>
+				<configuration>
+					<threadCount>1</threadCount>
+					<forkCount>1</forkCount>
+					<failIfNoTests>false</failIfNoTests>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+</project>

spring-cloud-dataflow-aggregate-task/src/main/java/org/springframework/cloud/dataflow/aggregate/task/AggregateExecutionSupport.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.cloud.dataflow.aggregate.task;
+
+import org.springframework.cloud.dataflow.core.AppRegistration;
+import org.springframework.cloud.dataflow.core.TaskDefinition;
+import org.springframework.cloud.dataflow.schema.AggregateTaskExecution;
+import org.springframework.cloud.dataflow.schema.SchemaVersionTarget;
+import org.springframework.cloud.task.repository.TaskExecution;
+
+/**
+ * Allows users to retrieve Task execution and SchemaVersion information from either {@link TaskExecution} as well as
+ * Task Name.
+ * @author Corneil du Plessis
+ */
+public interface AggregateExecutionSupport {
+
+	/**
+	 * Retrieves the {@link AggregateTaskExecution} for the task execution and {@link TaskDefinitionReader} provided.
+	 * @param execution A {@link TaskExecution} that contains the TaskName that will be used to find the {@link AggregateTaskExecution}.
+	 * @param taskDefinitionReader {@link TaskDefinitionReader} that will be used to find the {@link SchemaVersionTarget} for the task execution.
+	 * @return The {@link AggregateTaskExecution} containing the {@link SchemaVersionTarget} for the TaskExecution.
+	 */
+	AggregateTaskExecution from(TaskExecution execution, TaskDefinitionReader taskDefinitionReader, TaskDeploymentReader taskDeploymentReader);
+
+	/**
+	 * Retrieves the {@link SchemaVersionTarget} for the task name.
+	 * @param taskName The name of the {@link org.springframework.cloud.dataflow.core.TaskDefinition} from which the {@link SchemaVersionTarget} will be retreived.
+	 * @param taskDefinitionReader {@link TaskDefinitionReader} that will be used to find the {@link SchemaVersionTarget}
+	 * @return The {@link SchemaVersionTarget} for the taskName specified.
+	 */
+	SchemaVersionTarget findSchemaVersionTarget(String taskName, TaskDefinitionReader taskDefinitionReader);
+	SchemaVersionTarget findSchemaVersionTarget(String taskName, TaskDefinition taskDefinition);
+
+	/**
+	 * Retrieve the {@link AppRegistration} for the registeredName.
+	 */
+	AppRegistration findTaskAppRegistration(String registeredName);
+
+	/**
+	 * Return the {@link AggregateTaskExecution} for the {@link TaskExecution} and Schema Target name specified.
+	 */
+	AggregateTaskExecution from(TaskExecution execution, String schemaTarget, String platformName);
+}

spring-cloud-dataflow-aggregate-task/src/main/java/org/springframework/cloud/dataflow/aggregate/task/AggregateTaskConfiguration.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2017-2021 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.cloud.dataflow.aggregate.task;
+
+import javax.annotation.PostConstruct;
+import javax.sql.DataSource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.springframework.cloud.dataflow.aggregate.task.impl.AggregateDataFlowTaskExecutionQueryDao;
+import org.springframework.cloud.dataflow.aggregate.task.impl.DefaultAggregateExecutionSupport;
+import org.springframework.cloud.dataflow.aggregate.task.impl.DefaultAggregateTaskExplorer;
+import org.springframework.cloud.dataflow.aggregate.task.impl.DefaultTaskRepositoryContainer;
+import org.springframework.cloud.dataflow.registry.service.AppRegistryService;
+import org.springframework.cloud.dataflow.schema.service.SchemaService;
+import org.springframework.cloud.dataflow.schema.service.SchemaServiceConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.util.Assert;
+
+/**
+ * Configuration for aggregate task related components.
+ *
+ * @author Corneil du Plessis
+ */
+@Configuration
+@Import(SchemaServiceConfiguration.class)
+public class AggregateTaskConfiguration {
+	private static final Logger logger = LoggerFactory.getLogger(AggregateTaskConfiguration.class);
+
+
+	@Bean
+	public DataflowTaskExecutionQueryDao dataflowTaskExecutionQueryDao(
+			DataSource dataSource,
+			SchemaService schemaService
+	) {
+		return new AggregateDataFlowTaskExecutionQueryDao(dataSource, schemaService);
+	}
+
+	@Bean
+	public AggregateExecutionSupport aggregateExecutionSupport(
+			AppRegistryService registryService,
+			SchemaService schemaService
+	) {
+		return new DefaultAggregateExecutionSupport(registryService, schemaService);
+	}
+
+	@Bean
+	public TaskRepositoryContainer taskRepositoryContainer(
+			DataSource dataSource,
+			SchemaService schemaService
+	) {
+		return new DefaultTaskRepositoryContainer(dataSource, schemaService);
+	}
+
+	@Bean
+	public AggregateTaskExplorer aggregateTaskExplorer(
+			DataSource dataSource,
+			DataflowTaskExecutionQueryDao taskExecutionQueryDao,
+			SchemaService schemaService,
+			AggregateExecutionSupport aggregateExecutionSupport,
+			TaskDefinitionReader taskDefinitionReader,
+			TaskDeploymentReader taskDeploymentReader
+	) {
+		Assert.notNull(dataSource, "dataSource required");
+		Assert.notNull(taskExecutionQueryDao, "taskExecutionQueryDao required");
+		Assert.notNull(schemaService, "schemaService required");
+		Assert.notNull(aggregateExecutionSupport, "aggregateExecutionSupport required");
+		Assert.notNull(taskDefinitionReader, "taskDefinitionReader required");
+		Assert.notNull(taskDeploymentReader, "taskDeploymentReader required");
+		return new DefaultAggregateTaskExplorer(dataSource,
+				taskExecutionQueryDao,
+				schemaService,
+				aggregateExecutionSupport,
+				taskDefinitionReader,
+				taskDeploymentReader);
+	}
+
+	@PostConstruct
+	public void setup() {
+		logger.info("created: org.springframework.cloud.dataflow.aggregate.task.AggregateTaskConfiguration");
+	}
+}