Merge branch 'datahub-project:master' into master

.github/actions/ci-optimization/action.yml

@@ -48,7 +48,7 @@ runs:
             - "smoke-test/tests/cypress/**"
             - "docker/datahub-frontend/**"
           ingestion:
-            - "metadata-ingestion-modules/airflow-plugin/**"
+            - "metadata-ingestion-modules/**"
             - "metadata-ingestion/**"
             - "metadata-models/**"
             - "smoke-test/**"

.github/workflows/build-and-test.yml

@@ -69,6 +69,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
+        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:
           python-version: "3.10"
           cache: pip
@@ -82,6 +83,10 @@ jobs:
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
           NODE_OPTIONS: "--max-old-space-size=3072"
+      - name: Gradle compile (jdk8) for legacy Spark
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        run: |
+          ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/metadata-io.yml

@@ -24,9 +24,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }}
+      ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }}
+      backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }}
+      docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }}
+      frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }}
+      ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }}
+      kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }}
+      mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }}
+      postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }}
+      elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }}
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: ./.github/actions/ci-optimization
+        id: ci-optimize
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    needs: setup
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 17
@@ -36,6 +55,7 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - uses: actions/setup-python@v4
+        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:
           python-version: "3.10"
           cache: "pip"

build.gradle

@@ -1,6 +1,32 @@
 buildscript {
-  ext.jdkVersion = 17
-  ext.javaClassVersion = 11
+  ext.jdkVersionDefault = 17
+  ext.javaClassVersionDefault = 11
+
+  ext.jdkVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+      || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
+      return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault
+    }
+  }
+
+  ext.javaClassVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+              || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
+      return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault
+    }
+  }
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
@@ -217,6 +243,7 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
     'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
@@ -252,48 +279,56 @@ allprojects {
     }
   }
 
-  if (project.plugins.hasPlugin('java')
+  /**
+   * If making changes to this section also see the sections for pegasus below
+   * which use project.plugins.hasPlugin('pegasus')
+   **/
+  if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
           || project.plugins.hasPlugin('java-library')
-          || project.plugins.hasPlugin('application')
-          || project.plugins.hasPlugin('pegasus')) {
+          || project.plugins.hasPlugin('application'))) {
 
     java {
       toolchain {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
 
     compileJava {
-      options.release = javaClassVersion
+      options.release = javaClassVersion(project)
     }
+
     tasks.withType(JavaCompile).configureEach {
       javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
       // Puts parameter names into compiled class files, necessary for Spring 6
       options.compilerArgs.add("-parameters")
     }
 
     tasks.withType(JavaExec).configureEach {
       javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
+  }
+
+  // not duplicated, need to set this outside and inside afterEvaluate
+  afterEvaluate {
+    /**
+     * If making changes to this section also see the sections for pegasus below
+     * which use project.plugins.hasPlugin('pegasus')
+     **/
+    if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
+            || project.plugins.hasPlugin('java-library')
+            || project.plugins.hasPlugin('application'))) {
 
-    // not duplicated, need to set this outside and inside afterEvaluate
-    afterEvaluate {
       compileJava {
-        options.release = javaClassVersion
-      }
-      tasks.withType(JavaCompile).configureEach {
-        javaCompiler = javaToolchains.compilerFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
-        }
+        options.release = javaClassVersion(project)
       }
 
       tasks.withType(JavaExec).configureEach {
         javaLauncher = javaToolchains.launcherFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
         }
       }
     }
@@ -368,6 +403,30 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      java {
+        toolchain {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaCompile).configureEach {
+        javaCompiler = javaToolchains.compilerFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+        // Puts parameter names into compiled class files, necessary for Spring 6
+        options.compilerArgs.add("-parameters")
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
 
     afterEvaluate {
@@ -394,6 +453,16 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
   }
 }

buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java

@@ -183,6 +183,7 @@ private void generateSchema(final File file) {
     final String fileBaseName;
     try {
       final JsonNode schema = JsonLoader.fromFile(file);
+
       final JsonNode result = buildResult(schema.toString());
       String prettySchema = JacksonUtils.prettyPrint(result);
       Path absolutePath = file.getAbsoluteFile().toPath();
@@ -195,11 +196,21 @@ private void generateSchema(final File file) {
       } else {
         fileBaseName = getBaseName(file.getName());
       }
-      Files.write(Paths.get(jsonDirectory + sep + fileBaseName + ".json"),
+
+      final String targetName;
+      if (schema.has("Aspect") && schema.get("Aspect").has("name") &&
+              !schema.get("Aspect").get("name").asText().equalsIgnoreCase(fileBaseName)) {
+        targetName = OpenApiEntities.toUpperFirst(schema.get("Aspect").get("name").asText());
+        prettySchema = prettySchema.replaceAll(fileBaseName, targetName);
+      } else {
+        targetName = fileBaseName;
+      }
+
+      Files.write(Paths.get(jsonDirectory + sep + targetName + ".json"),
           prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
           StandardOpenOption.TRUNCATE_EXISTING);
       if (schema.has("Aspect")) {
-        aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + getBaseName(file.getName())));
+        aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + targetName));
       }
     } catch (IOException | ProcessingException e) {
       throw new RuntimeException(e);

buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java

@@ -6,6 +6,7 @@
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.linkedin.metadata.models.registry.config.Entities;
 import com.linkedin.metadata.models.registry.config.Entity;
@@ -58,8 +59,12 @@ public class OpenApiEntities {
                 .add("notebookInfo").add("editableNotebookProperties")
                 .add("dataProductProperties")
                 .add("institutionalMemory")
+                .add("forms").add("formInfo").add("dynamicFormAssignment")
                 .build();
 
+    private final static ImmutableSet<String> ENTITY_EXCLUSIONS = ImmutableSet.<String>builder()
+            .add("structuredProperty")
+            .build();
 
     public OpenApiEntities(JsonNodeFactory NODE_FACTORY) {
         this.NODE_FACTORY = NODE_FACTORY;
@@ -117,14 +122,27 @@ public ObjectNode entityExtension(List<ObjectNode> nodesList, ObjectNode schemas
         return componentsNode;
     }
 
-    private static String toUpperFirst(String s) {
-        return s.substring(0, 1).toUpperCase() + s.substring(1);
+    /**
+     * Convert the pdl model names to desired class names. Upper case first letter unless the 3rd character is upper case.
+     * i.e. mlModel -> MLModel
+     *      dataset -> Dataset
+     *      dataProduct -> DataProduct
+     * @param s input string
+     * @return class name
+     */
+    public static String toUpperFirst(String s) {
+        if (s.length() > 2 && s.substring(2, 3).equals(s.substring(2, 3).toUpperCase())) {
+            return s.substring(0, 2).toUpperCase() + s.substring(2);
+        } else {
+            return s.substring(0, 1).toUpperCase() + s.substring(1);
+        }
     }
 
     private Set<String> withEntitySchema(ObjectNode schemasNode, Set<String> definitions) {
         return entityMap.values().stream()
                 // Make sure the primary key is defined
                 .filter(entity -> definitions.contains(toUpperFirst(entity.getKeyAspect())))
+                .filter(entity -> !ENTITY_EXCLUSIONS.contains(entity.getName()))
                 .map(entity -> {
                     final String upperName = toUpperFirst(entity.getName());
 
@@ -547,7 +565,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
 
         ObjectNode getMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Get %s for %s.", aspect, entity.getName()))
-                .put("operationId", String.format("get%s", upperFirstAspect, upperFirstEntity));
+                .put("operationId", String.format("get%s", upperFirstAspect));
         getMethod.set("tags", tagsNode);
         ArrayNode singlePathParametersNode = NODE_FACTORY.arrayNode();
         getMethod.set("parameters", singlePathParametersNode);
@@ -575,13 +593,13 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
                         .set("application/json", NODE_FACTORY.objectNode())));
         ObjectNode headMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("%s on %s existence.", aspect, upperFirstEntity))
-                .put("operationId", String.format("head%s", upperFirstAspect, upperFirstEntity))
+                .put("operationId", String.format("head%s", upperFirstAspect))
                 .set("responses", headResponses);
         headMethod.set("tags", tagsNode);
 
         ObjectNode deleteMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Delete %s on entity %s", aspect, upperFirstEntity))
-                .put("operationId", String.format("delete%s", upperFirstAspect, upperFirstEntity))
+                .put("operationId", String.format("delete%s", upperFirstAspect))
                 .set("responses", NODE_FACTORY.objectNode()
                         .set("200", NODE_FACTORY.objectNode()
                                 .put("description", String.format("Delete %s on %s entity.", aspect, upperFirstEntity))
@@ -591,7 +609,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) {
 
         ObjectNode postMethod = NODE_FACTORY.objectNode()
                 .put("summary", String.format("Create aspect %s on %s ", aspect, upperFirstEntity))
-                .put("operationId", String.format("create%s", upperFirstAspect, upperFirstEntity));
+                .put("operationId", String.format("create%s", upperFirstAspect));
         postMethod.set("requestBody", NODE_FACTORY.objectNode()
                 .put("description", String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity))
                 .put("required", true).set("content", NODE_FACTORY.objectNode()

datahub-frontend/app/config/ConfigurationProvider.java

@@ -1,5 +1,6 @@
 package config;
 
+import com.linkedin.metadata.config.VisualConfiguration;
 import com.linkedin.metadata.config.cache.CacheConfiguration;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.spring.YamlPropertySourceFactory;
@@ -22,4 +23,7 @@ public class ConfigurationProvider {
 
   /** Configuration for caching */
   private CacheConfiguration cache;
+
+  /** Configuration for the view layer */
+  private VisualConfiguration visualConfig;
 }

datahub-frontend/app/controllers/Application.java

@@ -13,6 +13,7 @@
 import com.linkedin.util.Pair;
 import com.typesafe.config.Config;
 import java.io.InputStream;
+import java.net.URI;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
@@ -125,6 +126,12 @@ public CompletableFuture<Result> proxy(String path, Http.Request request)
       headers.put(Http.HeaderNames.X_FORWARDED_HOST, headers.get(Http.HeaderNames.HOST));
     }
 
+    if (!headers.containsKey(Http.HeaderNames.X_FORWARDED_PROTO)) {
+      final String schema =
+          Optional.ofNullable(URI.create(request.uri()).getScheme()).orElse("http");
+      headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema));
+    }
+
     return _ws.url(
             String.format(
                 "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri))