diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml index 404e0bab814e8..f6160fdbcff67 100644 --- a/.github/actions/ci-optimization/action.yml +++ b/.github/actions/ci-optimization/action.yml @@ -48,7 +48,7 @@ runs: - "smoke-test/tests/cypress/**" - "docker/datahub-frontend/**" ingestion: - - "metadata-ingestion-modules/airflow-plugin/**" + - "metadata-ingestion-modules/**" - "metadata-ingestion/**" - "metadata-models/**" - "smoke-test/**" diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 180e0472a8d99..6b7f2b5035c25 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -69,6 +69,7 @@ jobs: java-version: 17 - uses: gradle/gradle-build-action@v2 - uses: actions/setup-python@v4 + if: ${{ needs.setup.outputs.ingestion_change == 'true' }} with: python-version: "3.10" cache: pip @@ -82,6 +83,10 @@ jobs: ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel env: NODE_OPTIONS: "--max-old-space-size=3072" + - name: Gradle compile (jdk8) for legacy Spark + if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} + run: | + ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava - uses: actions/upload-artifact@v3 if: always() with: diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index eb5822b5b480d..243bd90cd6003 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -24,9 +24,28 @@ concurrency: cancel-in-progress: true jobs: + setup: + runs-on: ubuntu-latest + outputs: + frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' }} + ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' }} + backend_change: ${{ steps.ci-optimize.outputs.backend-change == 'true' }} + docker_change: ${{ steps.ci-optimize.outputs.docker-change == 'true' }} + frontend_only: ${{ steps.ci-optimize.outputs.frontend-only == 'true' }} + ingestion_only: ${{ steps.ci-optimize.outputs.ingestion-only == 'true' }} + kafka_setup_change: ${{ steps.ci-optimize.outputs.kafka-setup-change == 'true' }} + mysql_setup_change: ${{ steps.ci-optimize.outputs.mysql-setup-change == 'true' }} + postgres_setup_change: ${{ steps.ci-optimize.outputs.postgres-setup-change == 'true' }} + elasticsearch_setup_change: ${{ steps.ci-optimize.outputs.elasticsearch-setup-change == 'true' }} + steps: + - name: Check out the repo + uses: hsheth2/sane-checkout-action@v1 + - uses: ./.github/actions/ci-optimization + id: ci-optimize build: runs-on: ubuntu-latest timeout-minutes: 60 + needs: setup steps: - uses: actions/checkout@v3 - name: Set up JDK 17 @@ -36,6 +55,7 @@ jobs: java-version: 17 - uses: gradle/gradle-build-action@v2 - uses: actions/setup-python@v4 + if: ${{ needs.setup.outputs.ingestion_change == 'true' }} with: python-version: "3.10" cache: "pip" diff --git a/build.gradle b/build.gradle index 27455f8592e6f..ba61d97f0ed6e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,32 @@ buildscript { - ext.jdkVersion = 17 - ext.javaClassVersion = 11 + ext.jdkVersionDefault = 17 + ext.javaClassVersionDefault = 11 + + ext.jdkVersion = { p -> + // If Spring 6 is present, hard dependency on jdk17 + if (p.configurations.any { it.getDependencies().any{ + (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) + || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) + }}) { + return 17 + } else { + // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault + return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault + } + } + + ext.javaClassVersion = { p -> + // If Spring 6 is present, hard dependency on jdk17 + if (p.configurations.any { it.getDependencies().any{ + (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6.")) + || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test")) + }}) { + return 17 + } else { + // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault + return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault + } + } ext.junitJupiterVersion = '5.6.1' // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md @@ -217,6 +243,7 @@ project.ext.externalDependency = [ 'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion", 'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15', 'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46', + 'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18', 'testng': 'org.testng:testng:7.8.0', 'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion, 'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion, @@ -252,23 +279,27 @@ allprojects { } } - if (project.plugins.hasPlugin('java') + /** + * If making changes to this section also see the sections for pegasus below + * which use project.plugins.hasPlugin('pegasus') + **/ + if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java') || project.plugins.hasPlugin('java-library') - || project.plugins.hasPlugin('application') - || project.plugins.hasPlugin('pegasus')) { + || project.plugins.hasPlugin('application'))) { java { toolchain { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } compileJava { - options.release = javaClassVersion + options.release = javaClassVersion(project) } + tasks.withType(JavaCompile).configureEach { javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } // Puts parameter names into compiled class files, necessary for Spring 6 options.compilerArgs.add("-parameters") @@ -276,24 +307,28 @@ allprojects { tasks.withType(JavaExec).configureEach { javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } + } + + // not duplicated, need to set this outside and inside afterEvaluate + afterEvaluate { + /** + * If making changes to this section also see the sections for pegasus below + * which use project.plugins.hasPlugin('pegasus') + **/ + if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java') + || project.plugins.hasPlugin('java-library') + || project.plugins.hasPlugin('application'))) { - // not duplicated, need to set this outside and inside afterEvaluate - afterEvaluate { compileJava { - options.release = javaClassVersion - } - tasks.withType(JavaCompile).configureEach { - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) - } + options.release = javaClassVersion(project) } tasks.withType(JavaExec).configureEach { javaLauncher = javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(jdkVersion) + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) } } } @@ -368,6 +403,30 @@ subprojects { dataTemplateCompile externalDependency.annotationApi // support > jdk8 restClientCompile spec.product.pegasus.restliClient } + + java { + toolchain { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } + + compileJava { + options.release = javaClassVersion(project) + } + + tasks.withType(JavaCompile).configureEach { + javaCompiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + // Puts parameter names into compiled class files, necessary for Spring 6 + options.compilerArgs.add("-parameters") + } + + tasks.withType(JavaExec).configureEach { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } } afterEvaluate { @@ -394,6 +453,16 @@ subprojects { dataTemplateCompile externalDependency.annotationApi // support > jdk8 restClientCompile spec.product.pegasus.restliClient } + + compileJava { + options.release = javaClassVersion(project) + } + + tasks.withType(JavaExec).configureEach { + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } + } } } } diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java index 25bf239ab835b..1c9dfd4686610 100644 --- a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java +++ b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java @@ -183,6 +183,7 @@ private void generateSchema(final File file) { final String fileBaseName; try { final JsonNode schema = JsonLoader.fromFile(file); + final JsonNode result = buildResult(schema.toString()); String prettySchema = JacksonUtils.prettyPrint(result); Path absolutePath = file.getAbsoluteFile().toPath(); @@ -195,11 +196,21 @@ private void generateSchema(final File file) { } else { fileBaseName = getBaseName(file.getName()); } - Files.write(Paths.get(jsonDirectory + sep + fileBaseName + ".json"), + + final String targetName; + if (schema.has("Aspect") && schema.get("Aspect").has("name") && + !schema.get("Aspect").get("name").asText().equalsIgnoreCase(fileBaseName)) { + targetName = OpenApiEntities.toUpperFirst(schema.get("Aspect").get("name").asText()); + prettySchema = prettySchema.replaceAll(fileBaseName, targetName); + } else { + targetName = fileBaseName; + } + + Files.write(Paths.get(jsonDirectory + sep + targetName + ".json"), prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); if (schema.has("Aspect")) { - aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + getBaseName(file.getName()))); + aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + targetName)); } } catch (IOException | ProcessingException e) { throw new RuntimeException(e); diff --git a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java index 888c4a0e99931..04cbadcdc6b7b 100644 --- a/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java +++ b/buildSrc/src/main/java/io/datahubproject/OpenApiEntities.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.metadata.models.registry.config.Entities; import com.linkedin.metadata.models.registry.config.Entity; @@ -58,8 +59,12 @@ public class OpenApiEntities { .add("notebookInfo").add("editableNotebookProperties") .add("dataProductProperties") .add("institutionalMemory") + .add("forms").add("formInfo").add("dynamicFormAssignment") .build(); + private final static ImmutableSet ENTITY_EXCLUSIONS = ImmutableSet.builder() + .add("structuredProperty") + .build(); public OpenApiEntities(JsonNodeFactory NODE_FACTORY) { this.NODE_FACTORY = NODE_FACTORY; @@ -117,14 +122,27 @@ public ObjectNode entityExtension(List nodesList, ObjectNode schemas return componentsNode; } - private static String toUpperFirst(String s) { - return s.substring(0, 1).toUpperCase() + s.substring(1); + /** + * Convert the pdl model names to desired class names. Upper case first letter unless the 3rd character is upper case. + * i.e. mlModel -> MLModel + * dataset -> Dataset + * dataProduct -> DataProduct + * @param s input string + * @return class name + */ + public static String toUpperFirst(String s) { + if (s.length() > 2 && s.substring(2, 3).equals(s.substring(2, 3).toUpperCase())) { + return s.substring(0, 2).toUpperCase() + s.substring(2); + } else { + return s.substring(0, 1).toUpperCase() + s.substring(1); + } } private Set withEntitySchema(ObjectNode schemasNode, Set definitions) { return entityMap.values().stream() // Make sure the primary key is defined .filter(entity -> definitions.contains(toUpperFirst(entity.getKeyAspect()))) + .filter(entity -> !ENTITY_EXCLUSIONS.contains(entity.getName())) .map(entity -> { final String upperName = toUpperFirst(entity.getName()); @@ -547,7 +565,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) { ObjectNode getMethod = NODE_FACTORY.objectNode() .put("summary", String.format("Get %s for %s.", aspect, entity.getName())) - .put("operationId", String.format("get%s", upperFirstAspect, upperFirstEntity)); + .put("operationId", String.format("get%s", upperFirstAspect)); getMethod.set("tags", tagsNode); ArrayNode singlePathParametersNode = NODE_FACTORY.arrayNode(); getMethod.set("parameters", singlePathParametersNode); @@ -575,13 +593,13 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) { .set("application/json", NODE_FACTORY.objectNode()))); ObjectNode headMethod = NODE_FACTORY.objectNode() .put("summary", String.format("%s on %s existence.", aspect, upperFirstEntity)) - .put("operationId", String.format("head%s", upperFirstAspect, upperFirstEntity)) + .put("operationId", String.format("head%s", upperFirstAspect)) .set("responses", headResponses); headMethod.set("tags", tagsNode); ObjectNode deleteMethod = NODE_FACTORY.objectNode() .put("summary", String.format("Delete %s on entity %s", aspect, upperFirstEntity)) - .put("operationId", String.format("delete%s", upperFirstAspect, upperFirstEntity)) + .put("operationId", String.format("delete%s", upperFirstAspect)) .set("responses", NODE_FACTORY.objectNode() .set("200", NODE_FACTORY.objectNode() .put("description", String.format("Delete %s on %s entity.", aspect, upperFirstEntity)) @@ -591,7 +609,7 @@ private ObjectNode buildSingleEntityAspectPath(Entity entity, String aspect) { ObjectNode postMethod = NODE_FACTORY.objectNode() .put("summary", String.format("Create aspect %s on %s ", aspect, upperFirstEntity)) - .put("operationId", String.format("create%s", upperFirstAspect, upperFirstEntity)); + .put("operationId", String.format("create%s", upperFirstAspect)); postMethod.set("requestBody", NODE_FACTORY.objectNode() .put("description", String.format("Create aspect %s on %s entity.", aspect, upperFirstEntity)) .put("required", true).set("content", NODE_FACTORY.objectNode() diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java index 3d87267f8ebe3..0f2945d5d2393 100644 --- a/datahub-frontend/app/config/ConfigurationProvider.java +++ b/datahub-frontend/app/config/ConfigurationProvider.java @@ -1,5 +1,6 @@ package config; +import com.linkedin.metadata.config.VisualConfiguration; import com.linkedin.metadata.config.cache.CacheConfiguration; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.spring.YamlPropertySourceFactory; @@ -22,4 +23,7 @@ public class ConfigurationProvider { /** Configuration for caching */ private CacheConfiguration cache; + + /** Configuration for the view layer */ + private VisualConfiguration visualConfig; } diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index 60971bf06e27b..df0cd4f4ff82f 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -13,6 +13,7 @@ import com.linkedin.util.Pair; import com.typesafe.config.Config; import java.io.InputStream; +import java.net.URI; import java.time.Duration; import java.util.List; import java.util.Map; @@ -125,6 +126,12 @@ public CompletableFuture proxy(String path, Http.Request request) headers.put(Http.HeaderNames.X_FORWARDED_HOST, headers.get(Http.HeaderNames.HOST)); } + if (!headers.containsKey(Http.HeaderNames.X_FORWARDED_PROTO)) { + final String schema = + Optional.ofNullable(URI.create(request.uri()).getScheme()).orElse("http"); + headers.put(Http.HeaderNames.X_FORWARDED_PROTO, List.of(schema)); + } + return _ws.url( String.format( "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri)) diff --git a/datahub-frontend/app/controllers/RedirectController.java b/datahub-frontend/app/controllers/RedirectController.java new file mode 100644 index 0000000000000..17f86b7fbffae --- /dev/null +++ b/datahub-frontend/app/controllers/RedirectController.java @@ -0,0 +1,25 @@ +package controllers; + +import config.ConfigurationProvider; +import javax.inject.Inject; +import javax.inject.Singleton; +import play.mvc.Controller; +import play.mvc.Http; +import play.mvc.Result; + +@Singleton +public class RedirectController extends Controller { + + @Inject ConfigurationProvider config; + + public Result favicon(Http.Request request) { + if (config.getVisualConfig().getAssets().getFaviconUrl().startsWith("http")) { + return permanentRedirect(config.getVisualConfig().getAssets().getFaviconUrl()); + } else { + final String prefix = config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/") ? "/public" : "/public/"; + return ok(Application.class.getResourceAsStream( + prefix + config.getVisualConfig().getAssets().getFaviconUrl())) + .as("image/x-icon"); + } + } +} diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 6b53a2789e7cc..9eac7aa34c3e3 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -36,9 +36,13 @@ PUT /openapi/*path c HEAD /openapi/*path controllers.Application.proxy(path: String, request: Request) PATCH /openapi/*path controllers.Application.proxy(path: String, request: Request) + # Analytics route POST /track controllers.TrackingController.track(request: Request) +# Map static resources from the /public folder to the /assets URL path +GET /assets/icons/favicon.ico controllers.RedirectController.favicon(request: Request) + # Known React asset routes GET /assets/*file controllers.Assets.at(path="/public/assets", file) GET /node_modules/*file controllers.Assets.at(path="/public/node_modules", file) diff --git a/datahub-frontend/public b/datahub-frontend/public new file mode 120000 index 0000000000000..60c68c7b4b1bc --- /dev/null +++ b/datahub-frontend/public @@ -0,0 +1 @@ +../datahub-web-react/public \ No newline at end of file diff --git a/datahub-frontend/test/resources/public/logos/datahub-logo.png b/datahub-frontend/test/resources/public/logos/datahub-logo.png new file mode 100644 index 0000000000000..5e34e6425d23f Binary files /dev/null and b/datahub-frontend/test/resources/public/logos/datahub-logo.png differ diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 3d5a961d6f7c7..f273a4dd0eea5 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':metadata-service:services') implementation project(':metadata-io') implementation project(':metadata-utils') + implementation project(':metadata-models') implementation externalDependency.graphqlJava implementation externalDependency.graphqlJavaScalars @@ -40,8 +41,10 @@ graphqlCodegen { "$projectDir/src/main/resources/auth.graphql".toString(), "$projectDir/src/main/resources/timeline.graphql".toString(), "$projectDir/src/main/resources/tests.graphql".toString(), + "$projectDir/src/main/resources/properties.graphql".toString(), "$projectDir/src/main/resources/step.graphql".toString(), "$projectDir/src/main/resources/lineage.graphql".toString(), + "$projectDir/src/main/resources/forms.graphql".toString() ] outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java") packageName = "com.linkedin.datahub.graphql.generated" diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java index e45bed33eb023..2bde7cb61047b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java @@ -19,9 +19,10 @@ private Constants() {} public static final String TESTS_SCHEMA_FILE = "tests.graphql"; public static final String STEPS_SCHEMA_FILE = "step.graphql"; public static final String LINEAGE_SCHEMA_FILE = "lineage.graphql"; + public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql"; + public static final String FORMS_SCHEMA_FILE = "forms.graphql"; public static final String BROWSE_PATH_DELIMITER = "/"; public static final String BROWSE_PATH_V2_DELIMITER = "␟"; public static final String VERSION_STAMP_FIELD_NAME = "versionStamp"; - public static final String ENTITY_FILTER_NAME = "_entityType"; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index f61d76e72e8bd..4b5bbdb6e15ec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -57,6 +57,7 @@ import com.linkedin.datahub.graphql.generated.EntityRelationship; import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint; +import com.linkedin.datahub.graphql.generated.FormActorAssignment; import com.linkedin.datahub.graphql.generated.GetRootGlossaryNodesResult; import com.linkedin.datahub.graphql.generated.GetRootGlossaryTermsResult; import com.linkedin.datahub.graphql.generated.GlossaryNode; @@ -91,12 +92,17 @@ import com.linkedin.datahub.graphql.generated.QuerySubject; import com.linkedin.datahub.graphql.generated.QuickFilter; import com.linkedin.datahub.graphql.generated.RecommendationContent; +import com.linkedin.datahub.graphql.generated.SchemaField; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchResult; import com.linkedin.datahub.graphql.generated.SiblingProperties; +import com.linkedin.datahub.graphql.generated.StructuredPropertiesEntry; +import com.linkedin.datahub.graphql.generated.StructuredPropertyDefinition; +import com.linkedin.datahub.graphql.generated.StructuredPropertyParams; import com.linkedin.datahub.graphql.generated.Test; import com.linkedin.datahub.graphql.generated.TestResult; +import com.linkedin.datahub.graphql.generated.TypeQualifier; import com.linkedin.datahub.graphql.generated.UserUsageCounts; import com.linkedin.datahub.graphql.resolvers.MeResolver; import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver; @@ -135,6 +141,11 @@ import com.linkedin.datahub.graphql.resolvers.embed.UpdateEmbedResolver; import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver; import com.linkedin.datahub.graphql.resolvers.entity.EntityPrivilegesResolver; +import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver; +import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver; +import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver; +import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver; +import com.linkedin.datahub.graphql.resolvers.form.VerifyFormResolver; import com.linkedin.datahub.graphql.resolvers.glossary.AddRelatedTermsResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryNodeResolver; import com.linkedin.datahub.graphql.resolvers.glossary.CreateGlossaryTermResolver; @@ -159,6 +170,7 @@ import com.linkedin.datahub.graphql.resolvers.ingest.secret.DeleteSecretResolver; import com.linkedin.datahub.graphql.resolvers.ingest.secret.GetSecretValuesResolver; import com.linkedin.datahub.graphql.resolvers.ingest.secret.ListSecretsResolver; +import com.linkedin.datahub.graphql.resolvers.ingest.secret.UpdateSecretResolver; import com.linkedin.datahub.graphql.resolvers.ingest.source.DeleteIngestionSourceResolver; import com.linkedin.datahub.graphql.resolvers.ingest.source.GetIngestionSourceResolver; import com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver; @@ -254,6 +266,7 @@ import com.linkedin.datahub.graphql.resolvers.type.EntityInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver; +import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver; import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver; @@ -288,7 +301,10 @@ import com.linkedin.datahub.graphql.types.dataset.DatasetType; import com.linkedin.datahub.graphql.types.dataset.VersionedDatasetType; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetProfileMapper; +import com.linkedin.datahub.graphql.types.datatype.DataTypeType; import com.linkedin.datahub.graphql.types.domain.DomainType; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeType; +import com.linkedin.datahub.graphql.types.form.FormType; import com.linkedin.datahub.graphql.types.glossary.GlossaryNodeType; import com.linkedin.datahub.graphql.types.glossary.GlossaryTermType; import com.linkedin.datahub.graphql.types.mlmodel.MLFeatureTableType; @@ -303,6 +319,7 @@ import com.linkedin.datahub.graphql.types.role.DataHubRoleType; import com.linkedin.datahub.graphql.types.rolemetadata.RoleType; import com.linkedin.datahub.graphql.types.schemafield.SchemaFieldType; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyType; import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.test.TestType; import com.linkedin.datahub.graphql.types.view.DataHubViewType; @@ -323,6 +340,7 @@ import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.service.DataProductService; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.LineageService; import com.linkedin.metadata.service.OwnershipTypeService; import com.linkedin.metadata.service.QueryService; @@ -392,6 +410,7 @@ public class GmsGraphQLEngine { private final LineageService lineageService; private final QueryService queryService; private final DataProductService dataProductService; + private final FormService formService; private final FeatureFlags featureFlags; @@ -439,6 +458,10 @@ public class GmsGraphQLEngine { private final QueryType queryType; private final DataProductType dataProductType; private final OwnershipType ownershipType; + private final StructuredPropertyType structuredPropertyType; + private final DataTypeType dataTypeType; + private final EntityTypeType entityTypeType; + private final FormType formType; /** A list of GraphQL Plugins that extend the core engine */ private final List graphQLPlugins; @@ -494,6 +517,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.lineageService = args.lineageService; this.queryService = args.queryService; this.dataProductService = args.dataProductService; + this.formService = args.formService; this.ingestionConfiguration = Objects.requireNonNull(args.ingestionConfiguration); this.authenticationConfiguration = Objects.requireNonNull(args.authenticationConfiguration); @@ -533,11 +557,15 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.testType = new TestType(entityClient); this.dataHubPolicyType = new DataHubPolicyType(entityClient); this.dataHubRoleType = new DataHubRoleType(entityClient); - this.schemaFieldType = new SchemaFieldType(); + this.schemaFieldType = new SchemaFieldType(entityClient, featureFlags); this.dataHubViewType = new DataHubViewType(entityClient); this.queryType = new QueryType(entityClient); this.dataProductType = new DataProductType(entityClient); this.ownershipType = new OwnershipType(entityClient); + this.structuredPropertyType = new StructuredPropertyType(entityClient); + this.dataTypeType = new DataTypeType(entityClient); + this.entityTypeType = new EntityTypeType(entityClient); + this.formType = new FormType(entityClient); // Init Lists this.entityTypes = @@ -573,11 +601,16 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { dataHubViewType, queryType, dataProductType, - ownershipType); + ownershipType, + structuredPropertyType, + dataTypeType, + entityTypeType, + formType); this.loadableTypes = new ArrayList<>(entityTypes); // Extend loadable types with types from the plugins // This allows us to offer search and browse capabilities out of the box for those types for (GmsGraphQLPlugin plugin : this.graphQLPlugins) { + this.entityTypes.addAll(plugin.getEntityTypes()); Collection> pluginLoadableTypes = plugin.getLoadableTypes(); if (pluginLoadableTypes != null) { this.loadableTypes.addAll(pluginLoadableTypes); @@ -654,18 +687,23 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureRoleResolvers(builder); configureSchemaFieldResolvers(builder); configureEntityPathResolvers(builder); + configureResolvedAuditStampResolvers(builder); configureViewResolvers(builder); configureQueryEntityResolvers(builder); configureOwnershipTypeResolver(builder); configurePluginResolvers(builder); + configureStructuredPropertyResolvers(builder); + configureFormResolvers(builder); } private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) { builder.type( "Role", typeWiring -> - typeWiring.dataFetcher( - "relationships", new EntityRelationshipsResultResolver(graphClient))); + typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))); builder.type( "RoleAssociation", typeWiring -> @@ -703,7 +741,9 @@ public GraphQLEngine.Builder builder() { .addSchema(fileBasedSchema(TIMELINE_SCHEMA_FILE)) .addSchema(fileBasedSchema(TESTS_SCHEMA_FILE)) .addSchema(fileBasedSchema(STEPS_SCHEMA_FILE)) - .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE)); + .addSchema(fileBasedSchema(LINEAGE_SCHEMA_FILE)) + .addSchema(fileBasedSchema(PROPERTIES_SCHEMA_FILE)) + .addSchema(fileBasedSchema(FORMS_SCHEMA_FILE)); for (GmsGraphQLPlugin plugin : this.graphQLPlugins) { List pluginSchemaFiles = plugin.getSchemaFiles(); @@ -767,6 +807,8 @@ private void configureContainerResolvers(final RuntimeWiring.Builder builder) { typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("entities", new ContainerEntitiesResolver(entityClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("exists", new EntityExistsResolver(entityService)) .dataFetcher( "platform", @@ -841,7 +883,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { "scrollAcrossLineage", new ScrollAcrossLineageResolver(this.entityClient)) .dataFetcher( "aggregateAcrossEntities", - new AggregateAcrossEntitiesResolver(this.entityClient, this.viewService)) + new AggregateAcrossEntitiesResolver( + this.entityClient, this.viewService, this.formService)) .dataFetcher("autoComplete", new AutoCompleteResolver(searchableTypes)) .dataFetcher( "autoCompleteForMultiple", @@ -928,7 +971,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "listOwnershipTypes", new ListOwnershipTypesResolver(this.entityClient)) .dataFetcher( - "browseV2", new BrowseV2Resolver(this.entityClient, this.viewService))); + "browseV2", + new BrowseV2Resolver(this.entityClient, this.viewService, this.formService))); } private DataFetcher getEntitiesResolver() { @@ -1043,6 +1087,8 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher( "createSecret", new CreateSecretResolver(this.entityClient, this.secretService)) .dataFetcher("deleteSecret", new DeleteSecretResolver(this.entityClient)) + .dataFetcher( + "updateSecret", new UpdateSecretResolver(this.entityClient, this.secretService)) .dataFetcher( "createAccessToken", new CreateAccessTokenResolver(this.statefulTokenService)) .dataFetcher( @@ -1139,7 +1185,14 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { new UpdateOwnershipTypeResolver(this.ownershipTypeService)) .dataFetcher( "deleteOwnershipType", - new DeleteOwnershipTypeResolver(this.ownershipTypeService))); + new DeleteOwnershipTypeResolver(this.ownershipTypeService)) + .dataFetcher("submitFormPrompt", new SubmitFormPromptResolver(this.formService)) + .dataFetcher("batchAssignForm", new BatchAssignFormResolver(this.formService)) + .dataFetcher( + "createDynamicFormAssignment", + new CreateDynamicFormAssignmentResolver(this.formService)) + .dataFetcher( + "verifyForm", new VerifyFormResolver(this.formService, this.groupService))); } private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder) { @@ -1342,7 +1395,25 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder typeWiring.dataFetcher( "ownershipType", new EntityTypeResolver( - entityTypes, (env) -> ((Owner) env.getSource()).getOwnershipType()))); + entityTypes, (env) -> ((Owner) env.getSource()).getOwnershipType()))) + .type( + "StructuredPropertiesEntry", + typeWiring -> + typeWiring + .dataFetcher( + "structuredProperty", + new LoadableTypeResolver<>( + structuredPropertyType, + (env) -> + ((StructuredPropertiesEntry) env.getSource()) + .getStructuredProperty() + .getUrn())) + .dataFetcher( + "valueEntities", + new BatchGetEntitiesResolver( + entityTypes, + (env) -> + ((StructuredPropertiesEntry) env.getSource()).getValueEntities()))); } /** @@ -1422,6 +1493,14 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { "owner", new OwnerTypeResolver<>( ownerTypes, (env) -> ((Owner) env.getSource()).getOwner()))) + .type( + "SchemaField", + typeWiring -> + typeWiring.dataFetcher( + "schemaFieldEntity", + new LoadableTypeResolver<>( + schemaFieldType, + (env) -> ((SchemaField) env.getSource()).getSchemaFieldEntity().getUrn()))) .type( "UserUsageCounts", typeWiring -> @@ -1518,6 +1597,8 @@ private void configureGlossaryTermResolvers(final RuntimeWiring.Builder builder) .dataFetcher("schemaMetadata", new AspectResolver()) .dataFetcher("parentNodes", new ParentNodesResolver(entityClient)) .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("exists", new EntityExistsResolver(entityService))); } @@ -1528,7 +1609,9 @@ private void configureGlossaryNodeResolvers(final RuntimeWiring.Builder builder) typeWiring .dataFetcher("parentNodes", new ParentNodesResolver(entityClient)) .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient)) - .dataFetcher("exists", new EntityExistsResolver(entityService))); + .dataFetcher("exists", new EntityExistsResolver(entityService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))); } private void configureSchemaFieldResolvers(final RuntimeWiring.Builder builder) { @@ -1551,6 +1634,16 @@ private void configureEntityPathResolvers(final RuntimeWiring.Builder builder) { entityTypes, (env) -> ((EntityPath) env.getSource()).getPath()))); } + private void configureResolvedAuditStampResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "ResolvedAuditStamp", + typeWiring -> + typeWiring.dataFetcher( + "actor", + new LoadableTypeResolver<>( + corpUserType, (env) -> ((CorpUser) env.getSource()).getUrn()))); + } + /** * Configures resolvers responsible for resolving the {@link * com.linkedin.datahub.graphql.generated.CorpUser} type. @@ -1559,8 +1652,10 @@ private void configureCorpUserResolvers(final RuntimeWiring.Builder builder) { builder.type( "CorpUser", typeWiring -> - typeWiring.dataFetcher( - "relationships", new EntityRelationshipsResultResolver(graphClient))); + typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))); builder.type( "CorpUserInfo", typeWiring -> @@ -1581,6 +1676,8 @@ private void configureCorpGroupResolvers(final RuntimeWiring.Builder builder) { typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("exists", new EntityExistsResolver(entityService))); builder .type( @@ -1623,8 +1720,10 @@ private void configureTagAssociationResolver(final RuntimeWiring.Builder builder builder.type( "Tag", typeWiring -> - typeWiring.dataFetcher( - "relationships", new EntityRelationshipsResultResolver(graphClient))); + typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry))); builder.type( "TagAssociation", typeWiring -> @@ -1659,6 +1758,8 @@ private void configureNotebookResolvers(final RuntimeWiring.Builder builder) { typeWiring -> typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.notebookType)) .dataFetcher( "platform", @@ -1690,6 +1791,8 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dashboardType)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher( "platform", new LoadableTypeResolver<>( @@ -1758,6 +1861,42 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) { }))); } + private void configureStructuredPropertyResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "StructuredPropertyDefinition", + typeWiring -> + typeWiring + .dataFetcher( + "valueType", + new LoadableTypeResolver<>( + dataTypeType, + (env) -> + ((StructuredPropertyDefinition) env.getSource()) + .getValueType() + .getUrn())) + .dataFetcher( + "entityTypes", + new LoadableTypeBatchResolver<>( + entityTypeType, + (env) -> + ((StructuredPropertyDefinition) env.getSource()) + .getEntityTypes().stream() + .map(entityTypeType.getKeyProvider()) + .collect(Collectors.toList())))); + builder.type( + "TypeQualifier", + typeWiring -> + typeWiring.dataFetcher( + "allowedTypes", + new LoadableTypeBatchResolver<>( + entityTypeType, + (env) -> + ((TypeQualifier) env.getSource()) + .getAllowedTypes().stream() + .map(entityTypeType.getKeyProvider()) + .collect(Collectors.toList())))); + } + /** * Configures resolvers responsible for resolving the {@link * com.linkedin.datahub.graphql.generated.Chart} type. @@ -1769,6 +1908,8 @@ private void configureChartResolvers(final RuntimeWiring.Builder builder) { typeWiring .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.chartType)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher( "platform", @@ -1858,6 +1999,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) { .type( "HyperParameterValueType", typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver())) + .type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver())) .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver())) .type( "TimeSeriesAspect", @@ -1884,6 +2026,8 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) { "relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dataJobType)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher( "dataFlow", new LoadableTypeResolver<>( @@ -1947,6 +2091,8 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.dataFlowType)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher( "platform", new LoadableTypeResolver<>( @@ -1979,6 +2125,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde "relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher( "browsePaths", new EntityBrowsePathsResolver(this.mlFeatureTableType)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("exists", new EntityExistsResolver(entityService)) .dataFetcher( @@ -2064,6 +2212,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.mlModelType)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher("exists", new EntityExistsResolver(entityService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher( "platform", new LoadableTypeResolver<>( @@ -2103,6 +2253,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde "relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher( "browsePaths", new EntityBrowsePathsResolver(this.mlModelGroupType)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) .dataFetcher( "platform", @@ -2127,6 +2279,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde .dataFetcher( "relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("exists", new EntityExistsResolver(entityService)) .dataFetcher( "dataPlatformInstance", @@ -2145,6 +2299,8 @@ private void configureMLFeatureTableResolvers(final RuntimeWiring.Builder builde .dataFetcher( "relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("lineage", new EntityLineageResultResolver(siblingGraphService)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("exists", new EntityExistsResolver(entityService)) .dataFetcher( "dataPlatformInstance", @@ -2179,6 +2335,8 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) { typeWiring .dataFetcher("entities", new DomainEntitiesResolver(this.entityClient)) .dataFetcher("parentDomains", new ParentDomainsResolver(this.entityClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))); builder.type( "DomainAssociation", @@ -2193,12 +2351,64 @@ private void configureDomainResolvers(final RuntimeWiring.Builder builder) { .getUrn()))); } + private void configureFormResolvers(final RuntimeWiring.Builder builder) { + builder.type( + "FormAssociation", + typeWiring -> + typeWiring.dataFetcher( + "form", + new LoadableTypeResolver<>( + formType, + (env) -> + ((com.linkedin.datahub.graphql.generated.FormAssociation) env.getSource()) + .getForm() + .getUrn()))); + builder.type( + "StructuredPropertyParams", + typeWiring -> + typeWiring.dataFetcher( + "structuredProperty", + new LoadableTypeResolver<>( + structuredPropertyType, + (env) -> + ((StructuredPropertyParams) env.getSource()) + .getStructuredProperty() + .getUrn()))); + builder.type( + "FormActorAssignment", + typeWiring -> + typeWiring + .dataFetcher( + "users", + new LoadableTypeBatchResolver<>( + corpUserType, + (env) -> { + final FormActorAssignment actors = env.getSource(); + return actors.getUsers().stream() + .map(CorpUser::getUrn) + .collect(Collectors.toList()); + })) + .dataFetcher( + "groups", + new LoadableTypeBatchResolver<>( + corpGroupType, + (env) -> { + final FormActorAssignment actors = env.getSource(); + return actors.getGroups().stream() + .map(CorpGroup::getUrn) + .collect(Collectors.toList()); + })) + .dataFetcher("isAssignedToMe", new IsFormAssignedToMeResolver(groupService))); + } + private void configureDataProductResolvers(final RuntimeWiring.Builder builder) { builder.type( "DataProduct", typeWiring -> typeWiring .dataFetcher("entities", new ListDataProductAssetsResolver(this.entityClient)) + .dataFetcher( + "aspects", new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java index 4829194a8ce4d..5b780cc8cb40b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngineArgs.java @@ -25,6 +25,7 @@ import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.service.DataProductService; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.LineageService; import com.linkedin.metadata.service.OwnershipTypeService; import com.linkedin.metadata.service.QueryService; @@ -73,6 +74,7 @@ public class GmsGraphQLEngineArgs { QueryService queryService; FeatureFlags featureFlags; DataProductService dataProductService; + FormService formService; // any fork specific args should go below this line } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java index 472d9465aeee1..a544bd46527c4 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLPlugin.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.graphql; +import com.linkedin.datahub.graphql.types.EntityType; import com.linkedin.datahub.graphql.types.LoadableType; import graphql.schema.idl.RuntimeWiring; import java.util.Collection; @@ -34,6 +35,9 @@ public interface GmsGraphQLPlugin { */ Collection> getLoadableTypes(); + /** Return a list of Entity Types that the plugin services */ + Collection> getEntityTypes(); + /** * Optional callback that a plugin can implement to configure any Query, Mutation or Type specific * resolvers. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/SubTypesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/SubTypesResolver.java new file mode 100644 index 0000000000000..ae8ac4330e7fb --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/SubTypesResolver.java @@ -0,0 +1,55 @@ +package com.linkedin.datahub.graphql; + +import com.linkedin.common.SubTypes; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AllArgsConstructor +public class SubTypesResolver implements DataFetcher> { + + EntityClient _entityClient; + String _entityType; + String _aspectName; + + @Override + @Nullable + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + return CompletableFuture.supplyAsync( + () -> { + final QueryContext context = environment.getContext(); + SubTypes subType = null; + final String urnStr = ((Entity) environment.getSource()).getUrn(); + try { + final Urn urn = Urn.createFromString(urnStr); + EntityResponse entityResponse = + _entityClient + .batchGetV2( + urn.getEntityType(), + Collections.singleton(urn), + Collections.singleton(_aspectName), + context.getAuthentication()) + .get(urn); + if (entityResponse != null && entityResponse.getAspects().containsKey(_aspectName)) { + subType = + new SubTypes(entityResponse.getAspects().get(_aspectName).getValue().data()); + } + } catch (RemoteInvocationException | URISyntaxException e) { + throw new RuntimeException( + "Failed to fetch aspect " + _aspectName + " for urn " + urnStr + " ", e); + } + return subType; + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java index 22ee4d4d4845c..d8665ae784bd1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java @@ -10,7 +10,7 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.RawAspect; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.models.AspectSpec; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java index 31a8359f8f0e3..de389a358d936 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java @@ -12,8 +12,8 @@ import com.linkedin.datahub.graphql.generated.BarSegment; import com.linkedin.datahub.graphql.generated.MetadataAnalyticsInput; import com.linkedin.datahub.graphql.generated.NamedBar; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.filter.Filter; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java index 03333bda05f61..baea3ea4e6201 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsService.java @@ -10,7 +10,7 @@ import com.linkedin.datahub.graphql.generated.NamedLine; import com.linkedin.datahub.graphql.generated.NumericDataPoint; import com.linkedin.datahub.graphql.generated.Row; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.util.List; import java.util.Map; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index e74ed09849763..667ccd368a729 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -17,4 +17,5 @@ public class FeatureFlags { private boolean showAcrylInfo = false; private boolean showAccessManagement = false; private boolean nestedDomainsEnabled = false; + private boolean schemaFieldEntityFetchEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java index da4a3a76dd7e0..d9ce2472c8634 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java @@ -12,13 +12,14 @@ import com.linkedin.datahub.graphql.generated.BrowseResultsV2; import com.linkedin.datahub.graphql.generated.BrowseV2Input; import com.linkedin.datahub.graphql.generated.EntityType; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.resolvers.search.SearchUtils; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.browse.BrowseResultV2; import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; @@ -37,6 +38,7 @@ public class BrowseV2Resolver implements DataFetcher get(DataFetchingEnvironment environmen ? BROWSE_PATH_V2_DELIMITER + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath()) : ""; - final Filter filter = ResolverUtils.buildFilter(null, input.getOrFilters()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); BrowseResultV2 browseResults = _entityClient.browseV2( @@ -76,8 +78,8 @@ public CompletableFuture get(DataFetchingEnvironment environmen pathStr, maybeResolvedView != null ? SearchUtils.combineFilters( - filter, maybeResolvedView.getDefinition().getFilter()) - : filter, + inputFilter, maybeResolvedView.getDefinition().getFilter()) + : inputFilter, sanitizedQuery, start, count, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 81b52991cde90..f127e6a49abff 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -126,9 +126,15 @@ public CompletableFuture get(final DataFetchingEnvironment environmen appConfig.setAuthConfig(authConfig); final VisualConfig visualConfig = new VisualConfig(); - if (_visualConfiguration != null && _visualConfiguration.getAssets() != null) { - visualConfig.setLogoUrl(_visualConfiguration.getAssets().getLogoUrl()); - visualConfig.setFaviconUrl(_visualConfiguration.getAssets().getFaviconUrl()); + if (_visualConfiguration != null) { + if (_visualConfiguration.getAssets() != null) { + visualConfig.setLogoUrl(_visualConfiguration.getAssets().getLogoUrl()); + visualConfig.setFaviconUrl(_visualConfiguration.getAssets().getFaviconUrl()); + } + if (_visualConfiguration.getAppTitle() != null) { + visualConfig.setAppTitle(_visualConfiguration.getAppTitle()); + } + visualConfig.setHideGlossary(_visualConfiguration.isHideGlossary()); } if (_visualConfiguration != null && _visualConfiguration.getQueriesTab() != null) { QueriesTabConfig queriesTabConfig = new QueriesTabConfig(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java index a0f1698bf99e8..72912087190c0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java @@ -12,9 +12,9 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.dataproduct.DataProductAssociation; import com.linkedin.dataproduct.DataProductProperties; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java index 8f6d109e71b2c..6229e38954163 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java @@ -7,7 +7,7 @@ import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.DomainEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.filter.Condition; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/BatchAssignFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/BatchAssignFormResolver.java new file mode 100644 index 0000000000000..39c9210c289e1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/BatchAssignFormResolver.java @@ -0,0 +1,52 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.BatchAssignFormInput; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class BatchAssignFormResolver implements DataFetcher> { + + private final FormService _formService; + + public BatchAssignFormResolver(@Nonnull final FormService formService) { + _formService = Objects.requireNonNull(formService, "formService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final BatchAssignFormInput input = + bindArgument(environment.getArgument("input"), BatchAssignFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); + final List entityUrns = input.getEntityUrns(); + final Authentication authentication = context.getAuthentication(); + + return CompletableFuture.supplyAsync( + () -> { + try { + _formService.batchAssignFormToEntities( + entityUrns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()), + formUrn, + authentication); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java new file mode 100644 index 0000000000000..5b5f058dbdeac --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java @@ -0,0 +1,50 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateDynamicFormAssignmentInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class CreateDynamicFormAssignmentResolver + implements DataFetcher> { + + private final FormService _formService; + + public CreateDynamicFormAssignmentResolver(@Nonnull final FormService formService) { + _formService = Objects.requireNonNull(formService, "formService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final CreateDynamicFormAssignmentInput input = + bindArgument(environment.getArgument("input"), CreateDynamicFormAssignmentInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); + final DynamicFormAssignment formAssignment = FormUtils.mapDynamicFormAssignment(input); + final Authentication authentication = context.getAuthentication(); + + return CompletableFuture.supplyAsync( + () -> { + try { + _formService.createDynamicFormAssignment(formAssignment, formUrn, authentication); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolver.java new file mode 100644 index 0000000000000..e7bf87ae7644e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolver.java @@ -0,0 +1,80 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CorpGroup; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.FormActorAssignment; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class IsFormAssignedToMeResolver implements DataFetcher> { + + private final GroupService _groupService; + + public IsFormAssignedToMeResolver(@Nonnull final GroupService groupService) { + _groupService = Objects.requireNonNull(groupService, "groupService must not be null"); + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); + final FormActorAssignment parent = environment.getSource(); + + return CompletableFuture.supplyAsync( + () -> { + try { + + // Assign urn and group urns + final Set assignedUserUrns = + parent.getUsers() != null + ? parent.getUsers().stream().map(CorpUser::getUrn).collect(Collectors.toSet()) + : Collections.emptySet(); + + final Set assignedGroupUrns = + parent.getGroups() != null + ? parent.getGroups().stream().map(CorpGroup::getUrn).collect(Collectors.toSet()) + : Collections.emptySet(); + + final Urn userUrn = Urn.createFromString(context.getActorUrn()); + + // First check whether user is directly assigned. + if (assignedUserUrns.size() > 0) { + boolean isUserAssigned = assignedUserUrns.contains(userUrn.toString()); + if (isUserAssigned) { + return true; + } + } + + // Next check whether the user is assigned indirectly, by group. + if (assignedGroupUrns.size() > 0) { + final List groupUrns = + _groupService.getGroupsForUser(userUrn, context.getAuthentication()); + boolean isUserGroupAssigned = + groupUrns.stream() + .anyMatch(groupUrn -> assignedGroupUrns.contains(groupUrn.toString())); + if (isUserGroupAssigned) { + return true; + } + } + } catch (Exception e) { + log.error( + "Failed to determine whether the form is assigned to the currently authenticated user! Returning false.", + e); + } + + // Else the user is not directly assigned. + return false; + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/SubmitFormPromptResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/SubmitFormPromptResolver.java new file mode 100644 index 0000000000000..5b40c353b3809 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/SubmitFormPromptResolver.java @@ -0,0 +1,89 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.FormPromptType; +import com.linkedin.datahub.graphql.generated.SubmitFormPromptInput; +import com.linkedin.datahub.graphql.resolvers.mutate.util.FormUtils; +import com.linkedin.metadata.service.FormService; +import com.linkedin.structured.PrimitivePropertyValueArray; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class SubmitFormPromptResolver implements DataFetcher> { + + private final FormService _formService; + + public SubmitFormPromptResolver(@Nonnull final FormService formService) { + _formService = Objects.requireNonNull(formService, "formService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final Urn entityUrn = UrnUtils.getUrn(environment.getArgument("urn")); + final SubmitFormPromptInput input = + bindArgument(environment.getArgument("input"), SubmitFormPromptInput.class); + final String promptId = input.getPromptId(); + final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); + final String fieldPath = input.getFieldPath(); + + return CompletableFuture.supplyAsync( + () -> { + try { + if (input.getType().equals(FormPromptType.STRUCTURED_PROPERTY)) { + if (input.getStructuredPropertyParams() == null) { + throw new IllegalArgumentException( + "Failed to provide structured property params for prompt type STRUCTURED_PROPERTY"); + } + final Urn structuredPropertyUrn = + UrnUtils.getUrn(input.getStructuredPropertyParams().getStructuredPropertyUrn()); + final PrimitivePropertyValueArray values = + FormUtils.getStructuredPropertyValuesFromInput(input); + + return _formService.submitStructuredPropertyPromptResponse( + entityUrn, + structuredPropertyUrn, + values, + formUrn, + promptId, + context.getAuthentication()); + } else if (input.getType().equals(FormPromptType.FIELDS_STRUCTURED_PROPERTY)) { + if (input.getStructuredPropertyParams() == null) { + throw new IllegalArgumentException( + "Failed to provide structured property params for prompt type FIELDS_STRUCTURED_PROPERTY"); + } + if (fieldPath == null) { + throw new IllegalArgumentException( + "Failed to provide fieldPath for prompt type FIELDS_STRUCTURED_PROPERTY"); + } + final Urn structuredPropertyUrn = + UrnUtils.getUrn(input.getStructuredPropertyParams().getStructuredPropertyUrn()); + final PrimitivePropertyValueArray values = + FormUtils.getStructuredPropertyValuesFromInput(input); + + return _formService.submitFieldStructuredPropertyPromptResponse( + entityUrn, + structuredPropertyUrn, + values, + formUrn, + promptId, + fieldPath, + context.getAuthentication()); + } + return false; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolver.java new file mode 100644 index 0000000000000..54e3562c97add --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolver.java @@ -0,0 +1,63 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.VerifyFormInput; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +public class VerifyFormResolver implements DataFetcher> { + + private final FormService _formService; + private final GroupService _groupService; + + public VerifyFormResolver( + @Nonnull final FormService formService, @Nonnull final GroupService groupService) { + _formService = Objects.requireNonNull(formService, "formService must not be null"); + _groupService = Objects.requireNonNull(groupService, "groupService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) + throws Exception { + final QueryContext context = environment.getContext(); + + final VerifyFormInput input = + bindArgument(environment.getArgument("input"), VerifyFormInput.class); + final Urn formUrn = UrnUtils.getUrn(input.getFormUrn()); + final Urn entityUrn = UrnUtils.getUrn(input.getEntityUrn()); + final Authentication authentication = context.getAuthentication(); + final Urn actorUrn = UrnUtils.getUrn(authentication.getActor().toUrnStr()); + + return CompletableFuture.supplyAsync( + () -> { + try { + final List groupsForUser = + _groupService.getGroupsForUser(actorUrn, authentication); + if (!_formService.isFormAssignedToUser( + formUrn, entityUrn, actorUrn, groupsForUser, authentication)) { + throw new AuthorizationException( + String.format( + "Failed to authorize form on entity as form with urn %s is not assigned to user", + formUrn)); + } + _formService.verifyFormForEntity(formUrn, entityUrn, authentication); + return true; + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to perform update against input %s", input), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java index 6a204286ba44e..b52153d70fa7b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/glossary/CreateGlossaryNodeResolver.java @@ -37,7 +37,6 @@ public class CreateGlossaryNodeResolver implements DataFetcher get(DataFetchingEnvironment environment) throws Exception { - final QueryContext context = environment.getContext(); final CreateGlossaryEntityInput input = bindArgument(environment.getArgument("input"), CreateGlossaryEntityInput.class); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java index 93582fb956bd8..1f8c17ee72884 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java @@ -6,7 +6,7 @@ import com.linkedin.datahub.graphql.generated.EntityCountInput; import com.linkedin.datahub.graphql.generated.EntityCountResult; import com.linkedin.datahub.graphql.generated.EntityCountResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolver.java index 0b909dee51374..3e9583824a568 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolver.java @@ -44,7 +44,8 @@ public CompletableFuture rollbackIngestion( return CompletableFuture.supplyAsync( () -> { try { - _entityClient.rollbackIngestion(runId, context.getAuthentication()); + _entityClient.rollbackIngestion( + runId, context.getAuthorizer(), context.getAuthentication()); return true; } catch (Exception e) { throw new RuntimeException("Failed to rollback ingestion execution", e); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java index 577780e53ce86..f5e7cf4d69ce8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/CreateSecretResolver.java @@ -6,11 +6,11 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.UrnUtils; -import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateSecretInput; import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; +import com.linkedin.datahub.graphql.types.ingest.secret.mapper.DataHubSecretValueMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.key.DataHubSecretKey; import com.linkedin.metadata.secret.SecretService; @@ -58,14 +58,15 @@ public CompletableFuture get(final DataFetchingEnvironment environment) } // Create the secret value. - final DataHubSecretValue value = new DataHubSecretValue(); - value.setName(input.getName()); - value.setValue(_secretService.encrypt(input.getValue())); - value.setDescription(input.getDescription(), SetMode.IGNORE_NULL); - value.setCreated( - new AuditStamp() - .setActor(UrnUtils.getUrn(context.getActorUrn())) - .setTime(System.currentTimeMillis())); + final DataHubSecretValue value = + DataHubSecretValueMapper.map( + null, + input.getName(), + _secretService.encrypt(input.getValue()), + input.getDescription(), + new AuditStamp() + .setActor(UrnUtils.getUrn(context.getActorUrn())) + .setTime(System.currentTimeMillis())); final MetadataChangeProposal proposal = buildMetadataChangeProposalWithKey( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolver.java new file mode 100644 index 0000000000000..20a685265b545 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolver.java @@ -0,0 +1,82 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.secret; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn; +import static com.linkedin.metadata.Constants.SECRET_VALUE_ASPECT_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.UpdateSecretInput; +import com.linkedin.datahub.graphql.resolvers.ingest.IngestionAuthUtils; +import com.linkedin.datahub.graphql.types.ingest.secret.mapper.DataHubSecretValueMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.secret.SecretService; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.secret.DataHubSecretValue; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Creates an encrypted DataHub secret. Uses AES symmetric encryption / decryption. Requires the + * MANAGE_SECRETS privilege. + */ +@Slf4j +@RequiredArgsConstructor +public class UpdateSecretResolver implements DataFetcher> { + private final EntityClient entityClient; + private final SecretService secretService; + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final UpdateSecretInput input = + bindArgument(environment.getArgument("input"), UpdateSecretInput.class); + final Urn secretUrn = Urn.createFromString(input.getUrn()); + return CompletableFuture.supplyAsync( + () -> { + if (IngestionAuthUtils.canManageSecrets(context)) { + + try { + EntityResponse response = + entityClient.getV2( + secretUrn.getEntityType(), + secretUrn, + Set.of(SECRET_VALUE_ASPECT_NAME), + context.getAuthentication()); + if (!entityClient.exists(secretUrn, context.getAuthentication()) + || response == null) { + throw new IllegalArgumentException( + String.format("Secret for urn %s doesn't exists!", secretUrn)); + } + + DataHubSecretValue updatedVal = + DataHubSecretValueMapper.map( + response, + input.getName(), + secretService.encrypt(input.getValue()), + input.getDescription(), + null); + + final MetadataChangeProposal proposal = + buildMetadataChangeProposalWithUrn( + secretUrn, SECRET_VALUE_ASPECT_NAME, updatedVal); + return entityClient.ingestProposal(proposal, context.getAuthentication(), false); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Failed to update a secret with urn %s and name %s", + secretUrn, input.getName()), + e); + } + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java new file mode 100644 index 0000000000000..25768da819555 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java @@ -0,0 +1,105 @@ +package com.linkedin.datahub.graphql.resolvers.mutate.util; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.CreateDynamicFormAssignmentInput; +import com.linkedin.datahub.graphql.generated.SubmitFormPromptInput; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.form.FormInfo; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PrimitivePropertyValueArray; +import java.util.Objects; +import javax.annotation.Nonnull; + +public class FormUtils { + + private static final String COMPLETED_FORMS = "completedForms"; + private static final String INCOMPLETE_FORMS = "incompleteForms"; + private static final String VERIFIED_FORMS = "verifiedForms"; + private static final String OWNERS = "owners"; + private static final String COMPLETED_FORMS_COMPLETED_PROMPT_IDS = + "completedFormsCompletedPromptIds"; + private static final String INCOMPLETE_FORMS_COMPLETED_PROMPT_IDS = + "incompleteFormsCompletedPromptIds"; + + private FormUtils() {} + + public static PrimitivePropertyValueArray getStructuredPropertyValuesFromInput( + @Nonnull final SubmitFormPromptInput input) { + final PrimitivePropertyValueArray values = new PrimitivePropertyValueArray(); + + input + .getStructuredPropertyParams() + .getValues() + .forEach( + value -> { + if (value.getStringValue() != null) { + values.add(PrimitivePropertyValue.create(value.getStringValue())); + } else if (value.getNumberValue() != null) { + values.add(PrimitivePropertyValue.create(value.getNumberValue().doubleValue())); + } + }); + + return values; + } + + /** Map a GraphQL CreateDynamicFormAssignmentInput to the GMS DynamicFormAssignment aspect */ + @Nonnull + public static DynamicFormAssignment mapDynamicFormAssignment( + @Nonnull final CreateDynamicFormAssignmentInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final DynamicFormAssignment result = new DynamicFormAssignment(); + final Filter filter = + new Filter() + .setOr(ResolverUtils.buildConjunctiveCriterionArrayWithOr(input.getOrFilters())); + result.setFilter(filter); + return result; + } + + /** + * Creates a Filter where the provided formUrn is either in completedForms or incompleteForms for + * an entity + */ + private static Filter generateCompleteOrIncompleteFilter(@Nonnull final String formUrn) + throws Exception { + final CriterionArray completedFormsAndArray = new CriterionArray(); + final CriterionArray incompleteFormsAndArray = new CriterionArray(); + completedFormsAndArray.add(buildFormCriterion(formUrn, COMPLETED_FORMS)); + incompleteFormsAndArray.add(buildFormCriterion(formUrn, INCOMPLETE_FORMS)); + // need this to be an OR not two ANDs + return new Filter() + .setOr( + new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd(completedFormsAndArray), + new ConjunctiveCriterion().setAnd(incompleteFormsAndArray))); + } + + private static Criterion buildFormCriterion( + @Nonnull final String formUrn, @Nonnull final String field) { + return buildFormCriterion(formUrn, field, false); + } + + private static Criterion buildFormCriterion( + @Nonnull final String formUrn, @Nonnull final String field, final boolean negated) { + return new Criterion() + .setField(field) + .setValue(formUrn) + .setCondition(Condition.EQUAL) + .setNegated(negated); + } + + private static boolean isActorExplicitlyAssigned( + @Nonnull final Urn actorUrn, @Nonnull final FormInfo formInfo) { + return (formInfo.getActors().getUsers() != null + && formInfo.getActors().getUsers().stream().anyMatch(user -> user.equals(actorUrn))) + || (formInfo.getActors().getGroups() != null + && formInfo.getActors().getGroups().stream().anyMatch(group -> group.equals(actorUrn))); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java index 3328eff2bdf45..7bfd166b18a20 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java @@ -9,7 +9,7 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.GetGrantedPrivilegesInput; import com.linkedin.datahub.graphql.generated.Privileges; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.List; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java index ca1e01b45989d..e65666117b4fa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java @@ -15,8 +15,8 @@ import com.linkedin.datahub.graphql.generated.RecommendationRenderType; import com.linkedin.datahub.graphql.generated.RecommendationRequestContext; import com.linkedin.datahub.graphql.generated.SearchParams; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.recommendation.EntityRequestContext; import com.linkedin.metadata.recommendation.RecommendationsService; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 6d23456b76b4f..b54987dc0e9b0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -15,6 +15,7 @@ import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; @@ -36,6 +37,7 @@ public class AggregateAcrossEntitiesResolver private final EntityClient _entityClient; private final ViewService _viewService; + private final FormService _formService; @Override public CompletableFuture get(DataFetchingEnvironment environment) { @@ -58,7 +60,7 @@ public CompletableFuture get(DataFetchingEnvironment environme context.getAuthentication()) : null; - final Filter baseFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); + final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); final SearchFlags searchFlags = mapInputFlags(input.getSearchFlags()); @@ -75,8 +77,8 @@ public CompletableFuture get(DataFetchingEnvironment environme sanitizedQuery, maybeResolvedView != null ? SearchUtils.combineFilters( - baseFilter, maybeResolvedView.getDefinition().getFilter()) - : baseFilter, + inputFilter, maybeResolvedView.getDefinition().getFilter()) + : inputFilter, 0, 0, // 0 entity count because we don't want resolved entities searchFlags, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java index 6a01fa19867ad..f300331ab4bc8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AutoCompleteForMultipleResolver.java @@ -10,9 +10,9 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleInput; import com.linkedin.datahub.graphql.generated.AutoCompleteMultipleResults; import com.linkedin.datahub.graphql.generated.EntityType; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; @@ -66,6 +66,12 @@ public CompletableFuture get(DataFetchingEnvironmen : null; List types = getEntityTypes(input.getTypes(), maybeResolvedView); + types = + types != null + ? types.stream() + .filter(AUTO_COMPLETE_ENTITY_TYPES::contains) + .collect(Collectors.toList()) + : null; if (types != null && types.size() > 0) { return AutocompleteUtils.batchGetAutocompleteResults( types.stream() diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java index e54955e1857f0..1a380781385c3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java @@ -11,9 +11,9 @@ import com.linkedin.datahub.graphql.generated.GetQuickFiltersInput; import com.linkedin.datahub.graphql.generated.GetQuickFiltersResult; import com.linkedin.datahub.graphql.generated.QuickFilter; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.search.AggregationMetadata; import com.linkedin.metadata.search.AggregationMetadataArray; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java index 742d1d170de64..658138ae6e3dc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java @@ -8,9 +8,9 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.ScrollAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.ScrollResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnScrollResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java index adab62c22bb72..0af0a3827b1bb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java @@ -11,8 +11,8 @@ import com.linkedin.datahub.graphql.generated.LineageDirection; import com.linkedin.datahub.graphql.generated.ScrollAcrossLineageInput; import com.linkedin.datahub.graphql.generated.ScrollAcrossLineageResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnScrollAcrossLineageResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index 0f5d2d90ba0c2..2dc5032f2a4eb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -9,9 +9,9 @@ import com.linkedin.datahub.graphql.generated.LineageDirection; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageInput; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchAcrossLineageResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java index 6821423887923..bc177c600beee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java @@ -5,9 +5,9 @@ import com.linkedin.datahub.graphql.generated.SearchInput; import com.linkedin.datahub.graphql.generated.SearchResults; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 6746c30a2edbc..8c45df1b30b26 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -21,8 +21,8 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/PropertyValueResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/PropertyValueResolver.java new file mode 100644 index 0000000000000..cb0d24839056d --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/PropertyValueResolver.java @@ -0,0 +1,25 @@ +package com.linkedin.datahub.graphql.resolvers.type; + +import com.linkedin.datahub.graphql.generated.NumberValue; +import com.linkedin.datahub.graphql.generated.StringValue; +import graphql.TypeResolutionEnvironment; +import graphql.schema.GraphQLObjectType; +import graphql.schema.TypeResolver; + +public class PropertyValueResolver implements TypeResolver { + + public static final String STRING_VALUE = "StringValue"; + public static final String NUMBER_VALUE = "NumberValue"; + + @Override + public GraphQLObjectType getType(TypeResolutionEnvironment env) { + if (env.getObject() instanceof StringValue) { + return env.getSchema().getObjectType(STRING_VALUE); + } else if (env.getObject() instanceof NumberValue) { + return env.getSchema().getObjectType(NUMBER_VALUE); + } else { + throw new RuntimeException( + "Unrecognized object type provided to type resolver, Type:" + env.getObject().toString()); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java index 9da5f915ff31d..3a676f118c1ac 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java @@ -11,8 +11,8 @@ import com.linkedin.datahub.graphql.generated.DataHubViewFilterInput; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.LogicalOperator; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.CriterionArray; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index 4c452af126201..18a082fee95f1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -32,6 +32,7 @@ import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity; import com.linkedin.datahub.graphql.generated.Role; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; import com.linkedin.datahub.graphql.generated.Tag; import com.linkedin.datahub.graphql.generated.Test; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; @@ -192,6 +193,11 @@ public Entity apply(Urn input) { ((OwnershipTypeEntity) partialEntity).setUrn(input.toString()); ((OwnershipTypeEntity) partialEntity).setType(EntityType.CUSTOM_OWNERSHIP_TYPE); } + if (input.getEntityType().equals(STRUCTURED_PROPERTY_ENTITY_NAME)) { + partialEntity = new StructuredPropertyEntity(); + ((StructuredPropertyEntity) partialEntity).setUrn(input.toString()); + ((StructuredPropertyEntity) partialEntity).setType(EntityType.STRUCTURED_PROPERTY); + } return partialEntity; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index badb24810c82b..fd31e1d394a92 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -88,6 +88,8 @@ public class DatasetType DATA_PRODUCTS_ASPECT_NAME, BROWSE_PATHS_V2_ASPECT_NAME, ACCESS_DATASET_ASPECT_NAME, + STRUCTURED_PROPERTIES_ASPECT_NAME, + FORMS_ASPECT_NAME, SUB_TYPES_ASPECT_NAME); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetFilterMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetFilterMapper.java new file mode 100644 index 0000000000000..7e5372268170b --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetFilterMapper.java @@ -0,0 +1,24 @@ +package com.linkedin.datahub.graphql.types.dataset.mappers; + +import com.linkedin.datahub.graphql.generated.DatasetFilter; +import com.linkedin.datahub.graphql.generated.DatasetFilterType; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import javax.annotation.Nonnull; + +public class DatasetFilterMapper + implements ModelMapper { + + public static final DatasetFilterMapper INSTANCE = new DatasetFilterMapper(); + + public static DatasetFilter map(@Nonnull final com.linkedin.dataset.DatasetFilter metadata) { + return INSTANCE.apply(metadata); + } + + @Override + public DatasetFilter apply(@Nonnull final com.linkedin.dataset.DatasetFilter input) { + final DatasetFilter result = new DatasetFilter(); + result.setType(DatasetFilterType.valueOf(input.getType().name())); + result.setSql(input.getSql()); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 7fa1decdf7f55..163e8b9288d87 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -7,6 +7,7 @@ import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.Embed; +import com.linkedin.common.Forms; import com.linkedin.common.GlobalTags; import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InstitutionalMemory; @@ -38,9 +39,11 @@ import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.common.mappers.util.SystemMetadataUtils; import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; +import com.linkedin.datahub.graphql.types.form.FormsMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.dataset.DatasetDeprecation; import com.linkedin.dataset.DatasetProperties; @@ -53,6 +56,7 @@ import com.linkedin.metadata.key.DatasetKey; import com.linkedin.schema.EditableSchemaMetadata; import com.linkedin.schema.SchemaMetadata; +import com.linkedin.structured.StructuredProperties; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @@ -151,6 +155,15 @@ public Dataset apply(@Nonnull final EntityResponse entityResponse) { ACCESS_DATASET_ASPECT_NAME, ((dataset, dataMap) -> dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn)))); + mappingHelper.mapToResult( + STRUCTURED_PROPERTIES_ASPECT_NAME, + ((dataset, dataMap) -> + dataset.setStructuredProperties( + StructuredPropertiesMapper.map(new StructuredProperties(dataMap))))); + mappingHelper.mapToResult( + FORMS_ASPECT_NAME, + ((dataset, dataMap) -> + dataset.setForms(FormsMapper.map(new Forms(dataMap), entityUrn.toString())))); mappingHelper.mapToResult( SUB_TYPES_ASPECT_NAME, (dashboard, dataMap) -> dashboard.setSubTypes(SubTypesMapper.map(new SubTypes(dataMap)))); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java index edc9baf4ba9c5..e0a74d351125f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java @@ -1,10 +1,13 @@ package com.linkedin.datahub.graphql.types.dataset.mappers; import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SchemaField; import com.linkedin.datahub.graphql.generated.SchemaFieldDataType; +import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.metadata.utils.SchemaFieldUtils; import javax.annotation.Nonnull; public class SchemaFieldMapper { @@ -37,6 +40,7 @@ public SchemaField apply( result.setIsPartOfKey(input.isIsPartOfKey()); result.setIsPartitioningKey(input.isIsPartitioningKey()); result.setJsonProps(input.getJsonProps()); + result.setSchemaFieldEntity(this.createSchemaFieldEntity(input, entityUrn)); return result; } @@ -75,4 +79,14 @@ private SchemaFieldDataType mapSchemaFieldDataType( "Unrecognized SchemaFieldDataType provided %s", type.memberType().toString())); } } + + private SchemaFieldEntity createSchemaFieldEntity( + @Nonnull final com.linkedin.schema.SchemaField input, @Nonnull Urn entityUrn) { + SchemaFieldEntity schemaFieldEntity = new SchemaFieldEntity(); + schemaFieldEntity.setUrn( + SchemaFieldUtils.generateSchemaFieldUrn(entityUrn.toString(), input.getFieldPath()) + .toString()); + schemaFieldEntity.setType(EntityType.SCHEMA_FIELD); + return schemaFieldEntity; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java index 31381073a16dd..e550280a6c2db 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaMetadataMapper.java @@ -18,6 +18,11 @@ public static com.linkedin.datahub.graphql.generated.SchemaMetadata map( public com.linkedin.datahub.graphql.generated.SchemaMetadata apply( @Nonnull final EnvelopedAspect aspect, @Nonnull final Urn entityUrn) { final SchemaMetadata input = new SchemaMetadata(aspect.getValue().data()); + return apply(input, entityUrn, aspect.getVersion()); + } + + public com.linkedin.datahub.graphql.generated.SchemaMetadata apply( + @Nonnull final SchemaMetadata input, final Urn entityUrn, final long version) { final com.linkedin.datahub.graphql.generated.SchemaMetadata result = new com.linkedin.datahub.graphql.generated.SchemaMetadata(); @@ -35,7 +40,7 @@ public com.linkedin.datahub.graphql.generated.SchemaMetadata apply( .map(field -> SchemaFieldMapper.map(field, entityUrn)) .collect(Collectors.toList())); result.setPlatformSchema(PlatformSchemaMapper.map(input.getPlatformSchema())); - result.setAspectVersion(aspect.getVersion()); + result.setAspectVersion(version); if (input.hasForeignKeys()) { result.setForeignKeys( input.getForeignKeys().stream() diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeEntityMapper.java new file mode 100644 index 0000000000000..612644ae2dbb2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeEntityMapper.java @@ -0,0 +1,51 @@ +package com.linkedin.datahub.graphql.types.datatype; + +import static com.linkedin.metadata.Constants.DATA_TYPE_INFO_ASPECT_NAME; + +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.DataTypeEntity; +import com.linkedin.datahub.graphql.generated.DataTypeInfo; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import javax.annotation.Nonnull; + +public class DataTypeEntityMapper implements ModelMapper { + + public static final DataTypeEntityMapper INSTANCE = new DataTypeEntityMapper(); + + public static DataTypeEntity map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + @Override + public DataTypeEntity apply(@Nonnull final EntityResponse entityResponse) { + final DataTypeEntity result = new DataTypeEntity(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.DATA_TYPE); + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(DATA_TYPE_INFO_ASPECT_NAME, this::mapDataTypeInfo); + + // Set the standard Type ENUM for the data type. + if (result.getInfo() != null) { + result.getInfo().setType(DataTypeUrnMapper.getType(entityResponse.getUrn().toString())); + } + return mappingHelper.getResult(); + } + + private void mapDataTypeInfo(@Nonnull DataTypeEntity dataType, @Nonnull DataMap dataMap) { + com.linkedin.datatype.DataTypeInfo gmsInfo = new com.linkedin.datatype.DataTypeInfo(dataMap); + DataTypeInfo info = new DataTypeInfo(); + info.setQualifiedName(gmsInfo.getQualifiedName()); + if (gmsInfo.getDisplayName() != null) { + info.setDisplayName(gmsInfo.getDisplayName()); + } + if (gmsInfo.getDescription() != null) { + info.setDescription(gmsInfo.getDescription()); + } + dataType.setInfo(info); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeType.java new file mode 100644 index 0000000000000..5ea1680546ce6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeType.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.types.datatype; + +import static com.linkedin.metadata.Constants.DATA_TYPE_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_TYPE_INFO_ASPECT_NAME; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataTypeEntity; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class DataTypeType + implements com.linkedin.datahub.graphql.types.EntityType { + + public static final Set ASPECTS_TO_FETCH = ImmutableSet.of(DATA_TYPE_INFO_ASPECT_NAME); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.DATA_TYPE; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return DataTypeEntity.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List dataTypeUrns = urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2( + DATA_TYPE_ENTITY_NAME, + new HashSet<>(dataTypeUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : dataTypeUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(DataTypeEntityMapper.map(gmsResult)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load data type entities", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeUrnMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeUrnMapper.java new file mode 100644 index 0000000000000..ec71cd63a70d5 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datatype/DataTypeUrnMapper.java @@ -0,0 +1,40 @@ +package com.linkedin.datahub.graphql.types.datatype; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.datahub.graphql.generated.StdDataType; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class DataTypeUrnMapper { + + static final Map DATA_TYPE_ENUM_TO_URN = + ImmutableMap.builder() + .put(StdDataType.STRING, "urn:li:dataType:datahub.string") + .put(StdDataType.NUMBER, "urn:li:dataType:datahub.number") + .put(StdDataType.URN, "urn:li:dataType:datahub.urn") + .put(StdDataType.RICH_TEXT, "urn:li:dataType:datahub.rich_text") + .put(StdDataType.DATE, "urn:li:dataType:datahub.date") + .build(); + + private static final Map URN_TO_DATA_TYPE_ENUM = + DATA_TYPE_ENUM_TO_URN.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + private DataTypeUrnMapper() {} + + public static StdDataType getType(String dataTypeUrn) { + if (!URN_TO_DATA_TYPE_ENUM.containsKey(dataTypeUrn)) { + return StdDataType.OTHER; + } + return URN_TO_DATA_TYPE_ENUM.get(dataTypeUrn); + } + + @Nonnull + public static String getUrn(StdDataType dataType) { + if (!DATA_TYPE_ENUM_TO_URN.containsKey(dataType)) { + throw new IllegalArgumentException("Unknown data type: " + dataType); + } + return DATA_TYPE_ENUM_TO_URN.get(dataType); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeEntityMapper.java new file mode 100644 index 0000000000000..b942ff2325bf7 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeEntityMapper.java @@ -0,0 +1,54 @@ +package com.linkedin.datahub.graphql.types.entitytype; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.EntityTypeEntity; +import com.linkedin.datahub.graphql.generated.EntityTypeInfo; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import javax.annotation.Nonnull; + +public class EntityTypeEntityMapper implements ModelMapper { + + public static final EntityTypeEntityMapper INSTANCE = new EntityTypeEntityMapper(); + + public static EntityTypeEntity map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + @Override + public EntityTypeEntity apply(@Nonnull final EntityResponse entityResponse) { + final EntityTypeEntity result = new EntityTypeEntity(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.ENTITY_TYPE); + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(ENTITY_TYPE_INFO_ASPECT_NAME, this::mapEntityTypeInfo); + + // Set the standard Type ENUM for the entity type. + if (result.getInfo() != null) { + result + .getInfo() + .setType(EntityTypeUrnMapper.getEntityType(entityResponse.getUrn().toString())); + } + return mappingHelper.getResult(); + } + + private void mapEntityTypeInfo(@Nonnull EntityTypeEntity entityType, @Nonnull DataMap dataMap) { + com.linkedin.entitytype.EntityTypeInfo gmsInfo = + new com.linkedin.entitytype.EntityTypeInfo(dataMap); + EntityTypeInfo info = new EntityTypeInfo(); + info.setQualifiedName(gmsInfo.getQualifiedName()); + if (gmsInfo.getDisplayName() != null) { + info.setDisplayName(gmsInfo.getDisplayName()); + } + if (gmsInfo.getDescription() != null) { + info.setDescription(gmsInfo.getDescription()); + } + entityType.setInfo(info); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java similarity index 91% rename from datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java rename to datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java index aba781f9e1dc7..23e793782e8dc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java @@ -1,4 +1,4 @@ -package com.linkedin.datahub.graphql.resolvers; +package com.linkedin.datahub.graphql.types.entitytype; import com.google.common.collect.ImmutableMap; import com.linkedin.datahub.graphql.generated.EntityType; @@ -17,7 +17,6 @@ public class EntityTypeMapper { ImmutableMap.builder() .put(EntityType.DATASET, "dataset") .put(EntityType.ROLE, "role") - .put(EntityType.ASSERTION, Constants.ASSERTION_ENTITY_NAME) .put(EntityType.CORP_USER, "corpuser") .put(EntityType.CORP_GROUP, "corpGroup") .put(EntityType.DATA_PLATFORM, "dataPlatform") @@ -41,6 +40,9 @@ public class EntityTypeMapper { .put(EntityType.TEST, "test") .put(EntityType.DATAHUB_VIEW, Constants.DATAHUB_VIEW_ENTITY_NAME) .put(EntityType.DATA_PRODUCT, Constants.DATA_PRODUCT_ENTITY_NAME) + .put(EntityType.SCHEMA_FIELD, "schemaField") + .put(EntityType.STRUCTURED_PROPERTY, Constants.STRUCTURED_PROPERTY_ENTITY_NAME) + .put(EntityType.ASSERTION, Constants.ASSERTION_ENTITY_NAME) .build(); private static final Map ENTITY_NAME_TO_TYPE = @@ -52,7 +54,7 @@ private EntityTypeMapper() {} public static EntityType getType(String name) { String lowercaseName = name.toLowerCase(); if (!ENTITY_NAME_TO_TYPE.containsKey(lowercaseName)) { - throw new IllegalArgumentException("Unknown entity name: " + name); + return EntityType.OTHER; } return ENTITY_NAME_TO_TYPE.get(lowercaseName); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeType.java new file mode 100644 index 0000000000000..aa5dfc13ea757 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeType.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.types.entitytype; + +import static com.linkedin.metadata.Constants.*; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.EntityTypeEntity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class EntityTypeType + implements com.linkedin.datahub.graphql.types.EntityType { + + public static final Set ASPECTS_TO_FETCH = ImmutableSet.of(ENTITY_TYPE_INFO_ASPECT_NAME); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.ENTITY_TYPE; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return EntityTypeEntity.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List entityTypeUrns = + urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2( + ENTITY_TYPE_ENTITY_NAME, + new HashSet<>(entityTypeUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : entityTypeUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(EntityTypeEntityMapper.map(gmsResult)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load entity type entities", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeUrnMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeUrnMapper.java new file mode 100644 index 0000000000000..9e9bf86e5fe7f --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeUrnMapper.java @@ -0,0 +1,85 @@ +package com.linkedin.datahub.graphql.types.entitytype; + +import com.google.common.collect.ImmutableMap; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.metadata.Constants; +import java.util.Map; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +/** + * In this class we statically map "well-supported" entity types into a more usable Enum class + * served by our GraphQL API. + * + *

When we add new entity types / entity urns, we MAY NEED to update this. + * + *

Note that we currently do not support mapping entities that fall outside of this set. If you + * try to map an entity type without a corresponding enum symbol, the mapping WILL FAIL. + */ +public class EntityTypeUrnMapper { + + static final Map ENTITY_NAME_TO_ENTITY_TYPE_URN = + ImmutableMap.builder() + .put(Constants.DATASET_ENTITY_NAME, "urn:li:entityType:datahub.dataset") + .put(Constants.ROLE_ENTITY_NAME, "urn:li:entityType:datahub.role") + .put(Constants.CORP_USER_ENTITY_NAME, "urn:li:entityType:datahub.corpuser") + .put(Constants.CORP_GROUP_ENTITY_NAME, "urn:li:entityType:datahub.corpGroup") + .put(Constants.DATA_PLATFORM_ENTITY_NAME, "urn:li:entityType:datahub.dataPlatform") + .put(Constants.DASHBOARD_ENTITY_NAME, "urn:li:entityType:datahub.dashboard") + .put(Constants.CHART_ENTITY_NAME, "urn:li:entityType:datahub.chart") + .put(Constants.TAG_ENTITY_NAME, "urn:li:entityType:datahub.tag") + .put(Constants.DATA_FLOW_ENTITY_NAME, "urn:li:entityType:datahub.dataFlow") + .put(Constants.DATA_JOB_ENTITY_NAME, "urn:li:entityType:datahub.dataJob") + .put(Constants.GLOSSARY_TERM_ENTITY_NAME, "urn:li:entityType:datahub.glossaryTerm") + .put(Constants.GLOSSARY_NODE_ENTITY_NAME, "urn:li:entityType:datahub.glossaryNode") + .put(Constants.ML_MODEL_ENTITY_NAME, "urn:li:entityType:datahub.mlModel") + .put(Constants.ML_MODEL_GROUP_ENTITY_NAME, "urn:li:entityType:datahub.mlModelGroup") + .put(Constants.ML_FEATURE_TABLE_ENTITY_NAME, "urn:li:entityType:datahub.mlFeatureTable") + .put(Constants.ML_FEATURE_ENTITY_NAME, "urn:li:entityType:datahub.mlFeature") + .put(Constants.ML_PRIMARY_KEY_ENTITY_NAME, "urn:li:entityType:datahub.mlPrimaryKey") + .put(Constants.CONTAINER_ENTITY_NAME, "urn:li:entityType:datahub.container") + .put(Constants.DOMAIN_ENTITY_NAME, "urn:li:entityType:datahub.domain") + .put(Constants.NOTEBOOK_ENTITY_NAME, "urn:li:entityType:datahub.notebook") + .put( + Constants.DATA_PLATFORM_INSTANCE_ENTITY_NAME, + "urn:li:entityType:datahub.dataPlatformInstance") + .put(Constants.TEST_ENTITY_NAME, "urn:li:entityType:datahub.test") + .put(Constants.DATAHUB_VIEW_ENTITY_NAME, "urn:li:entityType:datahub.dataHubView") + .put(Constants.DATA_PRODUCT_ENTITY_NAME, "urn:li:entityType:datahub.dataProduct") + .put(Constants.ASSERTION_ENTITY_NAME, "urn:li:entityType:datahub.assertion") + .put(Constants.SCHEMA_FIELD_ENTITY_NAME, "urn:li:entityType:datahub.schemaField") + .build(); + + private static final Map ENTITY_TYPE_URN_TO_NAME = + ENTITY_NAME_TO_ENTITY_TYPE_URN.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + private EntityTypeUrnMapper() {} + + public static String getName(String entityTypeUrn) { + if (!ENTITY_TYPE_URN_TO_NAME.containsKey(entityTypeUrn)) { + throw new IllegalArgumentException("Unknown entityTypeUrn: " + entityTypeUrn); + } + return ENTITY_TYPE_URN_TO_NAME.get(entityTypeUrn); + } + + /* + * Takes in a entityTypeUrn and returns a GraphQL EntityType by first mapping + * the urn to the entity name, and then mapping the entity name to EntityType. + */ + public static EntityType getEntityType(String entityTypeUrn) { + if (!ENTITY_TYPE_URN_TO_NAME.containsKey(entityTypeUrn)) { + throw new IllegalArgumentException("Unknown entityTypeUrn: " + entityTypeUrn); + } + final String entityName = ENTITY_TYPE_URN_TO_NAME.get(entityTypeUrn); + return EntityTypeMapper.getType(entityName); + } + + @Nonnull + public static String getEntityTypeUrn(String name) { + if (!ENTITY_NAME_TO_ENTITY_TYPE_URN.containsKey(name)) { + throw new IllegalArgumentException("Unknown entity name: " + name); + } + return ENTITY_NAME_TO_ENTITY_TYPE_URN.get(name); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormMapper.java new file mode 100644 index 0000000000000..a0ddd4a5883d2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormMapper.java @@ -0,0 +1,129 @@ +package com.linkedin.datahub.graphql.types.form; + +import static com.linkedin.metadata.Constants.FORM_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; + +import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.CorpGroup; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormActorAssignment; +import com.linkedin.datahub.graphql.generated.FormInfo; +import com.linkedin.datahub.graphql.generated.FormPrompt; +import com.linkedin.datahub.graphql.generated.FormPromptType; +import com.linkedin.datahub.graphql.generated.FormType; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertyParams; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class FormMapper implements ModelMapper { + + public static final FormMapper INSTANCE = new FormMapper(); + + public static Form map(@Nonnull final EntityResponse form) { + return INSTANCE.apply(form); + } + + public Form apply(@Nonnull final EntityResponse entityResponse) { + Form result = new Form(); + Urn entityUrn = entityResponse.getUrn(); + result.setUrn(entityUrn.toString()); + result.setType(EntityType.FORM); + + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper

mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(FORM_INFO_ASPECT_NAME, this::mapFormInfo); + mappingHelper.mapToResult( + OWNERSHIP_ASPECT_NAME, + (form, dataMap) -> + form.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); + + return mappingHelper.getResult(); + } + + private void mapFormInfo(@Nonnull Form form, @Nonnull DataMap dataMap) { + com.linkedin.form.FormInfo gmsFormInfo = new com.linkedin.form.FormInfo(dataMap); + FormInfo formInfo = new FormInfo(); + formInfo.setName(gmsFormInfo.getName()); + formInfo.setType(FormType.valueOf(gmsFormInfo.getType().toString())); + if (gmsFormInfo.hasDescription()) { + formInfo.setDescription(gmsFormInfo.getDescription()); + } + formInfo.setPrompts(this.mapFormPrompts(gmsFormInfo, form.getUrn())); + formInfo.setActors(mapFormActors(gmsFormInfo.getActors())); + form.setInfo(formInfo); + } + + private List mapFormPrompts( + @Nonnull com.linkedin.form.FormInfo gmsFormInfo, @Nonnull String formUrn) { + List formPrompts = new ArrayList<>(); + if (gmsFormInfo.hasPrompts()) { + gmsFormInfo + .getPrompts() + .forEach(FormPrompt -> formPrompts.add(mapFormPrompt(FormPrompt, formUrn))); + } + return formPrompts; + } + + private FormPrompt mapFormPrompt( + @Nonnull com.linkedin.form.FormPrompt gmsFormPrompt, @Nonnull String formUrn) { + final FormPrompt formPrompt = new FormPrompt(); + formPrompt.setId(gmsFormPrompt.getId()); + formPrompt.setTitle(gmsFormPrompt.getTitle()); + formPrompt.setType(FormPromptType.valueOf(gmsFormPrompt.getType().toString())); + formPrompt.setRequired(gmsFormPrompt.isRequired()); + formPrompt.setFormUrn(formUrn); + if (gmsFormPrompt.hasDescription()) { + formPrompt.setDescription(gmsFormPrompt.getDescription()); + } + + if (gmsFormPrompt.hasStructuredPropertyParams()) { + final StructuredPropertyParams params = new StructuredPropertyParams(); + final Urn structuredPropUrn = gmsFormPrompt.getStructuredPropertyParams().getUrn(); + final StructuredPropertyEntity structuredProp = new StructuredPropertyEntity(); + structuredProp.setUrn(structuredPropUrn.toString()); + structuredProp.setType(EntityType.STRUCTURED_PROPERTY); + params.setStructuredProperty(structuredProp); + formPrompt.setStructuredPropertyParams(params); + } + + return formPrompt; + } + + private FormActorAssignment mapFormActors(com.linkedin.form.FormActorAssignment gmsFormActors) { + FormActorAssignment result = new FormActorAssignment(); + result.setOwners(gmsFormActors.isOwners()); + if (gmsFormActors.hasUsers()) { + result.setUsers( + gmsFormActors.getUsers().stream().map(this::mapUser).collect(Collectors.toList())); + } + if (gmsFormActors.hasGroups()) { + result.setGroups( + gmsFormActors.getGroups().stream().map(this::mapGroup).collect(Collectors.toList())); + } + return result; + } + + private CorpUser mapUser(Urn userUrn) { + CorpUser user = new CorpUser(); + user.setUrn(userUrn.toString()); + return user; + } + + private CorpGroup mapGroup(Urn groupUrn) { + CorpGroup group = new CorpGroup(); + group.setUrn(groupUrn.toString()); + return group; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormType.java new file mode 100644 index 0000000000000..8a09cee353cc9 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormType.java @@ -0,0 +1,76 @@ +package com.linkedin.datahub.graphql.types.form; + +import static com.linkedin.metadata.Constants.FORM_ENTITY_NAME; +import static com.linkedin.metadata.Constants.FORM_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class FormType implements com.linkedin.datahub.graphql.types.EntityType { + public static final Set ASPECTS_TO_FETCH = + ImmutableSet.of(FORM_INFO_ASPECT_NAME, OWNERSHIP_ASPECT_NAME); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.FORM; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return Form.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List formUrns = urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2( + FORM_ENTITY_NAME, + new HashSet<>(formUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : formUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult().data(FormMapper.map(gmsResult)).build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Forms", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormsMapper.java new file mode 100644 index 0000000000000..43665b37b9ee8 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/form/FormsMapper.java @@ -0,0 +1,133 @@ +package com.linkedin.datahub.graphql.types.form; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.FieldFormPromptAssociationArray; +import com.linkedin.common.FormPromptAssociationArray; +import com.linkedin.common.Forms; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FieldFormPromptAssociation; +import com.linkedin.datahub.graphql.generated.Form; +import com.linkedin.datahub.graphql.generated.FormAssociation; +import com.linkedin.datahub.graphql.generated.FormPromptAssociation; +import com.linkedin.datahub.graphql.generated.FormPromptFieldAssociations; +import com.linkedin.datahub.graphql.generated.FormVerificationAssociation; +import com.linkedin.datahub.graphql.generated.ResolvedAuditStamp; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nonnull; + +public class FormsMapper { + + public static final FormsMapper INSTANCE = new FormsMapper(); + + public static com.linkedin.datahub.graphql.generated.Forms map( + @Nonnull final Forms forms, @Nonnull final String entityUrn) { + return INSTANCE.apply(forms, entityUrn); + } + + public com.linkedin.datahub.graphql.generated.Forms apply( + @Nonnull final Forms forms, @Nonnull final String entityUrn) { + final List incompleteForms = new ArrayList<>(); + forms + .getIncompleteForms() + .forEach( + formAssociation -> + incompleteForms.add(this.mapFormAssociation(formAssociation, entityUrn))); + final List completeForms = new ArrayList<>(); + forms + .getCompletedForms() + .forEach( + formAssociation -> + completeForms.add(this.mapFormAssociation(formAssociation, entityUrn))); + final List verifications = new ArrayList<>(); + forms + .getVerifications() + .forEach( + verificationAssociation -> + verifications.add(this.mapVerificationAssociation(verificationAssociation))); + + return new com.linkedin.datahub.graphql.generated.Forms( + incompleteForms, completeForms, verifications); + } + + private FormAssociation mapFormAssociation( + @Nonnull final com.linkedin.common.FormAssociation association, + @Nonnull final String entityUrn) { + FormAssociation result = new FormAssociation(); + result.setForm( + Form.builder().setType(EntityType.FORM).setUrn(association.getUrn().toString()).build()); + result.setAssociatedUrn(entityUrn); + result.setCompletedPrompts(this.mapPrompts(association.getCompletedPrompts())); + result.setIncompletePrompts(this.mapPrompts(association.getIncompletePrompts())); + return result; + } + + private FormVerificationAssociation mapVerificationAssociation( + @Nonnull final com.linkedin.common.FormVerificationAssociation verificationAssociation) { + FormVerificationAssociation result = new FormVerificationAssociation(); + result.setForm( + Form.builder() + .setType(EntityType.FORM) + .setUrn(verificationAssociation.getForm().toString()) + .build()); + if (verificationAssociation.hasLastModified()) { + result.setLastModified(createAuditStamp(verificationAssociation.getLastModified())); + } + return result; + } + + private List mapPrompts( + @Nonnull final FormPromptAssociationArray promptAssociations) { + List result = new ArrayList<>(); + promptAssociations.forEach( + promptAssociation -> { + FormPromptAssociation association = new FormPromptAssociation(); + association.setId(promptAssociation.getId()); + association.setLastModified(createAuditStamp(promptAssociation.getLastModified())); + if (promptAssociation.hasFieldAssociations()) { + association.setFieldAssociations( + mapFieldAssociations(promptAssociation.getFieldAssociations())); + } + result.add(association); + }); + return result; + } + + private List mapFieldPrompts( + @Nonnull final FieldFormPromptAssociationArray fieldPromptAssociations) { + List result = new ArrayList<>(); + fieldPromptAssociations.forEach( + fieldFormPromptAssociation -> { + FieldFormPromptAssociation association = new FieldFormPromptAssociation(); + association.setFieldPath(fieldFormPromptAssociation.getFieldPath()); + association.setLastModified( + createAuditStamp(fieldFormPromptAssociation.getLastModified())); + result.add(association); + }); + return result; + } + + private FormPromptFieldAssociations mapFieldAssociations( + com.linkedin.common.FormPromptFieldAssociations associationsObj) { + final FormPromptFieldAssociations fieldAssociations = new FormPromptFieldAssociations(); + if (associationsObj.hasCompletedFieldPrompts()) { + fieldAssociations.setCompletedFieldPrompts( + this.mapFieldPrompts(associationsObj.getCompletedFieldPrompts())); + } + if (associationsObj.hasIncompleteFieldPrompts()) { + fieldAssociations.setIncompleteFieldPrompts( + this.mapFieldPrompts(associationsObj.getIncompleteFieldPrompts())); + } + return fieldAssociations; + } + + private ResolvedAuditStamp createAuditStamp(AuditStamp auditStamp) { + final ResolvedAuditStamp resolvedAuditStamp = new ResolvedAuditStamp(); + final CorpUser emptyCreatedUser = new CorpUser(); + emptyCreatedUser.setUrn(auditStamp.getActor().toString()); + resolvedAuditStamp.setActor(emptyCreatedUser); + resolvedAuditStamp.setTime(auditStamp.getTime()); + return resolvedAuditStamp; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java index 901361eb0b2be..31c8cec8cb5fa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java @@ -8,6 +8,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.GlossaryNode; import com.linkedin.datahub.graphql.generated.GlossaryNodeProperties; +import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; @@ -36,7 +37,8 @@ public GlossaryNode apply(@Nonnull final EntityResponse entityResponse) { MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult( GLOSSARY_NODE_INFO_ASPECT_NAME, - (glossaryNode, dataMap) -> glossaryNode.setProperties(mapGlossaryNodeProperties(dataMap))); + (glossaryNode, dataMap) -> + glossaryNode.setProperties(mapGlossaryNodeProperties(dataMap, entityUrn))); mappingHelper.mapToResult(GLOSSARY_NODE_KEY_ASPECT_NAME, this::mapGlossaryNodeKey); mappingHelper.mapToResult( OWNERSHIP_ASPECT_NAME, @@ -46,13 +48,18 @@ public GlossaryNode apply(@Nonnull final EntityResponse entityResponse) { return mappingHelper.getResult(); } - private GlossaryNodeProperties mapGlossaryNodeProperties(@Nonnull DataMap dataMap) { + private GlossaryNodeProperties mapGlossaryNodeProperties( + @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) { GlossaryNodeInfo glossaryNodeInfo = new GlossaryNodeInfo(dataMap); GlossaryNodeProperties result = new GlossaryNodeProperties(); result.setDescription(glossaryNodeInfo.getDefinition()); if (glossaryNodeInfo.hasName()) { result.setName(glossaryNodeInfo.getName()); } + if (glossaryNodeInfo.hasCustomProperties()) { + result.setCustomProperties( + CustomPropertiesMapper.map(glossaryNodeInfo.getCustomProperties(), entityUrn)); + } return result; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java index 8494eace22244..68475a2599158 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermsMapper.java @@ -2,6 +2,7 @@ import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.GlossaryTerm; import com.linkedin.datahub.graphql.generated.GlossaryTerms; @@ -46,7 +47,15 @@ private com.linkedin.datahub.graphql.generated.GlossaryTermAssociation mapGlossa resultGlossaryTerm.setName( GlossaryTermUtils.getGlossaryTermName(input.getUrn().getNameEntity())); result.setTerm(resultGlossaryTerm); - result.setAssociatedUrn(entityUrn.toString()); + if (input.hasActor()) { + CorpUser actor = new CorpUser(); + actor.setUrn(input.getActor().toString()); + actor.setType(EntityType.CORP_USER); + result.setActor(actor); + } + if (entityUrn != null) { + result.setAssociatedUrn(entityUrn.toString()); + } return result; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ingest/secret/mapper/DataHubSecretValueMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ingest/secret/mapper/DataHubSecretValueMapper.java new file mode 100644 index 0000000000000..2c5e84dad28c2 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/ingest/secret/mapper/DataHubSecretValueMapper.java @@ -0,0 +1,55 @@ +package com.linkedin.datahub.graphql.types.ingest.secret.mapper; + +import static com.linkedin.metadata.Constants.SECRET_VALUE_ASPECT_NAME; + +import com.linkedin.common.AuditStamp; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; +import com.linkedin.entity.EntityResponse; +import com.linkedin.secret.DataHubSecretValue; +import java.util.Objects; +import javax.annotation.Nonnull; + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + *

To be replaced by auto-generated mappers implementations + */ +public class DataHubSecretValueMapper { + + public static final DataHubSecretValueMapper INSTANCE = new DataHubSecretValueMapper(); + + public static DataHubSecretValue map( + EntityResponse fromSecret, + @Nonnull final String name, + @Nonnull final String value, + String description, + AuditStamp auditStamp) { + return INSTANCE.apply(fromSecret, name, value, description, auditStamp); + } + + public DataHubSecretValue apply( + EntityResponse existingSecret, + @Nonnull final String name, + @Nonnull final String value, + String description, + AuditStamp auditStamp) { + final DataHubSecretValue result; + if (Objects.nonNull(existingSecret)) { + result = + new DataHubSecretValue( + existingSecret.getAspects().get(SECRET_VALUE_ASPECT_NAME).getValue().data()); + } else { + result = new DataHubSecretValue(); + } + + result.setName(name); + result.setValue(value); + result.setDescription(description, SetMode.IGNORE_NULL); + if (Objects.nonNull(auditStamp)) { + result.setCreated(auditStamp); + } + + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 7c7dab2e02472..b5733626468d6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -9,8 +9,8 @@ import com.linkedin.datahub.graphql.generated.MatchedField; import com.linkedin.datahub.graphql.generated.SearchResult; import com.linkedin.datahub.graphql.generated.SearchSuggestion; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.utils.SearchUtils; import java.net.URISyntaxException; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java new file mode 100644 index 0000000000000..254a1ed1767f1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java @@ -0,0 +1,54 @@ +package com.linkedin.datahub.graphql.types.schemafield; + +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.structured.StructuredProperties; +import javax.annotation.Nonnull; + +public class SchemaFieldMapper implements ModelMapper { + + public static final SchemaFieldMapper INSTANCE = new SchemaFieldMapper(); + + public static SchemaFieldEntity map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + @Override + public SchemaFieldEntity apply(@Nonnull final EntityResponse entityResponse) { + Urn entityUrn = entityResponse.getUrn(); + final SchemaFieldEntity result = this.mapSchemaFieldUrn(entityUrn); + + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult( + STRUCTURED_PROPERTIES_ASPECT_NAME, + ((schemaField, dataMap) -> + schemaField.setStructuredProperties( + StructuredPropertiesMapper.map(new StructuredProperties(dataMap))))); + + return result; + } + + private SchemaFieldEntity mapSchemaFieldUrn(Urn urn) { + try { + SchemaFieldEntity result = new SchemaFieldEntity(); + result.setUrn(urn.toString()); + result.setType(EntityType.SCHEMA_FIELD); + result.setFieldPath(urn.getEntityKey().get(1)); + Urn parentUrn = Urn.createFromString(urn.getEntityKey().get(0)); + result.setParent(UrnToEntityMapper.map(parentUrn)); + return result; + } catch (Exception e) { + throw new RuntimeException("Failed to load schemaField entity", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java index b543a40cbac41..9f14bf52733ea 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldType.java @@ -1,22 +1,40 @@ package com.linkedin.datahub.graphql.types.schemafield; +import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; + +import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.featureflags.FeatureFlags; import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SchemaFieldEntity; -import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +@RequiredArgsConstructor public class SchemaFieldType implements com.linkedin.datahub.graphql.types.EntityType { - public SchemaFieldType() {} + public static final Set ASPECTS_TO_FETCH = + ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME); + + private final EntityClient _entityClient; + private final FeatureFlags _featureFlags; @Override public EntityType type() { @@ -40,29 +58,41 @@ public List> batchLoad( urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); try { - return schemaFieldUrns.stream() - .map(this::mapSchemaFieldUrn) + Map entities = new HashMap<>(); + if (_featureFlags.isSchemaFieldEntityFetchEnabled()) { + entities = + _entityClient.batchGetV2( + SCHEMA_FIELD_ENTITY_NAME, + new HashSet<>(schemaFieldUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + } + + final List gmsResults = new ArrayList<>(); + for (Urn urn : schemaFieldUrns) { + if (_featureFlags.isSchemaFieldEntityFetchEnabled()) { + gmsResults.add(entities.getOrDefault(urn, null)); + } else { + gmsResults.add( + new EntityResponse() + .setUrn(urn) + .setAspects(new EnvelopedAspectMap()) + .setEntityName(urn.getEntityType())); + } + } + + return gmsResults.stream() .map( - schemaFieldEntity -> - DataFetcherResult.newResult().data(schemaFieldEntity).build()) + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(SchemaFieldMapper.map(gmsResult)) + .build()) .collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException("Failed to load schemaField entity", e); } } - - private SchemaFieldEntity mapSchemaFieldUrn(Urn urn) { - try { - SchemaFieldEntity result = new SchemaFieldEntity(); - result.setUrn(urn.toString()); - result.setType(EntityType.SCHEMA_FIELD); - result.setFieldPath(urn.getEntityKey().get(1)); - Urn parentUrn = Urn.createFromString(urn.getEntityKey().get(0)); - result.setParent(UrnToEntityMapper.map(parentUrn)); - return result; - } catch (Exception e) { - throw new RuntimeException("Failed to load schemaField entity", e); - } - } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java new file mode 100644 index 0000000000000..ad48067599328 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java @@ -0,0 +1,80 @@ +package com.linkedin.datahub.graphql.types.structuredproperty; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.NumberValue; +import com.linkedin.datahub.graphql.generated.PropertyValue; +import com.linkedin.datahub.graphql.generated.StringValue; +import com.linkedin.datahub.graphql.generated.StructuredPropertiesEntry; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StructuredPropertiesMapper { + + public static final StructuredPropertiesMapper INSTANCE = new StructuredPropertiesMapper(); + + public static com.linkedin.datahub.graphql.generated.StructuredProperties map( + @Nonnull final StructuredProperties structuredProperties) { + return INSTANCE.apply(structuredProperties); + } + + public com.linkedin.datahub.graphql.generated.StructuredProperties apply( + @Nonnull final StructuredProperties structuredProperties) { + com.linkedin.datahub.graphql.generated.StructuredProperties result = + new com.linkedin.datahub.graphql.generated.StructuredProperties(); + result.setProperties( + structuredProperties.getProperties().stream() + .map(this::mapStructuredProperty) + .collect(Collectors.toList())); + return result; + } + + private StructuredPropertiesEntry mapStructuredProperty( + StructuredPropertyValueAssignment valueAssignment) { + StructuredPropertiesEntry entry = new StructuredPropertiesEntry(); + entry.setStructuredProperty(createStructuredPropertyEntity(valueAssignment)); + final List values = new ArrayList<>(); + final List entities = new ArrayList<>(); + valueAssignment + .getValues() + .forEach( + value -> { + if (value.isString()) { + this.mapStringValue(value.getString(), values, entities); + } else if (value.isDouble()) { + values.add(new NumberValue(value.getDouble())); + } + }); + entry.setValues(values); + entry.setValueEntities(entities); + return entry; + } + + private StructuredPropertyEntity createStructuredPropertyEntity( + StructuredPropertyValueAssignment assignment) { + StructuredPropertyEntity entity = new StructuredPropertyEntity(); + entity.setUrn(assignment.getPropertyUrn().toString()); + entity.setType(EntityType.STRUCTURED_PROPERTY); + return entity; + } + + private void mapStringValue( + String stringValue, List values, List entities) { + try { + final Urn urnValue = Urn.createFromString(stringValue); + entities.add(UrnToEntityMapper.map(urnValue)); + } catch (Exception e) { + log.debug("String value is not an urn for this structured property entry"); + } + values.add(new StringValue(stringValue)); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java new file mode 100644 index 0000000000000..259020b83bee1 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java @@ -0,0 +1,124 @@ +package com.linkedin.datahub.graphql.types.structuredproperty; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.datahub.graphql.generated.AllowedValue; +import com.linkedin.datahub.graphql.generated.DataTypeEntity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.EntityTypeEntity; +import com.linkedin.datahub.graphql.generated.NumberValue; +import com.linkedin.datahub.graphql.generated.PropertyCardinality; +import com.linkedin.datahub.graphql.generated.StringValue; +import com.linkedin.datahub.graphql.generated.StructuredPropertyDefinition; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.TypeQualifier; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.structured.PropertyValueArray; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class StructuredPropertyMapper + implements ModelMapper { + + private static final String ALLOWED_TYPES = "allowedTypes"; + + public static final StructuredPropertyMapper INSTANCE = new StructuredPropertyMapper(); + + public static StructuredPropertyEntity map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + @Override + public StructuredPropertyEntity apply(@Nonnull final EntityResponse entityResponse) { + final StructuredPropertyEntity result = new StructuredPropertyEntity(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.STRUCTURED_PROPERTY); + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, (this::mapStructuredPropertyDefinition)); + return mappingHelper.getResult(); + } + + private void mapStructuredPropertyDefinition( + @Nonnull StructuredPropertyEntity extendedProperty, @Nonnull DataMap dataMap) { + com.linkedin.structured.StructuredPropertyDefinition gmsDefinition = + new com.linkedin.structured.StructuredPropertyDefinition(dataMap); + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setQualifiedName(gmsDefinition.getQualifiedName()); + definition.setCardinality( + PropertyCardinality.valueOf(gmsDefinition.getCardinality().toString())); + definition.setValueType(createDataTypeEntity(gmsDefinition.getValueType())); + if (gmsDefinition.hasDisplayName()) { + definition.setDisplayName(gmsDefinition.getDisplayName()); + } + if (gmsDefinition.getDescription() != null) { + definition.setDescription(gmsDefinition.getDescription()); + } + if (gmsDefinition.hasAllowedValues()) { + definition.setAllowedValues(mapAllowedValues(gmsDefinition.getAllowedValues())); + } + if (gmsDefinition.hasTypeQualifier()) { + definition.setTypeQualifier(mapTypeQualifier(gmsDefinition.getTypeQualifier())); + } + definition.setEntityTypes( + gmsDefinition.getEntityTypes().stream() + .map(this::createEntityTypeEntity) + .collect(Collectors.toList())); + extendedProperty.setDefinition(definition); + } + + private List mapAllowedValues(@Nonnull PropertyValueArray gmsValues) { + List allowedValues = new ArrayList<>(); + gmsValues.forEach( + value -> { + final AllowedValue allowedValue = new AllowedValue(); + if (value.getValue().isString()) { + allowedValue.setValue(new StringValue(value.getValue().getString())); + } else if (value.getValue().isDouble()) { + allowedValue.setValue(new NumberValue(value.getValue().getDouble())); + } + if (value.getDescription() != null) { + allowedValue.setDescription(value.getDescription()); + } + allowedValues.add(allowedValue); + }); + return allowedValues; + } + + private DataTypeEntity createDataTypeEntity(final Urn dataTypeUrn) { + final DataTypeEntity dataType = new DataTypeEntity(); + dataType.setUrn(dataTypeUrn.toString()); + dataType.setType(EntityType.DATA_TYPE); + return dataType; + } + + private TypeQualifier mapTypeQualifier(final StringArrayMap gmsTypeQualifier) { + final TypeQualifier typeQualifier = new TypeQualifier(); + List allowedTypes = gmsTypeQualifier.get(ALLOWED_TYPES); + if (allowedTypes != null) { + typeQualifier.setAllowedTypes( + allowedTypes.stream().map(this::createEntityTypeEntity).collect(Collectors.toList())); + } + return typeQualifier; + } + + private EntityTypeEntity createEntityTypeEntity(final Urn entityTypeUrn) { + return createEntityTypeEntity(entityTypeUrn.toString()); + } + + private EntityTypeEntity createEntityTypeEntity(final String entityTypeUrnStr) { + final EntityTypeEntity entityType = new EntityTypeEntity(); + entityType.setUrn(entityTypeUrnStr); + entityType.setType(EntityType.ENTITY_TYPE); + return entityType; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java new file mode 100644 index 0000000000000..b028563b5253c --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java @@ -0,0 +1,79 @@ +package com.linkedin.datahub.graphql.types.structuredproperty; + +import static com.linkedin.metadata.Constants.*; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import graphql.execution.DataFetcherResult; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class StructuredPropertyType + implements com.linkedin.datahub.graphql.types.EntityType { + + public static final Set ASPECTS_TO_FETCH = + ImmutableSet.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.STRUCTURED_PROPERTY; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return StructuredPropertyEntity.class; + } + + @Override + public List> batchLoad( + @Nonnull List urns, @Nonnull QueryContext context) throws Exception { + final List extendedPropertyUrns = + urns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2( + STRUCTURED_PROPERTY_ENTITY_NAME, + new HashSet<>(extendedPropertyUrns), + ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : extendedPropertyUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map( + gmsResult -> + gmsResult == null + ? null + : DataFetcherResult.newResult() + .data(StructuredPropertyMapper.map(gmsResult)) + .build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Queries", e); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java index 8ea06f46d5133..a4bbd685fd4a2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java @@ -11,8 +11,8 @@ import com.linkedin.datahub.graphql.generated.FacetFilter; import com.linkedin.datahub.graphql.generated.FilterOperator; import com.linkedin.datahub.graphql.generated.LogicalOperator; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 52451e195ee84..7964f7e4fab23 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -212,6 +212,16 @@ type VisualConfig { """ faviconUrl: String + """ + Custom app title to show in the browser tab + """ + appTitle: String + + """ + Boolean flag disabling viewing the Business Glossary page for users without the 'Manage Glossaries' privilege + """ + hideGlossary: Boolean + """ Configuration for the queries tab """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index ebb13d32643ed..2ad4982579380 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -700,6 +700,31 @@ type Mutation { deleteOwnershipType( "Urn of the Custom Ownership Type to remove." urn: String!, deleteReferences: Boolean): Boolean + + """ + Submit a response to a prompt from a form collecting metadata on different entities. + Provide the urn of the entity you're submitting a form response as well as the required input. + """ + submitFormPrompt(urn: String!, input: SubmitFormPromptInput!): Boolean + + """ + Assign a form to different entities. This will be a patch by adding this form to the list + of forms on an entity. + """ + batchAssignForm(input: BatchAssignFormInput!): Boolean + + """ + Creates a filter for a form to apply it to certain entities. Entities that match this filter will have + a given form applied to them. + This feature is ONLY supported in Acryl DataHub. + """ + createDynamicFormAssignment(input: CreateDynamicFormAssignmentInput!): Boolean + + """ + Verifies a form on an entity when all of the required questions on the form are complete and the form + is of type VERIFICATION. + """ + verifyForm(input: VerifyFormInput!): Boolean } """ @@ -910,6 +935,31 @@ enum EntityType { A Role from an organisation """ ROLE + + """" + An structured property on entities + """ + STRUCTURED_PROPERTY + + """" + A form entity on entities + """ + FORM + + """" + A data type registered to DataHub + """ + DATA_TYPE + + """" + A type of entity registered to DataHub + """ + ENTITY_TYPE + + """ + Another entity type - refer to a provided entity type urn. + """ + OTHER } """ @@ -1284,6 +1334,11 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { """ domain: DomainAssociation + """ + The forms associated with the Dataset + """ + forms: Forms + """ The Roles and the properties to access the dataset """ @@ -1426,6 +1481,11 @@ type Dataset implements EntityWithRelationships & Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Structured properties about this Dataset + """ + structuredProperties: StructuredProperties } type RoleAssociation { @@ -1529,6 +1589,7 @@ type SiblingProperties { If this entity is the primary sibling among the sibling set """ isPrimary: Boolean + """ The sibling entities """ @@ -1910,6 +1971,12 @@ type GlossaryTerm implements Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -2047,6 +2114,12 @@ type GlossaryNode implements Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -2076,6 +2149,11 @@ type GlossaryNodeProperties { Description of the glossary term """ description: String + + """ + Custom properties of the Glossary Node + """ + customProperties: [CustomPropertiesEntry!] } """ @@ -2447,6 +2525,12 @@ type Container implements Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -2822,12 +2906,27 @@ type SchemaFieldEntity implements Entity { """ parent: Entity! + """ + Structured properties on this schema field + """ + structuredProperties: StructuredProperties + """ Granular API for querying edges extending from this entity """ relationships(input: RelationshipsInput!): EntityRelationshipsResult } +""" +Object containing structured properties for an entity +""" +type StructuredProperties { + """ + Structured properties on this entity + """ + properties: [StructuredPropertiesEntry!] +} + """ Information about an individual field in a Dataset schema """ @@ -2902,6 +3001,11 @@ type SchemaField { For schema fields that have other properties that are not modeled explicitly, represented as a JSON string. """ jsonProps: String + + """ + Schema field entity that exist in the database for this schema field + """ + schemaFieldEntity: SchemaFieldEntity } """ @@ -3444,6 +3548,12 @@ type CorpUser implements Entity { Settings that a user can customize through the datahub ui """ settings: CorpUserSettings + + """ + Experimental API. + For fetching extra aspects that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -3804,6 +3914,12 @@ type CorpGroup implements Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -4005,6 +4121,12 @@ type Tag implements Entity { Deprecated, use properties.description field instead """ description: String @deprecated + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -4099,6 +4221,11 @@ type GlossaryTermAssociation { """ term: GlossaryTerm! + """ + The actor who is responsible for the term being added" + """ + actor: CorpUser + """ Reference back to the associated urn for tracking purposes e.g. when sibling nodes are merged together """ @@ -4635,6 +4762,12 @@ type Notebook implements Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -4955,6 +5088,12 @@ type Dashboard implements EntityWithRelationships & Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -5265,6 +5404,12 @@ type Chart implements EntityWithRelationships & Entity & BrowsableEntity { Sub Types that this entity implements """ subTypes: SubTypes + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -5622,6 +5767,12 @@ type DataFlow implements EntityWithRelationships & Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -5822,6 +5973,12 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -6558,10 +6715,10 @@ type PartitionSpec { """ The partition identifier """ - partition: String! + partition: String """ - The optional time window partition information + The optional time window partition information - required if type is TIMESTAMP_FIELD. """ timePartition: TimeWindow } @@ -6587,7 +6744,6 @@ type TimeWindow { durationMillis: Long! } - """ An assertion represents a programmatic validation, check, or test performed periodically against another Entity. """ @@ -7048,10 +7204,29 @@ type AssertionStdParameter { The type of an AssertionStdParameter """ enum AssertionStdParameterType { + """ + A string value + """ STRING + + """ + A numeric value + """ NUMBER + + """ + A list of values. When used, the value should be formatted as a serialized JSON array. + """ LIST + + """ + A set of values. When used, the value should be formatted as a serialized JSON array. + """ SET + + """ + A value of unknown type + """ UNKNOWN } @@ -8738,6 +8913,12 @@ type MLModel implements EntityWithRelationships & Entity & BrowsableEntity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -8849,6 +9030,12 @@ type MLModelGroup implements EntityWithRelationships & Entity & BrowsableEntity Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } type MLModelGroupProperties { @@ -8973,6 +9160,12 @@ type MLFeature implements EntityWithRelationships & Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } type MLHyperParam { @@ -9142,6 +9335,12 @@ type MLPrimaryKey implements EntityWithRelationships & Entity { Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } type MLPrimaryKeyProperties { @@ -9269,6 +9468,12 @@ type MLFeatureTable implements EntityWithRelationships & Entity & BrowsableEntit Whether or not this entity exists on DataHub """ exists: Boolean + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } type MLFeatureTableEditableProperties { @@ -9577,6 +9782,22 @@ enum CostType { ORG_COST_TYPE } + +""" +Audit stamp containing a resolved actor +""" +type ResolvedAuditStamp { + """ + When the audited action took place + """ + time: Long! + + """ + Who performed the audited action + """ + actor: CorpUser +} + type SubTypes { """ The sub-types that this entity implements. e.g. Datasets that are views will implement the "view" subtype @@ -9644,6 +9865,12 @@ type Domain implements Entity { Edges extending from this entity """ relationships(input: RelationshipsInput!): EntityRelationshipsResult + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -10139,6 +10366,12 @@ type DataHubRole implements Entity { The description of the Role """ description: String! + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -11015,6 +11248,12 @@ type DataProduct implements Entity { Tags used for searching Data Product """ tags: GlobalTags + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] } """ @@ -11270,3 +11509,94 @@ input UpdateOwnershipTypeInput { """ description: String } + +""" +Describes a generic filter on a dataset +""" +type DatasetFilter { + """ + Type of partition + """ + type: DatasetFilterType! + + """ + The raw query if using a SQL FilterType + """ + sql: String +} + +""" +Type of partition +""" +enum DatasetFilterType { + """ + Use a SQL string to apply the filter + """ + SQL +} + + +""" +Input required to create or update a DatasetFilter +""" +input DatasetFilterInput { + """ + Type of partition + """ + type: DatasetFilterType! + + """ + The raw query if using a SQL FilterType + """ + sql: String +} + +""" +An entity type registered in DataHub +""" +type EntityTypeEntity implements Entity { + """ + A primary key associated with the Query + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + Info about this type including its name + """ + info: EntityTypeInfo! + + """ + Granular API for querying edges extending from this entity + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +Properties about an individual entity type +""" +type EntityTypeInfo { + """ + The standard entity type + """ + type: EntityType! + + """ + The fully qualified name of the entity type. This includes its namespace + """ + qualifiedName: String! + + """ + The display name of this type + """ + displayName: String + + """ + The description of this type + """ + description: String +} diff --git a/datahub-graphql-core/src/main/resources/forms.graphql b/datahub-graphql-core/src/main/resources/forms.graphql new file mode 100644 index 0000000000000..0ff55cfa9f173 --- /dev/null +++ b/datahub-graphql-core/src/main/resources/forms.graphql @@ -0,0 +1,407 @@ +""" +Requirements forms that are assigned to an entity. +""" +type Forms { + """ + Forms that are still incomplete. + """ + incompleteForms: [FormAssociation!]! + + """ + Forms that have been completed. + """ + completedForms: [FormAssociation!]! + + """ + Verifications that have been applied to the entity via completed forms. + """ + verifications: [FormVerificationAssociation!]! +} + +type FormAssociation { + """ + The form related to the associated urn + """ + form: Form! + + """ + Reference back to the urn with the form on it for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! + + """ + The prompt that still need to be completed for this form + """ + incompletePrompts: [FormPromptAssociation!] + + """ + The prompt that are already completed for this form + """ + completedPrompts: [FormPromptAssociation!] +} + +""" +Verification object that has been applied to the entity via a completed form. +""" +type FormVerificationAssociation { + """ + The form related to the associated urn + """ + form: Form! + + """ + When this verification was applied to this entity + """ + lastModified: ResolvedAuditStamp +} + +""" +A form that helps with filling out metadata on an entity +""" +type FormPromptAssociation { + """ + The unique id of the form prompt + """ + id: String! + + """ + When and by whom this form prompt has last been modified + """ + lastModified: ResolvedAuditStamp! + + """ + Optional information about the field-level prompt associations. + """ + fieldAssociations: FormPromptFieldAssociations +} + +""" +Information about the field-level prompt associations. +""" +type FormPromptFieldAssociations { + """ + If this form prompt is for fields, this will contain a list of completed associations per field + """ + completedFieldPrompts: [FieldFormPromptAssociation!] + + """ + If this form prompt is for fields, this will contain a list of incomlete associations per field + """ + incompleteFieldPrompts: [FieldFormPromptAssociation!] +} + +""" +An association for field-level form prompts +""" +type FieldFormPromptAssociation { + """ + The schema field path + """ + fieldPath: String! + + """ + When and by whom this form field-level prompt has last been modified + """ + lastModified: ResolvedAuditStamp! +} + +""" +A form that helps with filling out metadata on an entity +""" +type Form implements Entity { + """ + A primary key associated with the Form + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + Information about this form + """ + info: FormInfo! + + """ + Ownership metadata of the form + """ + ownership: Ownership + + """ + Granular API for querying edges extending from this entity + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +The type of a form. This is optional on a form entity +""" +enum FormType { + """ + This form is used for "verifying" entities as a state for governance and compliance + """ + VERIFICATION + + """ + This form is used to help with filling out metadata on entities + """ + COMPLETION +} + +""" +Properties about an individual Form +""" +type FormInfo { + """ + The name of this form + """ + name: String! + + """ + The description of this form + """ + description: String + + """ + The type of this form + """ + type: FormType! + + """ + The prompt for this form + """ + prompts: [FormPrompt!]! + + """ + The actors that are assigned to complete the forms for the associated entities. + """ + actors: FormActorAssignment! +} + +""" +A prompt shown to the user to collect metadata about an entity +""" +type FormPrompt { + """ + The ID of this prompt. This will be globally unique. + """ + id: String! + + """ + The title of this prompt + """ + title: String! + + """ + The urn of the parent form that this prompt is part of + """ + formUrn: String! + + """ + The description of this prompt + """ + description: String + + """ + The description of this prompt + """ + type: FormPromptType! + + """ + Whether the prompt is required for the form to be considered completed. + """ + required: Boolean! + + """ + The params for this prompt if type is STRUCTURED_PROPERTY + """ + structuredPropertyParams: StructuredPropertyParams +} + +""" +Enum of all form prompt types +""" +enum FormPromptType { + """ + A structured property form prompt type. + """ + STRUCTURED_PROPERTY + """ + A schema field-level structured property form prompt type. + """ + FIELDS_STRUCTURED_PROPERTY +} + +""" +A prompt shown to the user to collect metadata about an entity +""" +type StructuredPropertyParams { + """ + The structured property required for the prompt on this entity + """ + structuredProperty: StructuredPropertyEntity! +} + +""" +Input for responding to a singular prompt in a form +""" +input SubmitFormPromptInput { + """ + The unique ID of the prompt this input is responding to + """ + promptId: String! + + """ + The urn of the form that this prompt is a part of + """ + formUrn: String! + + """ + The type of prompt that this input is responding to + """ + type: FormPromptType! + + """ + The fieldPath on a schema field that this prompt submission is association with. + This should be provided when the prompt is type FIELDS_STRUCTURED_PROPERTY + """ + fieldPath: String + + """ + The structured property required for the prompt on this entity + """ + structuredPropertyParams: StructuredPropertyInputParams +} + +""" +Input for responding to a singular prompt in a form for a batch of entities +""" +input BatchSubmitFormPromptInput { + """ + The urns of the entities this prompt submission is for + """ + assetUrns: [String!]! + + """ + Input for responding to a specific prompt on a form + """ + input: SubmitFormPromptInput +} + +""" +Input for collecting structured property values to apply to entities +""" +input PropertyValueInput { + """ + The string value for this structured property + """ + stringValue: String + + """ + The number value for this structured property + """ + numberValue: Float +} + +""" +A prompt shown to the user to collect metadata about an entity +""" +input StructuredPropertyInputParams { + """ + The urn of the structured property being applied to an entity + """ + structuredPropertyUrn: String! + + """ + The list of values you want to apply on this structured property to an entity + """ + values: [PropertyValueInput!]! +} + +""" +Input for batch assigning a form to different entities +""" +input BatchAssignFormInput { + """ + The urn of the form being assigned to entities + """ + formUrn: String! + + """ + The entities that this form is being assigned to + """ + entityUrns: [String!]! +} + +""" +Input for batch assigning a form to different entities +""" +input CreateDynamicFormAssignmentInput { + """ + The urn of the form being assigned to entities that match some criteria + """ + formUrn: String! + + """ + A list of disjunctive criterion for the filter. (or operation to combine filters). + Entities that match this filter will have this form applied to them. + Currently, we only support a set of fields to filter on and they are: + (1) platform (2) subType (3) container (4) _entityType (5) domain + """ + orFilters: [AndFilterInput!]! +} + +type FormActorAssignment { + """ + Whether the form should be completed by owners of the assets which the form is applied to. + """ + owners: Boolean! + + """ + Urns of the users that the form is assigned to. If null, then no users are specifically targeted. + """ + users: [CorpUser!] + + """ + Groups that the form is assigned to. If null, then no groups are specifically targeted. + """ + groups: [CorpGroup!] + + """ + Whether or not the current actor is universally assigned to this form, either by user or by group. + Note that this does not take into account entity ownership based assignment. + """ + isAssignedToMe: Boolean! +} + +""" +Input for verifying forms on entities +""" +input VerifyFormInput { + """ + The urn of the form being verified on an entity + """ + formUrn: String! + + """ + The urn of the entity that is having a form verified on it + """ + entityUrn: String! +} + +""" +Input for verifying a batch of entities for a give form +""" +input BatchVerifyFormInput { + """ + The urns of the entities getting verified for this form + """ + assetUrns: [String!]! + + """ + The urn of the form being verified on the given entities + """ + formUrn: String! +} diff --git a/datahub-graphql-core/src/main/resources/ingestion.graphql b/datahub-graphql-core/src/main/resources/ingestion.graphql index 21f9fb2633119..d65343c0a16d2 100644 --- a/datahub-graphql-core/src/main/resources/ingestion.graphql +++ b/datahub-graphql-core/src/main/resources/ingestion.graphql @@ -36,6 +36,11 @@ extend type Mutation { """ createSecret(input: CreateSecretInput!): String + """ + Update a Secret + """ + updateSecret(input: UpdateSecretInput!): String + """ Delete a Secret """ @@ -560,6 +565,31 @@ input CreateSecretInput { description: String } +""" +Input arguments for updating a Secret +""" +input UpdateSecretInput { + """ + The primary key of the Secret to update + """ + urn: String! + + """ + The name of the secret for reference in ingestion recipes + """ + name: String! + + """ + The value of the secret, to be encrypted and stored + """ + value: String! + + """ + An optional description for the secret + """ + description: String +} + """ Input arguments for retrieving the plaintext values of a set of secrets """ diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql new file mode 100644 index 0000000000000..2bed0f1155ff1 --- /dev/null +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -0,0 +1,243 @@ +""" +A structured property that can be shared between different entities +""" +type StructuredPropertyEntity implements Entity { + """ + A primary key associated with the structured property + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + Definition of this structured property including its name + """ + definition: StructuredPropertyDefinition! + + """ + Granular API for querying edges extending from this entity + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +Properties about an individual Query +""" +type StructuredPropertyDefinition { + """ + The fully qualified name of the property. This includes its namespace + """ + qualifiedName: String! + + """ + The display name of this structured property + """ + displayName: String + + """ + The description of this property + """ + description: String + + """ + The cardinality of a Structured Property determining whether one or multiple values + can be applied to the entity from this property. + """ + cardinality: PropertyCardinality + + """ + A list of allowed values that the property is allowed to take. + """ + allowedValues: [AllowedValue!] + + """ + The type of this structured property + """ + valueType: DataTypeEntity! + + """ + Allows for type specialization of the valueType to be more specific about which + entity types are allowed, for example. + """ + typeQualifier: TypeQualifier + + """ + Entity types that this structured property can be applied to + """ + entityTypes: [EntityTypeEntity!]! +} + +""" +An entry for an allowed value for a structured property +""" +type AllowedValue { + """ + The allowed value + """ + value: PropertyValue! + + """ + The description of this allowed value + """ + description: String +} + +""" +The cardinality of a Structured Property determining whether one or multiple values +can be applied to the entity from this property. +""" +enum PropertyCardinality { + """ + Only one value of this property can applied to an entity + """ + SINGLE + + """ + Multiple values of this property can applied to an entity + """ + MULTIPLE +} + +""" +Allows for type specialization of the valueType to be more specific about which +entity types are allowed, for example. +""" +type TypeQualifier { + """ + The list of allowed entity types + """ + allowedTypes: [EntityTypeEntity!] +} + +""" +String property value +""" +type StringValue { + """ + The value of a string type property + """ + stringValue: String! +} + +""" +Numeric property value +""" +type NumberValue { + """ + The value of a number type property + """ + numberValue: Float! +} + +""" +The value of a property +""" +union PropertyValue = StringValue | NumberValue + +""" +An entry in an structured properties list represented as a tuple +""" +type StructuredPropertiesEntry { + """ + The key of the map entry + """ + structuredProperty: StructuredPropertyEntity! + + """ + The values of the structured property for this entity + """ + values: [PropertyValue]! + + """ + The optional entities associated with the values if the values are entity urns + """ + valueEntities: [Entity] +} + +""" +A data type registered in DataHub +""" +type DataTypeEntity implements Entity { + """ + A primary key associated with the Query + """ + urn: String! + + """ + A standard Entity Type + """ + type: EntityType! + + """ + Info about this type including its name + """ + info: DataTypeInfo! + + """ + Granular API for querying edges extending from this entity + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +A well-supported, standard DataHub Data Type. +""" +enum StdDataType { + """ + String data type + """ + STRING + + """ + Number data type + """ + NUMBER + + """ + Urn data type + """ + URN + + """ + Rich text data type. Right now this is markdown only. + """ + RICH_TEXT + + """ + Date data type in format YYYY-MM-DD + """ + DATE + + """ + Any other data type - refer to a provided data type urn. + """ + OTHER +} + +""" +Properties about an individual data type +""" +type DataTypeInfo { + """ + The standard data type + """ + type: StdDataType! + + """ + The fully qualified name of the type. This includes its namespace + """ + qualifiedName: String! + + """ + The display name of this type + """ + displayName: String + + """ + The description of this type + """ + description: String +} diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 8f2377edb546e..8896dd02b5ad3 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -1139,7 +1139,7 @@ type QuickFilter { } """ -Freshness stats for a query result. +Freshness stats for a query result. Captures whether the query was served out of a cache, what the staleness was, etc. """ type FreshnessStats { @@ -1154,7 +1154,7 @@ type FreshnessStats { In case an index was consulted, this reflects the freshness of the index """ systemFreshness: [SystemFreshness] - + } type SystemFreshness { @@ -1303,4 +1303,4 @@ input SortCriterion { The order in which we will be sorting """ sortOrder: SortOrder! -} +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/tests.graphql b/datahub-graphql-core/src/main/resources/tests.graphql index 9dce48ac60d83..579f4919bdc78 100644 --- a/datahub-graphql-core/src/main/resources/tests.graphql +++ b/datahub-graphql-core/src/main/resources/tests.graphql @@ -44,6 +44,7 @@ Definition of the test type TestDefinition { """ JSON-based def for the test + Deprecated! JSON representation is no longer supported. """ json: String } @@ -209,6 +210,7 @@ input UpdateTestInput { input TestDefinitionInput { """ The string representation of the Test + Deprecated! JSON representation is no longer supported. """ json: String } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java index de507eda8cdef..b75530773c352 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java @@ -122,13 +122,7 @@ public static void verifyIngestProposal( int numberOfInvocations, List proposals) { AspectsBatchImpl batch = - AspectsBatchImpl.builder() - .mcps( - proposals, - mock(AuditStamp.class), - mockService.getEntityRegistry(), - mockService.getSystemEntityClient()) - .build(); + AspectsBatchImpl.builder().mcps(proposals, mock(AuditStamp.class), mockService).build(); Mockito.verify(mockService, Mockito.times(numberOfInvocations)) .ingestProposal(Mockito.eq(batch), Mockito.eq(false)); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java index 433772d7e2cfe..c565e771a0475 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java @@ -26,6 +26,7 @@ import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.ViewService; import com.linkedin.view.DataHubViewDefinition; import com.linkedin.view.DataHubViewInfo; @@ -44,6 +45,7 @@ public class BrowseV2ResolverTest { @Test public static void testBrowseV2Success() throws Exception { + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = Mockito.mock(ViewService.class); EntityClient mockClient = initMockEntityClient( @@ -70,7 +72,8 @@ public static void testBrowseV2Success() throws Exception { .setFrom(0) .setPageSize(10)); - final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, mockService); + final BrowseV2Resolver resolver = + new BrowseV2Resolver(mockClient, mockService, mockFormService); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); QueryContext mockContext = getMockAllowContext(); @@ -87,6 +90,7 @@ public static void testBrowseV2Success() throws Exception { @Test public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = Mockito.mock(ViewService.class); List orFilters = new ArrayList<>(); @@ -123,7 +127,8 @@ public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { .setFrom(0) .setPageSize(10)); - final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, mockService); + final BrowseV2Resolver resolver = + new BrowseV2Resolver(mockClient, mockService, mockFormService); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); QueryContext mockContext = getMockAllowContext(); @@ -143,6 +148,7 @@ public static void testBrowseV2SuccessWithQueryAndFilter() throws Exception { @Test public static void testBrowseV2SuccessWithView() throws Exception { DataHubViewInfo viewInfo = createViewInfo(new StringArray()); + FormService mockFormService = Mockito.mock(FormService.class); ViewService viewService = initMockViewService(TEST_VIEW_URN, viewInfo); EntityClient mockClient = @@ -170,7 +176,8 @@ public static void testBrowseV2SuccessWithView() throws Exception { .setFrom(0) .setPageSize(10)); - final BrowseV2Resolver resolver = new BrowseV2Resolver(mockClient, viewService); + final BrowseV2Resolver resolver = + new BrowseV2Resolver(mockClient, viewService, mockFormService); DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); QueryContext mockContext = getMockAllowContext(); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java index 9596abf55d04f..c6e6cdc7f018e 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolverTest.java @@ -10,7 +10,7 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.DomainEntitiesInput; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolverTest.java new file mode 100644 index 0000000000000..0fe57d0a28fff --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/IsFormAssignedToMeResolverTest.java @@ -0,0 +1,167 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CorpGroup; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.FormActorAssignment; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class IsFormAssignedToMeResolverTest { + + private static final Urn TEST_USER_1 = UrnUtils.getUrn("urn:li:corpuser:test-1"); + private static final Urn TEST_USER_2 = UrnUtils.getUrn("urn:li:corpuser:test-2"); + private static final Urn TEST_GROUP_1 = UrnUtils.getUrn("urn:li:corpGroup:test-1"); + private static final Urn TEST_GROUP_2 = UrnUtils.getUrn("urn:li:corpGroup:test-2"); + + @Test + public void testGetSuccessUserMatch() throws Exception { + GroupService groupService = mockGroupService(TEST_USER_1, Collections.emptyList()); + + CorpGroup assignedGroup = new CorpGroup(); + assignedGroup.setUrn(TEST_GROUP_1.toString()); + + CorpUser assignedUser = new CorpUser(); + assignedUser.setUrn(TEST_USER_1.toString()); + + FormActorAssignment actors = new FormActorAssignment(); + actors.setGroups(new ArrayList<>(ImmutableList.of(assignedGroup))); + actors.setUsers(new ArrayList<>(ImmutableList.of(assignedUser))); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertTrue(resolver.get(mockEnv).get()); + Mockito.verifyNoMoreInteractions(groupService); // Should not perform group lookup. + } + + @Test + public void testGetSuccessGroupMatch() throws Exception { + GroupService groupService = + mockGroupService(TEST_USER_1, ImmutableList.of(TEST_GROUP_1)); // is in group + + CorpGroup assignedGroup = new CorpGroup(); + assignedGroup.setUrn(TEST_GROUP_1.toString()); + + CorpUser assignedUser = new CorpUser(); + assignedUser.setUrn(TEST_USER_2.toString()); // does not match + + FormActorAssignment actors = new FormActorAssignment(); + actors.setGroups(new ArrayList<>(ImmutableList.of(assignedGroup))); + actors.setUsers(new ArrayList<>(ImmutableList.of(assignedUser))); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertTrue(resolver.get(mockEnv).get()); + } + + @Test + public void testGetSuccessBothMatch() throws Exception { + GroupService groupService = + mockGroupService(TEST_USER_1, ImmutableList.of(TEST_GROUP_1)); // is in group + + CorpGroup assignedGroup = new CorpGroup(); + assignedGroup.setUrn(TEST_GROUP_1.toString()); + + CorpUser assignedUser = new CorpUser(); + assignedUser.setUrn(TEST_USER_1.toString()); // is matching user + + FormActorAssignment actors = new FormActorAssignment(); + actors.setGroups(new ArrayList<>(ImmutableList.of(assignedGroup))); + actors.setUsers(new ArrayList<>(ImmutableList.of(assignedUser))); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertTrue(resolver.get(mockEnv).get()); + Mockito.verifyNoMoreInteractions(groupService); // Should not perform group lookup. + } + + @Test + public void testGetSuccessNoMatchNullAssignment() throws Exception { + GroupService groupService = + mockGroupService(TEST_USER_1, ImmutableList.of(TEST_GROUP_1, TEST_GROUP_2)); + + FormActorAssignment actors = new FormActorAssignment(); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertFalse(resolver.get(mockEnv).get()); + } + + @Test + public void testGetSuccessNoMatchEmptyAssignment() throws Exception { + GroupService groupService = + mockGroupService(TEST_USER_1, ImmutableList.of(TEST_GROUP_1, TEST_GROUP_2)); + + FormActorAssignment actors = new FormActorAssignment(); + actors.setUsers(Collections.emptyList()); + actors.setGroups(Collections.emptyList()); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertFalse(resolver.get(mockEnv).get()); + } + + @Test + public void testGetSuccessNoMatchNoAssignmentMatch() throws Exception { + GroupService groupService = mockGroupService(TEST_USER_1, ImmutableList.of(TEST_GROUP_1)); + + CorpGroup assignedGroup = new CorpGroup(); + assignedGroup.setUrn(TEST_GROUP_2.toString()); // Does not match. + + CorpUser assignedUser = new CorpUser(); + assignedUser.setUrn(TEST_USER_2.toString()); // does not match + + FormActorAssignment actors = new FormActorAssignment(); + actors.setGroups(new ArrayList<>(ImmutableList.of(assignedGroup))); + actors.setUsers(new ArrayList<>(ImmutableList.of(assignedUser))); + + QueryContext mockContext = getMockAllowContext(TEST_USER_1.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + Mockito.when(mockEnv.getSource()).thenReturn(actors); + + IsFormAssignedToMeResolver resolver = new IsFormAssignedToMeResolver(groupService); + assertFalse(resolver.get(mockEnv).get()); + } + + private GroupService mockGroupService(final Urn userUrn, final List groupUrns) + throws Exception { + GroupService mockService = Mockito.mock(GroupService.class); + Mockito.when( + mockService.getGroupsForUser(Mockito.eq(userUrn), Mockito.any(Authentication.class))) + .thenReturn(groupUrns); + return mockService; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolverTest.java new file mode 100644 index 0000000000000..192f4ff9aa7c7 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/form/VerifyFormResolverTest.java @@ -0,0 +1,122 @@ +package com.linkedin.datahub.graphql.resolvers.form; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.group.GroupService; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.VerifyFormInput; +import com.linkedin.metadata.service.FormService; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class VerifyFormResolverTest { + + private static final String TEST_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)"; + private static final String TEST_FORM_URN = "urn:li:form:1"; + + private static final VerifyFormInput TEST_INPUT = + new VerifyFormInput(TEST_FORM_URN, TEST_DATASET_URN); + + @Test + public void testGetSuccess() throws Exception { + FormService mockFormService = initMockFormService(true, true); + GroupService mockGroupService = initMockGroupService(); + VerifyFormResolver resolver = new VerifyFormResolver(mockFormService, mockGroupService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + boolean success = resolver.get(mockEnv).get(); + + assertTrue(success); + + // Validate that we called verify on the service + Mockito.verify(mockFormService, Mockito.times(1)) + .verifyFormForEntity( + Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)), + Mockito.eq(UrnUtils.getUrn(TEST_DATASET_URN)), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetUnauthorized() throws Exception { + FormService mockFormService = initMockFormService(false, true); + GroupService mockGroupService = initMockGroupService(); + VerifyFormResolver resolver = new VerifyFormResolver(mockFormService, mockGroupService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + // Validate that we do not call verify on the service + Mockito.verify(mockFormService, Mockito.times(0)) + .verifyFormForEntity(Mockito.any(), Mockito.any(), Mockito.any(Authentication.class)); + } + + @Test + public void testThrowErrorOnVerification() throws Exception { + FormService mockFormService = initMockFormService(true, false); + GroupService mockGroupService = initMockGroupService(); + VerifyFormResolver resolver = new VerifyFormResolver(mockFormService, mockGroupService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + // Validate that we do call verifyFormForEntity but an error is thrown + Mockito.verify(mockFormService, Mockito.times(1)) + .verifyFormForEntity(Mockito.any(), Mockito.any(), Mockito.any(Authentication.class)); + } + + private FormService initMockFormService( + final boolean isFormAssignedToUser, final boolean shouldVerify) throws Exception { + FormService service = Mockito.mock(FormService.class); + Mockito.when( + service.isFormAssignedToUser( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class))) + .thenReturn(isFormAssignedToUser); + + if (shouldVerify) { + Mockito.when( + service.verifyFormForEntity( + Mockito.any(), Mockito.any(), Mockito.any(Authentication.class))) + .thenReturn(true); + } else { + Mockito.when( + service.verifyFormForEntity( + Mockito.any(), Mockito.any(), Mockito.any(Authentication.class))) + .thenThrow(new RuntimeException()); + } + + return service; + } + + private GroupService initMockGroupService() throws Exception { + GroupService service = Mockito.mock(GroupService.class); + Mockito.when(service.getGroupsForUser(Mockito.any(), Mockito.any(Authentication.class))) + .thenReturn(new ArrayList<>()); + + return service; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolverTest.java index bec141bddf260..6ae2fa7dcbf64 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/RollbackIngestionResolverTest.java @@ -1,6 +1,7 @@ package com.linkedin.datahub.graphql.resolvers.ingest.execution; import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.*; +import static org.mockito.ArgumentMatchers.any; import static org.testng.Assert.*; import com.datahub.authentication.Authentication; @@ -46,7 +47,7 @@ public void testGetUnauthorized() throws Exception { assertThrows(RuntimeException.class, () -> resolver.get(mockEnv).join()); Mockito.verify(mockClient, Mockito.times(0)) - .rollbackIngestion(Mockito.eq(RUN_ID), Mockito.any(Authentication.class)); + .rollbackIngestion(Mockito.eq(RUN_ID), any(), any(Authentication.class)); } @Test @@ -58,7 +59,7 @@ public void testRollbackIngestionMethod() throws Exception { resolver.rollbackIngestion(RUN_ID, mockContext).get(); Mockito.verify(mockClient, Mockito.times(1)) - .rollbackIngestion(Mockito.eq(RUN_ID), Mockito.any(Authentication.class)); + .rollbackIngestion(Mockito.eq(RUN_ID), any(), any(Authentication.class)); } @Test @@ -66,7 +67,7 @@ public void testGetEntityClientException() throws Exception { EntityClient mockClient = Mockito.mock(EntityClient.class); Mockito.doThrow(RuntimeException.class) .when(mockClient) - .rollbackIngestion(Mockito.any(), Mockito.any(Authentication.class)); + .rollbackIngestion(any(), any(), any(Authentication.class)); RollbackIngestionResolver resolver = new RollbackIngestionResolver(mockClient); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolverTest.java new file mode 100644 index 0000000000000..73d228d600266 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/UpdateSecretResolverTest.java @@ -0,0 +1,98 @@ +package com.linkedin.datahub.graphql.resolvers.ingest.secret; + +import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.resolvers.ingest.IngestTestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.SECRET_VALUE_ASPECT_NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.when; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.UpdateSecretInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.secret.SecretService; +import com.linkedin.secret.DataHubSecretValue; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class UpdateSecretResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:secret:secret-id"); + + private static final UpdateSecretInput TEST_INPUT = + new UpdateSecretInput(TEST_URN.toString(), "MY_SECRET", "mysecretvalue", "dummy"); + + private DataFetchingEnvironment mockEnv; + private EntityClient mockClient; + private SecretService mockSecretService; + private UpdateSecretResolver resolver; + + @BeforeMethod + public void before() { + mockClient = Mockito.mock(EntityClient.class); + mockSecretService = Mockito.mock(SecretService.class); + + resolver = new UpdateSecretResolver(mockClient, mockSecretService); + } + + private DataHubSecretValue createSecretAspect() { + DataHubSecretValue secretAspect = new DataHubSecretValue(); + secretAspect.setValue("encryptedvalue.updated"); + secretAspect.setName(TEST_INPUT.getName() + ".updated"); + secretAspect.setDescription(TEST_INPUT.getDescription() + ".updated"); + secretAspect.setCreated( + new AuditStamp().setActor(UrnUtils.getUrn("urn:li:corpuser:test")).setTime(0L)); + return secretAspect; + } + + @Test + public void testGetSuccess() throws Exception { + // with valid context + QueryContext mockContext = getMockAllowContext(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Mockito.when(mockClient.exists(any(), any())).thenReturn(true); + Mockito.when(mockSecretService.encrypt(any())).thenReturn("encrypted_value"); + final EntityResponse entityResponse = new EntityResponse(); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put( + SECRET_VALUE_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(createSecretAspect().data()))); + entityResponse.setAspects(aspectMap); + + when(mockClient.getV2(any(), any(), any(), any())).thenReturn(entityResponse); + + // Invoke the resolver + resolver.get(mockEnv).join(); + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal(any(), any(), anyBoolean()); + } + + @Test( + description = "validate if nothing provided throws Exception", + expectedExceptions = {AuthorizationException.class, CompletionException.class}) + public void testGetUnauthorized() throws Exception { + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).join(); + Mockito.verify(mockClient, Mockito.times(0)) + .ingestProposal(any(), any(Authentication.class), anyBoolean()); + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java index 8fc5ab6ebb828..05387123f9c96 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/mutate/MutableTypeBatchResolverTest.java @@ -22,7 +22,6 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.metadata.Constants; import com.linkedin.mxe.MetadataChangeProposal; import graphql.schema.DataFetchingEnvironment; @@ -74,7 +73,7 @@ public class MutableTypeBatchResolverTest { @Test public void testGetSuccess() throws Exception { - EntityClient mockClient = Mockito.mock(RestliEntityClient.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); BatchMutableType batchMutableType = new DatasetType(mockClient); @@ -167,7 +166,7 @@ public void testGetSuccess() throws Exception { @Test public void testGetFailureUnauthorized() throws Exception { - EntityClient mockClient = Mockito.mock(RestliEntityClient.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); BatchMutableType batchMutableType = new DatasetType(mockClient); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java index c7d397c5a4a73..4d56cc3d52af8 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java @@ -15,7 +15,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.filter.Condition; @@ -27,6 +27,7 @@ import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchResultMetadata; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.ViewService; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.view.DataHubViewDefinition; @@ -52,6 +53,7 @@ public static void testApplyViewNullBaseFilter() throws Exception { DataHubViewInfo info = getViewInfo(viewFilter); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); + FormService mockFormService = Mockito.mock(FormService.class); List facets = ImmutableList.of("platform", "domains"); @@ -71,7 +73,7 @@ public static void testApplyViewNullBaseFilter() throws Exception { .setMetadata(new SearchResultMetadata())); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput( @@ -102,6 +104,7 @@ public static void testApplyViewBaseFilter() throws Exception { Filter viewFilter = createFilter("field", "test"); DataHubViewInfo info = getViewInfo(viewFilter); + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); Filter baseFilter = createFilter("baseField.keyword", "baseTest"); @@ -122,7 +125,7 @@ public static void testApplyViewBaseFilter() throws Exception { .setMetadata(new SearchResultMetadata())); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput( @@ -166,6 +169,7 @@ public static void testApplyViewNullBaseEntityTypes() throws Exception { DataHubViewInfo info = getViewInfo(viewFilter); List facets = ImmutableList.of("platform"); + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); EntityClient mockClient = @@ -184,7 +188,7 @@ public static void testApplyViewNullBaseEntityTypes() throws Exception { .setMetadata(new SearchResultMetadata())); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput(null, "", facets, null, TEST_VIEW_URN.toString(), null); @@ -217,6 +221,7 @@ public static void testApplyViewEmptyBaseEntityTypes() throws Exception { DataHubViewInfo info = getViewInfo(viewFilter); List facets = ImmutableList.of(); + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, info); EntityClient mockClient = @@ -235,7 +240,7 @@ public static void testApplyViewEmptyBaseEntityTypes() throws Exception { .setMetadata(new SearchResultMetadata())); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput( @@ -267,6 +272,7 @@ public static void testApplyViewEmptyBaseEntityTypes() throws Exception { public static void testApplyViewViewDoesNotExist() throws Exception { // When a view does not exist, the endpoint should WARN and not apply the view. + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, null); List searchEntityTypes = @@ -290,7 +296,7 @@ public static void testApplyViewViewDoesNotExist() throws Exception { .setMetadata(new SearchResultMetadata())); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput( Collections.emptyList(), "", null, null, TEST_VIEW_URN.toString(), null); @@ -306,6 +312,7 @@ public static void testApplyViewViewDoesNotExist() throws Exception { @Test public static void testErrorFetchingResults() throws Exception { + FormService mockFormService = Mockito.mock(FormService.class); ViewService mockService = initMockViewService(TEST_VIEW_URN, null); EntityClient mockClient = Mockito.mock(EntityClient.class); @@ -322,7 +329,7 @@ public static void testErrorFetchingResults() throws Exception { .thenThrow(new RemoteInvocationException()); final AggregateAcrossEntitiesResolver resolver = - new AggregateAcrossEntitiesResolver(mockClient, mockService); + new AggregateAcrossEntitiesResolver(mockClient, mockService, mockFormService); final AggregateAcrossEntitiesInput testInput = new AggregateAcrossEntitiesInput( Collections.emptyList(), "", null, null, TEST_VIEW_URN.toString(), null); diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java index 29a2b3081aefe..f5accdfb02043 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolverTest.java @@ -9,8 +9,8 @@ import com.linkedin.datahub.graphql.generated.GetQuickFiltersInput; import com.linkedin.datahub.graphql.generated.GetQuickFiltersResult; import com.linkedin.datahub.graphql.generated.QuickFilter; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.AggregationMetadata; diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index d0bbfd126b9b9..0b8c1f1aeb83f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -15,7 +15,7 @@ import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.query.filter.Condition; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java index 909ceeb8f3bab..ff8bd542fbdff 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeCliApplication.java @@ -1,5 +1,7 @@ package com.linkedin.datahub.upgrade; +import com.linkedin.gms.factory.auth.AuthorizerChainFactory; +import com.linkedin.gms.factory.auth.DataHubAuthorizerFactory; import com.linkedin.gms.factory.telemetry.ScheduledAnalyticsFactory; import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -19,7 +21,11 @@ excludeFilters = { @ComponentScan.Filter( type = FilterType.ASSIGNABLE_TYPE, - classes = ScheduledAnalyticsFactory.class) + classes = { + ScheduledAnalyticsFactory.class, + AuthorizerChainFactory.class, + DataHubAuthorizerFactory.class + }) }) public class UpgradeCliApplication { public static void main(String[] args) { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java index dd6c3fd1e44aa..4be39ac3c4bfc 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSDisableWriteModeStep.java @@ -4,14 +4,16 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import java.util.function.Function; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +@Slf4j @RequiredArgsConstructor public class GMSDisableWriteModeStep implements UpgradeStep { - private final SystemRestliEntityClient _entityClient; + private final SystemEntityClient entityClient; @Override public String id() { @@ -27,9 +29,9 @@ public int retryCount() { public Function executable() { return (context) -> { try { - _entityClient.setWritable(false); + entityClient.setWritable(false); } catch (Exception e) { - e.printStackTrace(); + log.error("Failed to turn write mode off in GMS", e); context.report().addLine("Failed to turn write mode off in GMS"); return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java index 8a0d374d6ee3e..09713dc78ee27 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/common/steps/GMSEnableWriteModeStep.java @@ -4,13 +4,15 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import java.util.function.Function; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +@Slf4j @RequiredArgsConstructor public class GMSEnableWriteModeStep implements UpgradeStep { - private final SystemRestliEntityClient _entityClient; + private final SystemEntityClient entityClient; @Override public String id() { @@ -26,9 +28,9 @@ public int retryCount() { public Function executable() { return (context) -> { try { - _entityClient.setWritable(true); + entityClient.setWritable(true); } catch (Exception e) { - e.printStackTrace(); + log.error("Failed to turn write mode back on in GMS", e); context.report().addLine("Failed to turn write mode back on in GMS"); return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java index abd144bf453ed..406963c58fd71 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BackfillBrowsePathsV2Config.java @@ -11,7 +11,7 @@ public class BackfillBrowsePathsV2Config { @Bean public BackfillBrowsePathsV2 backfillBrowsePathsV2( - EntityService entityService, SearchService searchService) { + EntityService entityService, SearchService searchService) { return new BackfillBrowsePathsV2(entityService, searchService); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java index 1e9298bc60612..caa45988733df 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BuildIndicesConfig.java @@ -3,7 +3,9 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.BuildIndices; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.graph.GraphService; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -20,7 +22,9 @@ public BuildIndices buildIndices( final GraphService graphService, final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, - final ConfigurationProvider configurationProvider) { + final ConfigurationProvider configurationProvider, + final AspectDao aspectDao, + final EntityRegistry entityRegistry) { return new BuildIndices( systemMetadataService, @@ -28,6 +32,8 @@ public BuildIndices buildIndices( entitySearchService, graphService, baseElasticSearchComponents, - configurationProvider); + configurationProvider, + aspectDao, + entityRegistry); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java index d968e8521867e..741aeece1cf62 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java @@ -1,7 +1,7 @@ package com.linkedin.datahub.upgrade.config; import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; import io.ebean.Database; @@ -21,14 +21,13 @@ public class NoCodeUpgradeConfig { @Autowired ApplicationContext applicationContext; @Bean(name = "noCodeUpgrade") - @DependsOn({"ebeanServer", "entityService", "systemRestliEntityClient", "entityRegistry"}) + @DependsOn({"ebeanServer", "entityService", "systemEntityClient", "entityRegistry"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull public NoCodeUpgrade createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); - final EntityService entityService = applicationContext.getBean(EntityService.class); - final SystemRestliEntityClient entityClient = - applicationContext.getBean(SystemRestliEntityClient.class); + final EntityService entityService = applicationContext.getBean(EntityService.class); + final SystemEntityClient entityClient = applicationContext.getBean(SystemEntityClient.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); return new NoCodeUpgrade(ebeanServer, entityService, entityRegistry, entityClient); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java index 0b46133209382..5bf1241e21305 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RemoveUnknownAspectsConfig.java @@ -8,7 +8,7 @@ @Configuration public class RemoveUnknownAspectsConfig { @Bean(name = "removeUnknownAspects") - public RemoveUnknownAspects removeUnknownAspects(EntityService entityService) { + public RemoveUnknownAspects removeUnknownAspects(EntityService entityService) { return new RemoveUnknownAspects(entityService); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java index 116d62878f5c6..ec6e5a4a8f04d 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java @@ -1,7 +1,7 @@ package com.linkedin.datahub.upgrade.config; import com.linkedin.datahub.upgrade.restorebackup.RestoreBackup; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -25,7 +25,7 @@ public class RestoreBackupConfig { @DependsOn({ "ebeanServer", "entityService", - "systemRestliEntityClient", + "systemEntityClient", "graphService", "searchService", "entityRegistry" @@ -34,9 +34,8 @@ public class RestoreBackupConfig { @Nonnull public RestoreBackup createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); - final EntityService entityService = applicationContext.getBean(EntityService.class); - final SystemRestliEntityClient entityClient = - applicationContext.getBean(SystemRestliEntityClient.class); + final EntityService entityService = applicationContext.getBean(EntityService.class); + final SystemEntityClient entityClient = applicationContext.getBean(SystemEntityClient.class); final GraphService graphClient = applicationContext.getBean(GraphService.class); final EntitySearchService searchClient = applicationContext.getBean(EntitySearchService.class); final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java index 9d229f315d709..008bdf5cfac38 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java @@ -3,7 +3,6 @@ import com.linkedin.datahub.upgrade.restoreindices.RestoreIndices; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import io.ebean.Database; import javax.annotation.Nonnull; @@ -21,19 +20,17 @@ public class RestoreIndicesConfig { @Autowired ApplicationContext applicationContext; @Bean(name = "restoreIndices") - @DependsOn({"ebeanServer", "entityService", "searchService", "graphService", "entityRegistry"}) + @DependsOn({"ebeanServer", "entityService", "searchService", "graphService"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull public RestoreIndices createInstance() { final Database ebeanServer = applicationContext.getBean(Database.class); - final EntityService entityService = applicationContext.getBean(EntityService.class); + final EntityService entityService = applicationContext.getBean(EntityService.class); final EntitySearchService entitySearchService = applicationContext.getBean(EntitySearchService.class); final GraphService graphService = applicationContext.getBean(GraphService.class); - final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class); - return new RestoreIndices( - ebeanServer, entityService, entityRegistry, entitySearchService, graphService); + return new RestoreIndices(ebeanServer, entityService, entitySearchService, graphService); } @Bean(name = "restoreIndices") @@ -41,6 +38,6 @@ public RestoreIndices createInstance() { @Nonnull public RestoreIndices createNotImplInstance() { log.warn("restoreIndices is not supported for cassandra!"); - return new RestoreIndices(null, null, null, null, null); + return new RestoreIndices(null, null, null, null); } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeContext.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeContext.java index 6cc94fbed5bf3..57e16eb72d025 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeContext.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeContext.java @@ -8,49 +8,33 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import lombok.Getter; +import lombok.experimental.Accessors; +@Getter +@Accessors(fluent = true) public class DefaultUpgradeContext implements UpgradeContext { - private final Upgrade _upgrade; - private final UpgradeReport _report; - private final List _previousStepResults; - private final List _args; - private final Map> _parsedArgs; + private final Upgrade upgrade; + private final UpgradeReport report; + private final List previousStepResults; + private final List args; + private final Map> parsedArgs; DefaultUpgradeContext( Upgrade upgrade, UpgradeReport report, List previousStepResults, List args) { - _upgrade = upgrade; - _report = report; - _previousStepResults = previousStepResults; - _args = args; - _parsedArgs = UpgradeUtils.parseArgs(args); - } - - @Override - public Upgrade upgrade() { - return _upgrade; + this.upgrade = upgrade; + this.report = report; + this.previousStepResults = previousStepResults; + this.args = args; + this.parsedArgs = UpgradeUtils.parseArgs(args); } @Override public List stepResults() { - return _previousStepResults; - } - - @Override - public UpgradeReport report() { - return _report; - } - - @Override - public List args() { - return _args; - } - - @Override - public Map> parsedArgs() { - return _parsedArgs; + return previousStepResults; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java index 623c8a71e861d..bddf53a274905 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java @@ -16,7 +16,9 @@ import java.util.List; import java.util.Map; import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class DefaultUpgradeManager implements UpgradeManager { private final Map _upgrades = new HashMap<>(); @@ -137,6 +139,7 @@ private UpgradeStepResult executeStepInternal(UpgradeContext context, UpgradeSte break; } } catch (Exception e) { + log.error("Caught exception during attempt {} of Step with id {}", i, step.id(), e); context .report() .addLine( diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java index 674efb2b8ba78..1524a015e414e 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java @@ -6,7 +6,7 @@ import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSQualificationStep; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.models.registry.EntityRegistry; import io.ebean.Database; @@ -28,9 +28,9 @@ public class NoCodeUpgrade implements Upgrade { // Upgrade requires the Database. public NoCodeUpgrade( @Nullable final Database server, - final EntityService entityService, + final EntityService entityService, final EntityRegistry entityRegistry, - final SystemRestliEntityClient entityClient) { + final SystemEntityClient entityClient) { if (server != null) { _steps = buildUpgradeSteps(server, entityService, entityRegistry, entityClient); _cleanupSteps = buildCleanupSteps(); @@ -61,9 +61,9 @@ private List buildCleanupSteps() { private List buildUpgradeSteps( final Database server, - final EntityService entityService, + final EntityService entityService, final EntityRegistry entityRegistry, - final SystemRestliEntityClient entityClient) { + final SystemEntityClient entityClient) { final List steps = new ArrayList<>(); steps.add(new RemoveAspectV2TableStep(server)); steps.add(new GMSQualificationStep(ImmutableMap.of("noCode", "true"))); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java index 7e55dcddc639f..74d97767d1c39 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveClientIdAspectStep.java @@ -17,7 +17,7 @@ public class RemoveClientIdAspectStep implements UpgradeStep { private static final String INVALID_CLIENT_ID_ASPECT = "clientId"; - private final EntityService _entityService; + private final EntityService _entityService; @Override public String id() { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java index dc95b7605ef88..3ea449051b355 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/removeunknownaspects/RemoveUnknownAspects.java @@ -12,7 +12,7 @@ public class RemoveUnknownAspects implements Upgrade { private final List _steps; - public RemoveUnknownAspects(final EntityService entityService) { + public RemoveUnknownAspects(final EntityService entityService) { _steps = buildSteps(entityService); } @@ -26,7 +26,7 @@ public List steps() { return _steps; } - private List buildSteps(final EntityService entityService) { + private List buildSteps(final EntityService entityService) { final List steps = new ArrayList<>(); steps.add(new RemoveClientIdAspectStep(entityService)); return steps; diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java index 4ac295b4fdfb7..bcaeaa34e8936 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java @@ -8,7 +8,7 @@ import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; import com.linkedin.datahub.upgrade.common.steps.GMSDisableWriteModeStep; import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -24,9 +24,9 @@ public class RestoreBackup implements Upgrade { public RestoreBackup( @Nullable final Database server, - final EntityService entityService, + final EntityService entityService, final EntityRegistry entityRegistry, - final SystemRestliEntityClient entityClient, + final SystemEntityClient entityClient, final GraphService graphClient, final EntitySearchService searchClient) { if (server != null) { @@ -50,9 +50,9 @@ public List steps() { private List buildSteps( final Database server, - final EntityService entityService, + final EntityService entityService, final EntityRegistry entityRegistry, - final SystemRestliEntityClient entityClient, + final SystemEntityClient entityClient, final GraphService graphClient, final EntitySearchService searchClient) { final List steps = new ArrayList<>(); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java index 5c4e8cdc47e34..c756407832a36 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java @@ -47,7 +47,7 @@ public class RestoreStorageStep implements UpgradeStep { private final ExecutorService _gmsThreadPool; public RestoreStorageStep( - final EntityService entityService, final EntityRegistry entityRegistry) { + final EntityService entityService, final EntityRegistry entityRegistry) { _entityService = entityService; _entityRegistry = entityRegistry; _backupReaders = ImmutableBiMap.of(LocalParquetReader.READER_NAME, LocalParquetReader.class); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/backupreader/BackupReader.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/backupreader/BackupReader.java index 212f0da9f592d..c6839c0e63f05 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/backupreader/BackupReader.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/backupreader/BackupReader.java @@ -9,6 +9,7 @@ * Strings */ public interface BackupReader { + String getName(); @Nonnull diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java index f46bb9b05624d..9bc42e23a9974 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java @@ -8,7 +8,6 @@ import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; import io.ebean.Database; import java.util.ArrayList; @@ -32,12 +31,11 @@ public class RestoreIndices implements Upgrade { public RestoreIndices( @Nullable final Database server, - final EntityService entityService, - final EntityRegistry entityRegistry, + final EntityService entityService, final EntitySearchService entitySearchService, final GraphService graphService) { if (server != null) { - _steps = buildSteps(server, entityService, entityRegistry, entitySearchService, graphService); + _steps = buildSteps(server, entityService, entitySearchService, graphService); } else { _steps = List.of(); } @@ -55,14 +53,13 @@ public List steps() { private List buildSteps( final Database server, - final EntityService entityService, - final EntityRegistry entityRegistry, + final EntityService entityService, final EntitySearchService entitySearchService, final GraphService graphService) { final List steps = new ArrayList<>(); steps.add(new ClearSearchServiceStep(entitySearchService, false)); steps.add(new ClearGraphServiceStep(graphService, false)); - steps.add(new SendMAEStep(server, entityService, entityRegistry)); + steps.add(new SendMAEStep(server, entityService)); return steps; } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java index bedf200a1c055..aca27892d2e3a 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/SendMAEStep.java @@ -10,7 +10,6 @@ import com.linkedin.metadata.entity.ebean.EbeanAspectV2; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; -import com.linkedin.metadata.models.registry.EntityRegistry; import io.ebean.Database; import io.ebean.ExpressionList; import java.util.ArrayList; @@ -23,7 +22,9 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class SendMAEStep implements UpgradeStep { private static final int DEFAULT_BATCH_SIZE = 1000; @@ -51,10 +52,7 @@ public RestoreIndicesResult call() { } } - public SendMAEStep( - final Database server, - final EntityService entityService, - final EntityRegistry entityRegistry) { + public SendMAEStep(final Database server, final EntityService entityService) { _server = server; _entityService = entityService; } @@ -77,7 +75,7 @@ private List iterateFutures(List iterateFutures(List indexedServices = Stream.of(graphService, entitySearchService, systemMetadataService, timeseriesAspectService) @@ -36,7 +40,13 @@ public BuildIndices( .map(service -> (ElasticSearchIndexed) service) .collect(Collectors.toList()); - _steps = buildSteps(indexedServices, baseElasticSearchComponents, configurationProvider); + _steps = + buildSteps( + indexedServices, + baseElasticSearchComponents, + configurationProvider, + aspectDao, + entityRegistry); } @Override @@ -53,13 +63,19 @@ private List buildSteps( final List indexedServices, final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents baseElasticSearchComponents, - final ConfigurationProvider configurationProvider) { + final ConfigurationProvider configurationProvider, + final AspectDao aspectDao, + final EntityRegistry entityRegistry) { final List steps = new ArrayList<>(); // Disable ES write mode/change refresh rate and clone indices steps.add( new BuildIndicesPreStep( - baseElasticSearchComponents, indexedServices, configurationProvider)); + baseElasticSearchComponents, + indexedServices, + configurationProvider, + aspectDao, + entityRegistry)); // Configure graphService, entitySearchService, systemMetadataService, timeseriesAspectService steps.add(new BuildIndicesStep(indexedServices)); // Reset configuration (and delete clones? Or just do this regularly? Or delete clone in diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java index c25888be07f89..894075417a349 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/steps/BuildIndicesPreStep.java @@ -2,6 +2,8 @@ import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.INDEX_BLOCKS_WRITE_SETTING; import static com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils.getAllReindexConfigs; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; import com.google.common.collect.ImmutableMap; import com.linkedin.datahub.upgrade.UpgradeContext; @@ -11,8 +13,12 @@ import com.linkedin.datahub.upgrade.system.elasticsearch.util.IndexUtils; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory; +import com.linkedin.metadata.entity.AspectDao; +import com.linkedin.metadata.entity.EntityUtils; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; import java.io.IOException; import java.util.List; import java.util.Map; @@ -31,6 +37,8 @@ public class BuildIndicesPreStep implements UpgradeStep { private final BaseElasticSearchComponentsFactory.BaseElasticSearchComponents _esComponents; private final List _services; private final ConfigurationProvider _configurationProvider; + private final AspectDao _aspectDao; + private final EntityRegistry _entityRegistry; @Override public String id() { @@ -46,9 +54,28 @@ public int retryCount() { public Function executable() { return (context) -> { try { + List reindexConfigs = + _configurationProvider.getStructuredProperties().isSystemUpdateEnabled() + ? getAllReindexConfigs( + _services, + _aspectDao + .streamAspects( + STRUCTURED_PROPERTY_ENTITY_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .map( + entityAspect -> + EntityUtils.toAspectRecord( + STRUCTURED_PROPERTY_ENTITY_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + entityAspect.getMetadata(), + _entityRegistry)) + .map(recordTemplate -> (StructuredPropertyDefinition) recordTemplate) + .collect(Collectors.toSet())) + : getAllReindexConfigs(_services); + // Get indices to update List indexConfigs = - getAllReindexConfigs(_services).stream() + reindexConfigs.stream() .filter(ReindexConfig::requiresReindex) .collect(Collectors.toList()); diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java index b3de7c503fb3e..52b34200991c3 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/elasticsearch/util/IndexUtils.java @@ -2,8 +2,10 @@ import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; @@ -39,6 +41,23 @@ public static List getAllReindexConfigs( return reindexConfigs; } + public static List getAllReindexConfigs( + List elasticSearchIndexedList, + Collection structuredProperties) + throws IOException { + // Avoid locking & reprocessing + List reindexConfigs = new ArrayList<>(_reindexConfigs); + if (reindexConfigs.isEmpty()) { + for (ElasticSearchIndexed elasticSearchIndexed : elasticSearchIndexedList) { + reindexConfigs.addAll( + elasticSearchIndexed.buildReindexConfigsWithAllStructProps(structuredProperties)); + } + _reindexConfigs = new ArrayList<>(reindexConfigs); + } + + return reindexConfigs; + } + public static boolean validateWriteBlock( RestHighLevelClient esClient, String indexName, boolean expectedState) throws IOException, InterruptedException { diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java index 03f0b0b7f2ec2..4b9fc5bba0204 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2.java @@ -11,7 +11,7 @@ public class BackfillBrowsePathsV2 implements Upgrade { private final List _steps; - public BackfillBrowsePathsV2(EntityService entityService, SearchService searchService) { + public BackfillBrowsePathsV2(EntityService entityService, SearchService searchService) { _steps = ImmutableList.of(new BackfillBrowsePathsV2Step(entityService, searchService)); } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java index 610d9069337a5..9a426369cfb02 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java @@ -54,10 +54,10 @@ public class BackfillBrowsePathsV2Step implements UpgradeStep { Constants.ML_FEATURE_ENTITY_NAME); private static final Integer BATCH_SIZE = 5000; - private final EntityService _entityService; + private final EntityService _entityService; private final SearchService _searchService; - public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { + public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { _searchService = searchService; _entityService = entityService; } diff --git a/datahub-upgrade/src/main/resources/application.properties b/datahub-upgrade/src/main/resources/application.properties new file mode 100644 index 0000000000000..b884c92f74bd4 --- /dev/null +++ b/datahub-upgrade/src/main/resources/application.properties @@ -0,0 +1,5 @@ +management.health.elasticsearch.enabled=false +management.health.neo4j.enabled=false +ingestion.enabled=false +spring.main.allow-bean-definition-overriding=true +entityClient.impl=restli diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java index 0e7bf5ddd5250..be28b7f739cf5 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java @@ -20,7 +20,7 @@ public class UpgradeCliApplicationTestConfiguration { @MockBean private Database ebeanServer; - @MockBean private EntityService _entityService; + @MockBean private EntityService _entityService; @MockBean private SearchService searchService; diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index c0355b935137a..05af6871715ce 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -117,7 +117,6 @@ task cleanExtraDirs { delete 'dist' delete 'tmp' delete 'just' - delete fileTree('../datahub-frontend/public') delete fileTree(dir: 'src', include: '*.generated.ts') } clean.finalizedBy(cleanExtraDirs) diff --git a/datahub-web-react/index.html b/datahub-web-react/index.html index 9490881246e12..bb86e2f350e1a 100644 --- a/datahub-web-react/index.html +++ b/datahub-web-react/index.html @@ -2,7 +2,8 @@ - + + diff --git a/datahub-web-react/public/assets/favicon.ico b/datahub-web-react/public/assets/icons/favicon.ico similarity index 100% rename from datahub-web-react/public/assets/favicon.ico rename to datahub-web-react/public/assets/icons/favicon.ico diff --git a/datahub-web-react/public/assets/logo.png b/datahub-web-react/public/assets/logo.png deleted file mode 100644 index 5e34e6425d23f..0000000000000 Binary files a/datahub-web-react/public/assets/logo.png and /dev/null differ diff --git a/datahub-web-react/public/assets/logo.png b/datahub-web-react/public/assets/logo.png new file mode 120000 index 0000000000000..c570fd37bed97 --- /dev/null +++ b/datahub-web-react/public/assets/logo.png @@ -0,0 +1 @@ +logos/datahub-logo.png \ No newline at end of file diff --git a/datahub-web-react/public/assets/logos/datahub-logo.png b/datahub-web-react/public/assets/logos/datahub-logo.png new file mode 100644 index 0000000000000..5e34e6425d23f Binary files /dev/null and b/datahub-web-react/public/assets/logos/datahub-logo.png differ diff --git a/datahub-web-react/public/browserconfig.xml b/datahub-web-react/public/browserconfig.xml new file mode 100644 index 0000000000000..0f5fd50ca7ce4 --- /dev/null +++ b/datahub-web-react/public/browserconfig.xml @@ -0,0 +1,9 @@ + + + + + + #020d10 + + + diff --git a/datahub-web-react/public/manifest.json b/datahub-web-react/public/manifest.json index 35dad30b4bb57..1ff1cb2a1f269 100644 --- a/datahub-web-react/public/manifest.json +++ b/datahub-web-react/public/manifest.json @@ -3,7 +3,7 @@ "name": "DataHub", "icons": [ { - "src": "/assets/favicon.ico", + "src": "/assets/icons/favicon.ico", "sizes": "64x64 32x32 24x24 16x16", "type": "image/x-icon" } diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index b32b296af38c5..03d6f4a624c3d 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -88,6 +88,7 @@ export const user1 = { editableInfo: null, properties: null, editableProperties: null, + autoRenderAspects: [], }; const user2 = { @@ -295,6 +296,7 @@ export const dataset1 = { statsSummary: null, embed: null, browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' }, + autoRenderAspects: [], }; export const dataset2 = { @@ -390,6 +392,7 @@ export const dataset2 = { statsSummary: null, embed: null, browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' }, + autoRenderAspects: [], }; export const dataset3 = { @@ -595,7 +598,7 @@ export const dataset3 = { viewProperties: null, autoRenderAspects: [ { - __typename: 'AutoRenderAspect', + __typename: 'RawAspect', aspectName: 'autoRenderAspect', payload: '{ "values": [{ "autoField1": "autoValue1", "autoField2": "autoValue2" }] }', renderSpec: { @@ -962,6 +965,7 @@ export const container1 = { externalUrl: null, __typename: 'ContainerProperties', }, + autoRenderAspects: [], __typename: 'Container', } as Container; @@ -976,6 +980,7 @@ export const container2 = { externalUrl: null, __typename: 'ContainerProperties', }, + autoRenderAspects: [], __typename: 'Container', } as Container; @@ -1023,6 +1028,7 @@ export const glossaryTerm1 = { }, parentNodes: null, deprecation: null, + autoRenderAspects: [], } as GlossaryTerm; const glossaryTerm2 = { @@ -1095,6 +1101,7 @@ const glossaryTerm2 = { __typename: 'EntityRelationshipsResult', }, parentNodes: null, + autoRenderAspects: [], __typename: 'GlossaryTerm', }; @@ -1161,6 +1168,7 @@ const glossaryTerm3 = { __typename: 'GlossaryRelatedTerms', }, deprecation: null, + autoRenderAspects: [], __typename: 'GlossaryTerm', } as GlossaryTerm; @@ -1257,6 +1265,7 @@ export const sampleTag = { description: 'sample tag description', colorHex: 'sample tag color', }, + autoRenderAspects: [], }; export const dataFlow1 = { @@ -1328,6 +1337,7 @@ export const dataFlow1 = { }, domain: null, deprecation: null, + autoRenderAspects: [], } as DataFlow; export const dataJob1 = { @@ -1414,6 +1424,7 @@ export const dataJob1 = { domain: null, status: null, deprecation: null, + autoRenderAspects: [], } as DataJob; export const dataJob2 = { @@ -1483,6 +1494,7 @@ export const dataJob2 = { upstream: null, downstream: null, deprecation: null, + autoRenderAspects: [], } as DataJob; export const dataJob3 = { @@ -1555,6 +1567,7 @@ export const dataJob3 = { downstream: null, status: null, deprecation: null, + autoRenderAspects: [], } as DataJob; export const mlModel = { @@ -1636,6 +1649,7 @@ export const mlModel = { downstream: null, status: null, deprecation: null, + autoRenderAspects: [], } as MlModel; export const dataset1FetchedEntity = { diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 2734026400933..dd670b35d49e0 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -48,6 +48,7 @@ export enum EventType { CreateResetCredentialsLinkEvent, DeleteEntityEvent, SelectUserRoleEvent, + SelectGroupRoleEvent, BatchSelectUserRoleEvent, CreatePolicyEvent, UpdatePolicyEvent, @@ -412,6 +413,12 @@ export interface SelectUserRoleEvent extends BaseEvent { userUrn: string; } +export interface SelectGroupRoleEvent extends BaseEvent { + type: EventType.SelectGroupRoleEvent; + roleUrn: string; + groupUrn?: string; +} + export interface BatchSelectUserRoleEvent extends BaseEvent { type: EventType.BatchSelectUserRoleEvent; roleUrn: string; @@ -668,6 +675,7 @@ export type Event = | CreateResetCredentialsLinkEvent | DeleteEntityEvent | SelectUserRoleEvent + | SelectGroupRoleEvent | BatchSelectUserRoleEvent | CreatePolicyEvent | UpdatePolicyEvent diff --git a/datahub-web-react/src/app/identity/group/AssignRoletoGroupConfirmation.tsx b/datahub-web-react/src/app/identity/group/AssignRoletoGroupConfirmation.tsx new file mode 100644 index 0000000000000..f08b607222de6 --- /dev/null +++ b/datahub-web-react/src/app/identity/group/AssignRoletoGroupConfirmation.tsx @@ -0,0 +1,67 @@ +import React from 'react'; +import { message, Popconfirm } from 'antd'; +import { useBatchAssignRoleMutation } from '../../../graphql/mutations.generated'; +import { DataHubRole } from '../../../types.generated'; +import analytics, { EventType } from '../../analytics'; + +type Props = { + visible: boolean; + roleToAssign: DataHubRole | undefined; + groupName: string; + groupUrn: string; + onClose: () => void; + onConfirm: () => void; +}; + +export default function AssignRoletoGroupConfirmation({ + visible, + roleToAssign, + groupName, + groupUrn, + onClose, + onConfirm, +}: Props) { + const [batchAssignRoleMutation] = useBatchAssignRoleMutation(); + // eslint-disable-next-line + const batchAssignRole = () => { + batchAssignRoleMutation({ + variables: { + input: { + roleUrn: roleToAssign?.urn, + actors: [groupUrn], + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + analytics.event({ + type: EventType.SelectGroupRoleEvent, + roleUrn: roleToAssign?.urn || 'undefined', + groupUrn, + }); + message.success({ + content: roleToAssign + ? `Assigned role ${roleToAssign?.name} to group ${groupName}!` + : `Removed role from user ${groupName}!`, + duration: 2, + }); + onConfirm(); + } + }) + .catch((e) => { + message.destroy(); + message.error({ + content: roleToAssign + ? `Failed to assign role ${roleToAssign?.name} to group ${groupName}: \n ${e.message || ''}` + : `Failed to remove role from group ${groupName}: \n ${e.message || ''}`, + duration: 3, + }); + }); + }; + + const assignRoleText = roleToAssign + ? `Would you like to assign the role ${roleToAssign?.name} to group ${groupName}?` + : `Would you like to remove group ${groupName}'s existing role?`; + + return ; +} diff --git a/datahub-web-react/src/app/identity/group/GroupList.tsx b/datahub-web-react/src/app/identity/group/GroupList.tsx index 788b9eccafc0a..a8ebbedc2ac6d 100644 --- a/datahub-web-react/src/app/identity/group/GroupList.tsx +++ b/datahub-web-react/src/app/identity/group/GroupList.tsx @@ -4,7 +4,7 @@ import styled from 'styled-components'; import { useLocation } from 'react-router'; import * as QueryString from 'query-string'; import { UsergroupAddOutlined } from '@ant-design/icons'; -import { CorpGroup } from '../../../types.generated'; +import { CorpGroup, DataHubRole } from '../../../types.generated'; import { Message } from '../../shared/Message'; import { useListGroupsQuery } from '../../../graphql/group.generated'; import GroupListItem from './GroupListItem'; @@ -16,6 +16,7 @@ import { scrollToTop } from '../../shared/searchUtils'; import { GROUPS_CREATE_GROUP_ID, GROUPS_INTRO_ID } from '../../onboarding/config/GroupsOnboardingConfig'; import { OnboardingTour } from '../../onboarding/OnboardingTour'; import { addGroupToListGroupsCache, DEFAULT_GROUP_LIST_PAGE_SIZE, removeGroupFromListGroupsCache } from './cacheUtils'; +import { useListRolesQuery } from '../../../graphql/role.generated'; const GroupContainer = styled.div` display: flex; @@ -53,7 +54,13 @@ export const GroupList = () => { const pageSize = DEFAULT_GROUP_LIST_PAGE_SIZE; const start = (page - 1) * pageSize; - const { loading, error, data, refetch, client } = useListGroupsQuery({ + const { + loading, + error, + data, + refetch: groupRefetch, + client, + } = useListGroupsQuery({ variables: { input: { start, @@ -76,6 +83,18 @@ export const GroupList = () => { removeGroupFromListGroupsCache(urn, client, page, pageSize); }; + const { data: rolesData } = useListRolesQuery({ + fetchPolicy: 'cache-first', + variables: { + input: { + start: 0, + count: 10, + }, + }, + }); + + const selectRoleOptions = rolesData?.listRoles?.roles?.map((role) => role as DataHubRole) || []; + return ( <> @@ -114,7 +133,12 @@ export const GroupList = () => { }} dataSource={groups} renderItem={(item: any) => ( - handleDelete(item.urn)} group={item as CorpGroup} /> + handleDelete(item.urn)} + group={item as CorpGroup} + selectRoleOptions={selectRoleOptions} + refetch={groupRefetch} + /> )} /> @@ -131,9 +155,9 @@ export const GroupList = () => { {isCreatingGroup && ( setIsCreatingGroup(false)} - onCreate={(group) => { + onCreate={(group: CorpGroup) => { addGroupToListGroupsCache(group, client); - setTimeout(() => refetch(), 3000); + setTimeout(() => groupRefetch(), 3000); }} /> )} diff --git a/datahub-web-react/src/app/identity/group/GroupListItem.tsx b/datahub-web-react/src/app/identity/group/GroupListItem.tsx index 40c5afbbda5ef..74c0a8afb4d02 100644 --- a/datahub-web-react/src/app/identity/group/GroupListItem.tsx +++ b/datahub-web-react/src/app/identity/group/GroupListItem.tsx @@ -3,16 +3,19 @@ import React from 'react'; import styled from 'styled-components'; import { List, Tag, Tooltip, Typography } from 'antd'; import { Link } from 'react-router-dom'; -import { CorpGroup, EntityType, OriginType } from '../../../types.generated'; +import { CorpGroup, DataHubRole, EntityType, OriginType } from '../../../types.generated'; import CustomAvatar from '../../shared/avatar/CustomAvatar'; import { useEntityRegistry } from '../../useEntityRegistry'; import EntityDropdown from '../../entity/shared/EntityDropdown'; import { EntityMenuItems } from '../../entity/shared/EntityDropdown/EntityDropdown'; import { getElasticCappedTotalValueText } from '../../entity/shared/constants'; +import SelectRoleGroup from './SelectRoleGroup'; type Props = { group: CorpGroup; onDelete?: () => void; + selectRoleOptions: Array; + refetch?: () => void; }; const GroupItemContainer = styled.div` @@ -35,11 +38,16 @@ const GroupItemButtonGroup = styled.div` align-items: center; `; -export default function GroupListItem({ group, onDelete }: Props) { +export default function GroupListItem({ group, onDelete, selectRoleOptions, refetch }: Props) { const entityRegistry = useEntityRegistry(); const displayName = entityRegistry.getDisplayName(EntityType.CorpGroup, group); const isExternalGroup: boolean = group.origin?.type === OriginType.External; const externalGroupType: string = group.origin?.externalType || 'outside DataHub'; + const castedCorpUser = group as any; + const groupRelationships = castedCorpUser?.roles?.relationships; + const userRole = + groupRelationships && groupRelationships.length > 0 && (groupRelationships[0]?.entity as DataHubRole); + const groupRoleUrn = userRole && userRole.urn; return ( @@ -66,6 +74,12 @@ export default function GroupListItem({ group, onDelete }: Props) { )} + ; + refetch?: () => void; +}; + +const RoleSelect = styled(Select)<{ color?: string }>` + min-width: 105px; + ${(props) => (props.color ? ` color: ${props.color};` : '')} +`; + +const RoleIcon = styled.span` + margin-right: 6px; + font-size: 12px; +`; + +export default function SelectRoleGroup({ group, groupRoleUrn, selectRoleOptions, refetch }: Props) { + const client = useApolloClient(); + const rolesMap: Map = new Map(); + selectRoleOptions.forEach((role) => { + rolesMap.set(role.urn, role); + }); + const allSelectRoleOptions = [{ urn: NO_ROLE_URN, name: NO_ROLE_TEXT }, ...selectRoleOptions]; + const selectOptions = allSelectRoleOptions.map((role) => { + return ( + + {mapRoleIcon(role.name)} + {role.name} + + ); + }); + + const defaultRoleUrn = groupRoleUrn || NO_ROLE_URN; + const [currentRoleUrn, setCurrentRoleUrn] = useState(defaultRoleUrn); + const [isViewingAssignRole, setIsViewingAssignRole] = useState(false); + + useEffect(() => { + setCurrentRoleUrn(defaultRoleUrn); + }, [defaultRoleUrn]); + + const onSelectRole = (roleUrn: string) => { + setCurrentRoleUrn(roleUrn); + setIsViewingAssignRole(true); + }; + + const onCancel = () => { + setCurrentRoleUrn(defaultRoleUrn); + setIsViewingAssignRole(false); + }; + + const onConfirm = () => { + setIsViewingAssignRole(false); + setTimeout(() => { + refetch?.(); + clearRoleListCache(client); // Update roles. + }, 3000); + }; + + // wait for available roles to load + if (!selectRoleOptions.length) return null; + + return ( + <> + + + {NO_ROLE_TEXT} + + } + value={currentRoleUrn} + onChange={(e) => onSelectRole(e as string)} + color={currentRoleUrn === NO_ROLE_URN ? ANTD_GRAY[6] : undefined} + > + {selectOptions} + + + + ); +} diff --git a/datahub-web-react/src/app/identity/group/cacheUtils.ts b/datahub-web-react/src/app/identity/group/cacheUtils.ts index d4ecd40a40a97..272b9f841d25c 100644 --- a/datahub-web-react/src/app/identity/group/cacheUtils.ts +++ b/datahub-web-react/src/app/identity/group/cacheUtils.ts @@ -44,6 +44,7 @@ const createFullGroup = (baseGroup) => { email: null, }, memberCount: null, + roles: null, }; }; diff --git a/datahub-web-react/src/app/settings/posts/CreatePostModal.tsx b/datahub-web-react/src/app/settings/posts/CreatePostModal.tsx index 10c4ee880fe85..2a3e2204f2392 100644 --- a/datahub-web-react/src/app/settings/posts/CreatePostModal.tsx +++ b/datahub-web-react/src/app/settings/posts/CreatePostModal.tsx @@ -13,6 +13,7 @@ import { useEnterKeyListener } from '../../shared/useEnterKeyListener'; import { MediaType, PostContentType, PostType } from '../../../types.generated'; import { useCreatePostMutation, useUpdatePostMutation } from '../../../graphql/mutations.generated'; import { PostEntry } from './PostsListColumns'; +import handleGraphQLError from '../../shared/handleGraphQLError'; type Props = { editData: PostEntry; @@ -84,10 +85,12 @@ export default function CreatePostModal({ onClose, onCreate, editData, onEdit }: form.resetFields(); } }) - .catch((e) => { - message.destroy(); - message.error({ content: 'Failed to create Post! An unknown error occured.', duration: 3 }); - console.error('Failed to create Post:', e.message); + .catch((error) => { + handleGraphQLError({ + error, + defaultMessage: 'Failed to create Post! An unexpected error occurred', + permissionMessage: 'Unauthorized to create Post. Please contact your DataHub administrator.', + }); }); onClose(); }; diff --git a/datahub-web-react/src/app/settings/posts/PostItemMenu.tsx b/datahub-web-react/src/app/settings/posts/PostItemMenu.tsx index 3708c04ab1ad3..10e2996c36f69 100644 --- a/datahub-web-react/src/app/settings/posts/PostItemMenu.tsx +++ b/datahub-web-react/src/app/settings/posts/PostItemMenu.tsx @@ -3,6 +3,7 @@ import { DeleteOutlined, EditOutlined } from '@ant-design/icons'; import { Dropdown, Menu, message, Modal } from 'antd'; import { MenuIcon } from '../../entity/shared/EntityDropdown/EntityDropdown'; import { useDeletePostMutation } from '../../../graphql/post.generated'; +import handleGraphQLError from '../../shared/handleGraphQLError'; type Props = { urn: string; @@ -26,9 +27,12 @@ export default function PostItemMenu({ title, urn, onDelete, onEdit }: Props) { onDelete?.(); } }) - .catch(() => { - message.destroy(); - message.error({ content: `Failed to delete Post!: An unknown error occurred.`, duration: 3 }); + .catch((error) => { + handleGraphQLError({ + error, + defaultMessage: 'Failed to delete Post! An unexpected error occurred', + permissionMessage: 'Unauthorized to delete Post. Please contact your DataHub administrator.', + }); }); }; diff --git a/datahub-web-react/src/graphql/chart.graphql b/datahub-web-react/src/graphql/chart.graphql index a4b430720fa3d..da2dae3bd6d86 100644 --- a/datahub-web-react/src/graphql/chart.graphql +++ b/datahub-web-react/src/graphql/chart.graphql @@ -103,6 +103,9 @@ query getChart($urn: String!) { subTypes { typeNames } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql index 94d2f53ee30a5..e75b26b45aacd 100644 --- a/datahub-web-react/src/graphql/container.graphql +++ b/datahub-web-react/src/graphql/container.graphql @@ -54,5 +54,8 @@ query getContainer($urn: String!) { status { removed } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/dashboard.graphql b/datahub-web-react/src/graphql/dashboard.graphql index d77f6f5c8107f..2690de0c507de 100644 --- a/datahub-web-react/src/graphql/dashboard.graphql +++ b/datahub-web-react/src/graphql/dashboard.graphql @@ -7,6 +7,9 @@ query getDashboard($urn: String!) { datasets: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) { ...fullRelationshipResults } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/dataFlow.graphql b/datahub-web-react/src/graphql/dataFlow.graphql index 122b35f7b5704..fccec29e082d6 100644 --- a/datahub-web-react/src/graphql/dataFlow.graphql +++ b/datahub-web-react/src/graphql/dataFlow.graphql @@ -102,6 +102,9 @@ query getDataFlow($urn: String!) { } } } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/dataJob.graphql b/datahub-web-react/src/graphql/dataJob.graphql index a41c242a71b8f..161ea859fc36a 100644 --- a/datahub-web-react/src/graphql/dataJob.graphql +++ b/datahub-web-react/src/graphql/dataJob.graphql @@ -6,6 +6,9 @@ query getDataJob($urn: String!) { start total } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/dataProduct.graphql b/datahub-web-react/src/graphql/dataProduct.graphql index 464ab7cc12164..4c44639e89d7d 100644 --- a/datahub-web-react/src/graphql/dataProduct.graphql +++ b/datahub-web-react/src/graphql/dataProduct.graphql @@ -1,6 +1,9 @@ query getDataProduct($urn: String!) { dataProduct(urn: $urn) { ...dataProductFields + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 658ce2b47c567..57c74e0c65d69 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -112,13 +112,7 @@ fragment nonSiblingDatasetFields on Dataset { } ...viewProperties autoRenderAspects: aspects(input: { autoRenderOnly: true }) { - aspectName - payload - renderSpec { - displayType - displayName - key - } + ...autoRenderAspectFields } status { removed diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 170a5b5df476b..76c59ad0ed2ae 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -27,6 +27,9 @@ query getDomain($urn: String!) { } } } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } ...domainEntitiesFields } } diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index b77ef9d1ad29c..bb06ccb90a46d 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -1162,3 +1162,13 @@ fragment entityDisplayNameFields on Entity { instanceId } } + +fragment autoRenderAspectFields on RawAspect { + aspectName + payload + renderSpec { + displayType + displayName + key + } +} diff --git a/datahub-web-react/src/graphql/glossaryNode.graphql b/datahub-web-react/src/graphql/glossaryNode.graphql index 9cb01b98b3efb..4a531eb55248f 100644 --- a/datahub-web-react/src/graphql/glossaryNode.graphql +++ b/datahub-web-react/src/graphql/glossaryNode.graphql @@ -27,6 +27,9 @@ query getGlossaryNode($urn: String!) { canManageEntity canManageChildren } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } children: relationships( input: { types: ["IsPartOf"] diff --git a/datahub-web-react/src/graphql/glossaryTerm.graphql b/datahub-web-react/src/graphql/glossaryTerm.graphql index f2a311f50fe51..4eb0747e0aeba 100644 --- a/datahub-web-react/src/graphql/glossaryTerm.graphql +++ b/datahub-web-react/src/graphql/glossaryTerm.graphql @@ -87,6 +87,9 @@ query getGlossaryTerm($urn: String!, $start: Int, $count: Int) { privileges { canManageEntity } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/group.graphql b/datahub-web-react/src/graphql/group.graphql index 1007721e51a4e..2b8db6483632d 100644 --- a/datahub-web-react/src/graphql/group.graphql +++ b/datahub-web-react/src/graphql/group.graphql @@ -24,6 +24,9 @@ query getGroup($urn: String!, $membersCount: Int!) { email slack } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } ownership { ...ownershipFields } @@ -195,6 +198,17 @@ query listGroups($input: ListGroupsInput!) { ) { total } + roles: relationships(input: { types: ["IsMemberOfRole"], direction: OUTGOING, start: 0 }) { + relationships { + entity { + ... on DataHubRole { + urn + type + name + } + } + } + } } } } diff --git a/datahub-web-react/src/graphql/ingestion.graphql b/datahub-web-react/src/graphql/ingestion.graphql index 1767fe34bfef0..4d6f090b99356 100644 --- a/datahub-web-react/src/graphql/ingestion.graphql +++ b/datahub-web-react/src/graphql/ingestion.graphql @@ -145,6 +145,10 @@ mutation createSecret($input: CreateSecretInput!) { createSecret(input: $input) } +mutation updateSecret($input: UpdateSecretInput!) { + updateSecret(input: $input) +} + mutation deleteSecret($urn: String!) { deleteSecret(urn: $urn) } diff --git a/datahub-web-react/src/graphql/mlFeature.graphql b/datahub-web-react/src/graphql/mlFeature.graphql index f9cd2b66d900e..9fcb871dc4f49 100644 --- a/datahub-web-react/src/graphql/mlFeature.graphql +++ b/datahub-web-react/src/graphql/mlFeature.graphql @@ -4,5 +4,8 @@ query getMLFeature($urn: String!) { featureTables: relationships(input: { types: ["Contains"], direction: INCOMING, start: 0, count: 100 }) { ...fullRelationshipResults } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/mlFeatureTable.graphql b/datahub-web-react/src/graphql/mlFeatureTable.graphql index 3c52dccf7672c..4bf3972e722b6 100644 --- a/datahub-web-react/src/graphql/mlFeatureTable.graphql +++ b/datahub-web-react/src/graphql/mlFeatureTable.graphql @@ -1,5 +1,8 @@ query getMLFeatureTable($urn: String!) { mlFeatureTable(urn: $urn) { ...nonRecursiveMLFeatureTable + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/mlModel.graphql b/datahub-web-react/src/graphql/mlModel.graphql index e5330480039f8..5375485a8a9f6 100644 --- a/datahub-web-react/src/graphql/mlModel.graphql +++ b/datahub-web-react/src/graphql/mlModel.graphql @@ -18,5 +18,8 @@ query getMLModel($urn: String!) { } } } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/mlModelGroup.graphql b/datahub-web-react/src/graphql/mlModelGroup.graphql index 12a1c04586198..57249d543bb86 100644 --- a/datahub-web-react/src/graphql/mlModelGroup.graphql +++ b/datahub-web-react/src/graphql/mlModelGroup.graphql @@ -21,5 +21,8 @@ query getMLModelGroup($urn: String!) { ) { ...fullRelationshipResults } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/mlPrimaryKey.graphql b/datahub-web-react/src/graphql/mlPrimaryKey.graphql index a70550a44a88d..2bfceb37ce16b 100644 --- a/datahub-web-react/src/graphql/mlPrimaryKey.graphql +++ b/datahub-web-react/src/graphql/mlPrimaryKey.graphql @@ -4,5 +4,8 @@ query getMLPrimaryKey($urn: String!) { featureTables: relationships(input: { types: ["KeyedBy"], direction: INCOMING, start: 0, count: 100 }) { ...fullRelationshipResults } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/tag.graphql b/datahub-web-react/src/graphql/tag.graphql index 37aaebc265032..031d923276bfe 100644 --- a/datahub-web-react/src/graphql/tag.graphql +++ b/datahub-web-react/src/graphql/tag.graphql @@ -11,6 +11,9 @@ query getTag($urn: String!) { ownership { ...ownershipFields } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } } } diff --git a/datahub-web-react/src/graphql/user.graphql b/datahub-web-react/src/graphql/user.graphql index 4757b8a7e28dd..48c0d7de8c63c 100644 --- a/datahub-web-react/src/graphql/user.graphql +++ b/datahub-web-react/src/graphql/user.graphql @@ -27,6 +27,9 @@ query getUser($urn: String!, $groupsCount: Int!) { globalTags { ...globalTagsFields } + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + ...autoRenderAspectFields + } groups: relationships( input: { types: ["IsMemberOfGroup", "IsMemberOfNativeGroup"] diff --git a/docker/build.gradle b/docker/build.gradle index 8b71ff1f6f06b..cc95e12f26f76 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -61,6 +61,7 @@ dockerCompose { composeAdditionalArgs = ['--profile', 'quickstart-consumers'] environment.put 'DATAHUB_VERSION', "v${version}" + environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' @@ -78,6 +79,7 @@ dockerCompose { composeAdditionalArgs = ['--profile', 'quickstart-postgres'] environment.put 'DATAHUB_VERSION', "v${version}" + environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' @@ -97,6 +99,7 @@ dockerCompose { environment.put "ACTIONS_VERSION", "v${version}-slim" environment.put "ACTIONS_EXTRA_PACKAGES", 'acryl-datahub-actions[executor] acryl-datahub-actions' environment.put "ACTIONS_CONFIG", 'https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml' + environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' @@ -113,6 +116,8 @@ dockerCompose { isRequiredBy(tasks.named('quickstartDebug')) composeAdditionalArgs = ['--profile', 'debug'] + environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally + useComposeFiles = ['profiles/docker-compose.yml'] projectName = 'datahub' projectNamePrefix = '' diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 558a5afe2c2cf..0bf0d2f88af73 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -48,6 +48,8 @@ RUN apt-get update && apt-get install -y -qq \ zip \ unzip \ ldap-utils \ + unixodbc \ + libodbc2 \ && python -m pip install --no-cache --upgrade pip wheel setuptools \ && rm -rf /var/lib/apt/lists/* /var/cache/apk/* diff --git a/docker/datahub-ingestion-base/smoke.Dockerfile b/docker/datahub-ingestion-base/smoke.Dockerfile index 15dc46ae5b882..5c6738720e05e 100644 --- a/docker/datahub-ingestion-base/smoke.Dockerfile +++ b/docker/datahub-ingestion-base/smoke.Dockerfile @@ -15,12 +15,12 @@ RUN apt-get update && apt-get install -y \ xauth \ xvfb -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-17-jdk COPY . /datahub-src ARG RELEASE_VERSION RUN cd /datahub-src/metadata-ingestion && \ - sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py && \ cd ../ && \ ./gradlew :metadata-ingestion:installAll diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 2898a363a0a18..4f0e66251b154 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -13,8 +13,8 @@ COPY ./metadata-ingestion-modules/airflow-plugin /datahub-ingestion/airflow-plug ARG RELEASE_VERSION WORKDIR /datahub-ingestion -RUN sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ - sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \ +RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \ + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \ cat src/datahub/__init__.py && \ chown -R datahub /datahub-ingestion diff --git a/docker/datahub-ingestion/Dockerfile-slim-only b/docker/datahub-ingestion/Dockerfile-slim-only index 4112f470c25be..24412958a2d08 100644 --- a/docker/datahub-ingestion/Dockerfile-slim-only +++ b/docker/datahub-ingestion/Dockerfile-slim-only @@ -10,7 +10,7 @@ COPY ./metadata-ingestion /datahub-ingestion ARG RELEASE_VERSION WORKDIR /datahub-ingestion -RUN sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py && \ +RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \ cat src/datahub/__init__.py && \ chown -R datahub /datahub-ingestion diff --git a/docker/datahub-mae-consumer/start.sh b/docker/datahub-mae-consumer/start.sh index 2af7ce6855d1c..f839d3646bdc6 100755 --- a/docker/datahub-mae-consumer/start.sh +++ b/docker/datahub-mae-consumer/start.sh @@ -33,6 +33,9 @@ fi if [[ ${GRAPH_SERVICE_IMPL:-} != elasticsearch ]] && [[ ${SKIP_NEO4J_CHECK:-false} != true ]]; then dockerize_args+=("-wait" "$NEO4J_HOST") fi +if [[ "${KAFKA_SCHEMAREGISTRY_URL:-}" && ${SKIP_SCHEMA_REGISTRY_CHECK:-false} != true ]]; then + dockerize_args+=("-wait" "$KAFKA_SCHEMAREGISTRY_URL") +fi JAVA_TOOL_OPTIONS="${JDK_JAVA_OPTIONS:-}${JAVA_OPTS:+ $JAVA_OPTS}${JMX_OPTS:+ $JMX_OPTS}" if [[ ${ENABLE_OTEL:-false} == true ]]; then diff --git a/docker/datahub-mce-consumer/start.sh b/docker/datahub-mce-consumer/start.sh index ef183d41856aa..a00127a841188 100755 --- a/docker/datahub-mce-consumer/start.sh +++ b/docker/datahub-mce-consumer/start.sh @@ -5,6 +5,11 @@ if [[ $SKIP_KAFKA_CHECK != true ]]; then WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') " fi +WAIT_FOR_SCHEMA_REGISTRY="" +if [[ "$KAFKA_SCHEMAREGISTRY_URL" && $SKIP_SCHEMA_REGISTRY_CHECK != true ]]; then + WAIT_FOR_SCHEMA_REGISTRY="-wait $KAFKA_SCHEMAREGISTRY_URL" +fi + OTEL_AGENT="" if [[ $ENABLE_OTEL == true ]]; then OTEL_AGENT="-javaagent:opentelemetry-javaagent.jar " @@ -17,5 +22,6 @@ fi exec dockerize \ $WAIT_FOR_KAFKA \ + $WAIT_FOR_SCHEMA_REGISTRY \ -timeout 240s \ - java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar \ No newline at end of file + java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mce-consumer/bin/mce-consumer-job.jar diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile index ea64f94f88727..fdaf9ddbaf813 100644 --- a/docker/elasticsearch-setup/Dockerfile +++ b/docker/elasticsearch-setup/Dockerfile @@ -44,9 +44,9 @@ FROM base AS dev-install # See this excellent thread https://github.com/docker/cli/issues/1134 FROM ${APP_ENV}-install AS final + CMD if [ "$ELASTICSEARCH_USE_SSL" == "true" ]; then ELASTICSEARCH_PROTOCOL=https; else ELASTICSEARCH_PROTOCOL=http; fi \ && if [[ -n "$ELASTICSEARCH_USERNAME" ]]; then ELASTICSEARCH_HTTP_HEADERS="Authorization: Basic $(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64)"; else ELASTICSEARCH_HTTP_HEADERS="Accept: */*"; fi \ && if [[ "$SKIP_ELASTICSEARCH_CHECK" != "true" ]]; then \ dockerize -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "${ELASTICSEARCH_HTTP_HEADERS}" -timeout 120s /create-indices.sh; \ else /create-indices.sh; fi - diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index f863dff7a59c5..769bce3105a7f 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -64,6 +64,8 @@ x-datahub-system-update-service: &datahub-system-update-service SCHEMA_REGISTRY_SYSTEM_UPDATE: ${SCHEMA_REGISTRY_SYSTEM_UPDATE:-true} SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS: ${SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS:-true} SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION: ${SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION:-true} + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins x-datahub-system-update-service-dev: &datahub-system-update-service-dev <<: *datahub-system-update-service @@ -99,6 +101,8 @@ x-datahub-gms-service: &datahub-gms-service timeout: 5s volumes: - ${HOME}/.datahub/plugins:/etc/datahub/plugins + labels: + io.datahubproject.datahub.component: "gms" x-datahub-gms-service-dev: &datahub-gms-service-dev <<: *datahub-gms-service diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js index 0926d92ad01e1..d28552fc02734 100644 --- a/docs-website/docusaurus.config.js +++ b/docs-website/docusaurus.config.js @@ -100,6 +100,14 @@ module.exports = { href: "https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w", label: "YouTube", }, + { + href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvGCKMQC2dJEZ6cUqWsREbFi", + label: "Case Studies", + }, + { + href: "https://www.youtube.com/playlist?list=PLdCtLs64vZvErAXMiqUYH9e63wyDaMBgg", + label: "DataHub Basics", + }, ], }, { diff --git a/docs-website/graphql/generateGraphQLSchema.sh b/docs-website/graphql/generateGraphQLSchema.sh index 4e41c5dfbfacd..c6d7ec528b613 100755 --- a/docs-website/graphql/generateGraphQLSchema.sh +++ b/docs-website/graphql/generateGraphQLSchema.sh @@ -16,3 +16,5 @@ cat ../../datahub-graphql-core/src/main/resources/tests.graphql >> combined.grap cat ../../datahub-graphql-core/src/main/resources/timeline.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/step.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/lineage.graphql >> combined.graphql +cat ../../datahub-graphql-core/src/main/resources/properties.graphql >> combined.graphql +cat ../../datahub-graphql-core/src/main/resources/forms.graphql >> combined.graphql \ No newline at end of file diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 2b8873c678778..1e6d8bec01813 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -561,9 +561,18 @@ module.exports = { ], }, { - type: "doc", - label: "OpenAPI", - id: "docs/api/openapi/openapi-usage-guide", + OpenAPI: [ + { + type: "doc", + label: "OpenAPI", + id: "docs/api/openapi/openapi-usage-guide", + }, + { + type: "doc", + label: "Structured Properties", + id: "docs/api/openapi/openapi-structured-properties", + }, + ], }, "docs/dev-guides/timeline", { @@ -768,6 +777,7 @@ module.exports = { // "docs/how/add-user-data", // "docs/_feature-guide-template" // - "metadata-service/services/README" + // "metadata-ingestion/examples/structured_properties/README" // ], ], }; diff --git a/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss b/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss index 3d30c65f89539..862fb04c8370b 100644 --- a/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss +++ b/docs-website/src/pages/_components/TownhallButton/townhallbutton.module.scss @@ -26,4 +26,4 @@ background-image: linear-gradient(to right, #1890ff 0%, #48DBFB 100%); background-origin: border-box; } - } \ No newline at end of file + } diff --git a/docs/api/openapi/openapi-structured-properties.md b/docs/api/openapi/openapi-structured-properties.md new file mode 100644 index 0000000000000..521ce8789db0d --- /dev/null +++ b/docs/api/openapi/openapi-structured-properties.md @@ -0,0 +1,284 @@ +# Structured Properties - DataHub OpenAPI v2 Guide + +This guides walks through the process of creating and using a Structured Property using the `v2` version +of the DataHub OpenAPI implementation. Note that this refers to DataHub's OpenAPI version and not the version of OpenAPI itself. + +Requirements: +* curl +* jq + +## Structured Property Definition + +Before a structured property can be added to an entity it must first be defined. Here is an example +structured property being created against a local quickstart instance. + +### Create Property Definition + +Example Request: + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Amy.test.MyProperty01/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "my.test.MyProperty01", + "displayName": "MyProperty01", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo"}, + "description": "test foo value" + }, + { + "value": {"string": "bar"}, + "description": "test bar value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ], + "description": "test description" +}' | jq +``` + +### Read Property Definition + +Example Request: + +```shell +curl -X 'GET' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Amy.test.MyProperty01/propertyDefinition' \ + -H 'accept: application/json' | jq +``` + +Example Response: + +```json +{ + "value": { + "allowedValues": [ + { + "value": { + "string": "foo" + }, + "description": "test foo value" + }, + { + "value": { + "string": "bar" + }, + "description": "test bar value" + } + ], + "qualifiedName": "my.test.MyProperty01", + "displayName": "MyProperty01", + "valueType": "urn:li:dataType:datahub.string", + "description": "test description", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ], + "cardinality": "SINGLE" + } +} +``` + +### Delete Property Definition + +⚠ **Not Implemented** ⚠ + +## Applying Structured Properties + +Structured Properties can now be added to entities which have the `structuredProperties` as aspect. In the following +example we'll attach and remove properties to an example dataset entity with urn `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. + +### Set Structured Property Values + +This will set/replace all structured properties on the entity. See `PATCH` operations to add/remove a single property. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty01", + "values": [ + {"string": "foo"} + ] + } + ] +}' | jq +``` + +### Patch Structured Property Value + +For this example, we'll extend create a second structured property and apply both properties to the same +dataset used previously. After this your system should include both `my.test.MyProperty01` and `my.test.MyProperty02`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Amy.test.MyProperty02/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "my.test.MyProperty02", + "displayName": "MyProperty02", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo2"}, + "description": "test foo2 value" + }, + { + "value": {"string": "bar2"}, + "description": "test bar2 value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ] +}' | jq +``` + +This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty01", + "values": [ + {"string": "foo"} + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty02", + "values": [ + {"string": "bar2"} + ] + } + ] +}' | jq +``` + +#### Remove Structured Property Value + +The expected state of our test dataset include 2 structured properties. We'd like to remove the first one and preserve +the second property. + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "remove", + "path": "/properties/urn:li:structuredProperty:my.test.MyProperty01" + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +The response will show that the expected property has been removed. + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": { + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty02" + } + ] + } + } + } +} +``` + +#### Add Structured Property Value + +In this example, we'll add the property back with a different value, preserving the existing property. + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "add", + "path": "/properties/urn:li:structuredProperty:my.test.MyProperty01", + "value": { + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty01", + "values": [ + { + "string": "bar" + } + ] + } + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +The response shows that the property was re-added with the new value `bar` instead of the previous value `foo`. + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": { + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty02" + }, + { + "values": [ + { + "string": "bar" + } + ], + "propertyUrn": "urn:li:structuredProperty:my.test.MyProperty01" + } + ] + } + } + } +} +``` diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java index 83e40b22a5e44..806fd47c721ec 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java @@ -1,7 +1,6 @@ package com.linkedin.metadata.aspect.batch; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; import java.util.HashSet; @@ -33,14 +32,12 @@ default List getMCPItems() { } Pair>, List> toUpsertBatchItems( - Map> latestAspects, - EntityRegistry entityRegistry, - AspectRetriever aspectRetriever); + Map> latestAspects, AspectRetriever aspectRetriever); default Stream applyMCPSideEffects( - List items, EntityRegistry entityRegistry, AspectRetriever aspectRetriever) { - return entityRegistry.getAllMCPSideEffects().stream() - .flatMap(mcpSideEffect -> mcpSideEffect.apply(items, entityRegistry, aspectRetriever)); + List items, AspectRetriever aspectRetriever) { + return aspectRetriever.getEntityRegistry().getAllMCPSideEffects().stream() + .flatMap(mcpSideEffect -> mcpSideEffect.apply(items, aspectRetriever)); } default boolean containsDuplicateAspects() { @@ -53,22 +50,21 @@ default boolean containsDuplicateAspects() { default Map> getUrnAspectsMap() { return getItems().stream() - .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName())) + .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName())) .collect( Collectors.groupingBy( - Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toSet()))); + Pair::getKey, Collectors.mapping(Pair::getValue, Collectors.toSet()))); } default Map> getNewUrnAspectsMap( Map> existingMap, List items) { Map> newItemsMap = items.stream() - .map(aspect -> Map.entry(aspect.getUrn().toString(), aspect.getAspectName())) + .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName())) .collect( Collectors.groupingBy( - Map.Entry::getKey, - Collectors.mapping( - Map.Entry::getValue, Collectors.toCollection(HashSet::new)))); + Pair::getKey, + Collectors.mapping(Pair::getValue, Collectors.toCollection(HashSet::new)))); return newItemsMap.entrySet().stream() .filter( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPBatchItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPBatchItem.java index bb5e0ac53934a..dd0d0ec68dac6 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPBatchItem.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCPBatchItem.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.aspect.batch; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.mxe.MetadataChangeProposal; import javax.annotation.Nullable; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/PatchItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/PatchItem.java index f790c12ee5335..e9e30f7f2bd96 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/PatchItem.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/PatchItem.java @@ -3,7 +3,6 @@ import com.github.fge.jsonpatch.Patch; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; -import com.linkedin.metadata.models.registry.EntityRegistry; /** * A change proposal represented as a patch to an exiting stored object in the primary data store. @@ -13,14 +12,11 @@ public abstract class PatchItem extends MCPBatchItem { /** * Convert a Patch to an Upsert * - * @param entityRegistry the entity registry * @param recordTemplate the current value record template * @return the upsert */ public abstract UpsertItem applyPatch( - EntityRegistry entityRegistry, - RecordTemplate recordTemplate, - AspectRetriever aspectRetriever); + RecordTemplate recordTemplate, AspectRetriever aspectRetriever); public abstract Patch getPatch(); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/UpsertItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/UpsertItem.java index 4e4d2a38799dc..c337e4f848e5c 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/UpsertItem.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/UpsertItem.java @@ -3,7 +3,6 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; -import com.linkedin.metadata.models.registry.EntityRegistry; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -17,8 +16,6 @@ public abstract class UpsertItem extends MCPBatchItem { public abstract SystemAspect toLatestEntityAspect(); public abstract void validatePreCommit( - @Nullable RecordTemplate previous, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) + @Nullable RecordTemplate previous, @Nonnull AspectRetriever aspectRetriever) throws AspectValidationException; } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java new file mode 100644 index 0000000000000..484603b9c1f85 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/GenericJsonPatch.java @@ -0,0 +1,35 @@ +package com.linkedin.metadata.aspect.patch; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonpatch.JsonPatch; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class GenericJsonPatch { + @Nullable private Map> arrayPrimaryKeys; + + @Nonnull private JsonNode patch; + + @Nonnull + public Map> getArrayPrimaryKeys() { + return arrayPrimaryKeys == null ? Collections.emptyMap() : arrayPrimaryKeys; + } + + @JsonIgnore + public JsonPatch getJsonPatch() throws IOException { + return JsonPatch.fromJson(patch); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/PatchOperationType.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/PatchOperationType.java similarity index 81% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/PatchOperationType.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/PatchOperationType.java index ac93fd24fee02..6eaa6069267ba 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/PatchOperationType.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/PatchOperationType.java @@ -1,4 +1,4 @@ -package datahub.client.patch; +package com.linkedin.metadata.aspect.patch; import lombok.Getter; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/AbstractMultiFieldPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/AbstractMultiFieldPatchBuilder.java similarity index 95% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/AbstractMultiFieldPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/AbstractMultiFieldPatchBuilder.java index 943aaefec469b..165a4d26c339c 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/AbstractMultiFieldPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/AbstractMultiFieldPatchBuilder.java @@ -1,6 +1,6 @@ -package datahub.client.patch; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.http.entity.ContentType; public abstract class AbstractMultiFieldPatchBuilder> { @@ -87,7 +86,7 @@ protected GenericAspect buildPatch() { .set(VALUE_KEY, triple.right))); GenericAspect genericAspect = new GenericAspect(); - genericAspect.setContentType(ContentType.APPLICATION_JSON.getMimeType()); + genericAspect.setContentType("application/json"); genericAspect.setValue(ByteString.copyString(patches.toString(), StandardCharsets.UTF_8)); return genericAspect; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/chart/ChartInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/ChartInfoPatchBuilder.java similarity index 75% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/chart/ChartInfoPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/ChartInfoPatchBuilder.java index 0655d2b3eb8eb..09f9dad134a0b 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/chart/ChartInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/ChartInfoPatchBuilder.java @@ -1,12 +1,12 @@ -package datahub.client.patch.chart; +package com.linkedin.metadata.aspect.patch.builder; -import static com.linkedin.metadata.Constants.*; -import static datahub.client.patch.common.PatchUtil.*; +import static com.linkedin.metadata.Constants.CHART_ENTITY_NAME; +import static com.linkedin.metadata.Constants.CHART_INFO_ASPECT_NAME; +import static com.linkedin.metadata.aspect.patch.builder.PatchUtil.createEdgeValue; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.Urn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/CustomPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java similarity index 90% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/CustomPropertiesPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java index e621aaf57ff97..e4143851afbe5 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/CustomPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java @@ -1,12 +1,11 @@ -package datahub.client.patch.common; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; -import datahub.client.patch.subtypesupport.IntermediatePatchBuilder; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.builder.subtypesupport.IntermediatePatchBuilder; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dashboard/DashboardInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java similarity index 86% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dashboard/DashboardInfoPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java index cadde582f1c64..9156b304a394e 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dashboard/DashboardInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DashboardInfoPatchBuilder.java @@ -1,15 +1,17 @@ -package datahub.client.patch.dashboard; +package com.linkedin.metadata.aspect.patch.builder; -import static com.linkedin.metadata.Constants.*; -import static datahub.client.patch.common.PatchUtil.*; +import static com.linkedin.metadata.Constants.CHART_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DASHBOARD_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DASHBOARD_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.aspect.patch.builder.PatchUtil.createEdgeValue; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.Edge; import com.linkedin.common.urn.ChartUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataflow/DataFlowInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java similarity index 92% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataflow/DataFlowInfoPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java index 9e55ab4fc6db4..6a114d90875fe 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataflow/DataFlowInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java @@ -1,15 +1,14 @@ -package datahub.client.patch.dataflow; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATA_FLOW_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_FLOW_INFO_ASPECT_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.TimeStamp; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; -import datahub.client.patch.common.CustomPropertiesPatchBuilder; -import datahub.client.patch.subtypesupport.CustomPropertiesPatchBuilderSupport; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java similarity index 93% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInfoPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java index 581616f54e9b9..99c0ac6c15eb1 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java @@ -1,16 +1,15 @@ -package datahub.client.patch.datajob; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_INFO_ASPECT_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.TimeStamp; import com.linkedin.common.urn.DataFlowUrn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; -import datahub.client.patch.common.CustomPropertiesPatchBuilder; -import datahub.client.patch.subtypesupport.CustomPropertiesPatchBuilderSupport; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java similarity index 93% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java index fc250daffe916..8e2168e5b6a33 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/datajob/DataJobInputOutputPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilder.java @@ -1,8 +1,10 @@ -package datahub.client.patch.datajob; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; -import static datahub.client.patch.common.PatchUtil.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_INPUT_OUTPUT_ASPECT_NAME; +import static com.linkedin.metadata.aspect.patch.builder.PatchUtil.createEdgeValue; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; @@ -10,9 +12,8 @@ import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.graph.LineageDirection; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/DatasetPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java similarity index 91% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/DatasetPropertiesPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java index f4329c84f33ff..31e181fc244fb 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/DatasetPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java @@ -1,13 +1,12 @@ -package datahub.client.patch.dataset; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATASET_PROPERTIES_ASPECT_NAME; import com.fasterxml.jackson.databind.JsonNode; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; -import datahub.client.patch.common.CustomPropertiesPatchBuilder; -import datahub.client.patch.subtypesupport.CustomPropertiesPatchBuilderSupport; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; import java.util.List; import java.util.Map; import javax.annotation.Nonnull; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/EditableSchemaMetadataPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/EditableSchemaMetadataPatchBuilder.java similarity index 90% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/EditableSchemaMetadataPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/EditableSchemaMetadataPatchBuilder.java index 6478b31d27ef0..5e9e1911925fa 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/EditableSchemaMetadataPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/EditableSchemaMetadataPatchBuilder.java @@ -1,15 +1,15 @@ -package datahub.client.patch.dataset; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NAME; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.GlossaryTermAssociation; import com.linkedin.common.TagAssociation; import com.linkedin.common.urn.GlossaryTermUrn; import com.linkedin.common.urn.TagUrn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlobalTagsPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlobalTagsPatchBuilder.java similarity index 88% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlobalTagsPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlobalTagsPatchBuilder.java index 84db0ba307cf2..ff34b187f6151 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlobalTagsPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlobalTagsPatchBuilder.java @@ -1,12 +1,11 @@ -package datahub.client.patch.common; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.TagUrn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlossaryTermsPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlossaryTermsPatchBuilder.java similarity index 89% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlossaryTermsPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlossaryTermsPatchBuilder.java index 6f31025406b1b..16d9beded3066 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/GlossaryTermsPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/GlossaryTermsPatchBuilder.java @@ -1,12 +1,11 @@ -package datahub.client.patch.common; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.GLOSSARY_TERMS_ASPECT_NAME; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.GlossaryTermUrn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/OwnershipPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/OwnershipPatchBuilder.java similarity index 91% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/OwnershipPatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/OwnershipPatchBuilder.java index 20e0c930a8c95..35a647424a88a 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/OwnershipPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/OwnershipPatchBuilder.java @@ -1,13 +1,12 @@ -package datahub.client.patch.common; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.OwnershipType; import com.linkedin.common.urn.Urn; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/PatchUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/PatchUtil.java similarity index 96% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/PatchUtil.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/PatchUtil.java index 69db36c6e038c..7556a8b1d9418 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/common/PatchUtil.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/PatchUtil.java @@ -1,7 +1,7 @@ -package datahub.client.patch.common; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.UNKNOWN_ACTOR; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.Edge; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java new file mode 100644 index 0000000000000..fab81e0af5bf5 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/StructuredPropertiesPatchBuilder.java @@ -0,0 +1,110 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.ImmutableTriple; + +public class StructuredPropertiesPatchBuilder + extends AbstractMultiFieldPatchBuilder { + + private static final String BASE_PATH = "/properties"; + private static final String URN_KEY = "urn"; + private static final String CONTEXT_KEY = "context"; + + /** + * Remove a property from a structured properties aspect. If the property doesn't exist, this is a + * no-op. + * + * @param propertyUrn + * @return + */ + public StructuredPropertiesPatchBuilder removeProperty(Urn propertyUrn) { + pathValues.add( + ImmutableTriple.of( + PatchOperationType.REMOVE.getValue(), BASE_PATH + "/" + propertyUrn, null)); + return this; + } + + public StructuredPropertiesPatchBuilder setProperty( + @Nonnull Urn propertyUrn, @Nullable List propertyValues) { + propertyValues.stream() + .map( + propertyValue -> + propertyValue instanceof Integer + ? this.setProperty(propertyUrn, (Integer) propertyValue) + : this.setProperty(propertyUrn, String.valueOf(propertyValue))) + .collect(Collectors.toList()); + return this; + } + + public StructuredPropertiesPatchBuilder setProperty( + @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { + ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); + ObjectNode value = instance.objectNode(); + value.put(URN_KEY, propertyUrn.toString()); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + return this; + } + + public StructuredPropertiesPatchBuilder setProperty( + @Nonnull Urn propertyUrn, @Nullable String propertyValue) { + ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); + ObjectNode value = instance.objectNode(); + value.put(URN_KEY, propertyUrn.toString()); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), BASE_PATH + "/" + propertyUrn, propertyValueNode)); + return this; + } + + public StructuredPropertiesPatchBuilder addProperty( + @Nonnull Urn propertyUrn, @Nullable Integer propertyValue) { + ValueNode propertyValueNode = instance.numberNode((Integer) propertyValue); + ObjectNode value = instance.objectNode(); + value.put(URN_KEY, propertyUrn.toString()); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), + propertyValueNode)); + return this; + } + + public StructuredPropertiesPatchBuilder addProperty( + @Nonnull Urn propertyUrn, @Nullable String propertyValue) { + ValueNode propertyValueNode = instance.textNode(String.valueOf(propertyValue)); + ObjectNode value = instance.objectNode(); + value.put(URN_KEY, propertyUrn.toString()); + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), + BASE_PATH + "/" + propertyUrn + "/" + String.valueOf(propertyValue), + propertyValueNode)); + return this; + } + + @Override + protected String getAspectName() { + return STRUCTURED_PROPERTIES_ASPECT_NAME; + } + + @Override + protected String getEntityType() { + if (this.targetEntityUrn == null) { + throw new IllegalStateException( + "Target Entity Urn must be set to determine entity type before building Patch."); + } + return this.targetEntityUrn.getEntityType(); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/UpstreamLineagePatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java similarity index 96% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/UpstreamLineagePatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java index 9db2ebc522e09..bfb46d8fc5773 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/dataset/UpstreamLineagePatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/UpstreamLineagePatchBuilder.java @@ -1,7 +1,9 @@ -package datahub.client.patch.dataset; +package com.linkedin.metadata.aspect.patch.builder; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.UNKNOWN_ACTOR; +import static com.linkedin.metadata.Constants.UPSTREAM_LINEAGE_ASPECT_NAME; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.DatasetUrn; @@ -9,8 +11,7 @@ import com.linkedin.dataset.DatasetLineageType; import com.linkedin.dataset.FineGrainedLineageDownstreamType; import com.linkedin.dataset.FineGrainedLineageUpstreamType; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; -import datahub.client.patch.PatchOperationType; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.ToString; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/CustomPropertiesPatchBuilderSupport.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/CustomPropertiesPatchBuilderSupport.java similarity index 81% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/CustomPropertiesPatchBuilderSupport.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/CustomPropertiesPatchBuilderSupport.java index 9f221bac15be4..5e1cd094b204e 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/CustomPropertiesPatchBuilderSupport.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/CustomPropertiesPatchBuilderSupport.java @@ -1,6 +1,6 @@ -package datahub.client.patch.subtypesupport; +package com.linkedin.metadata.aspect.patch.builder.subtypesupport; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.AbstractMultiFieldPatchBuilder; import java.util.Map; import javax.annotation.Nonnull; diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/IntermediatePatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java similarity index 83% rename from metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/IntermediatePatchBuilder.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java index e3b14c0838ad6..d891a6b9673da 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch/subtypesupport/IntermediatePatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java @@ -1,7 +1,7 @@ -package datahub.client.patch.subtypesupport; +package com.linkedin.metadata.aspect.patch.builder.subtypesupport; import com.fasterxml.jackson.databind.JsonNode; -import datahub.client.patch.AbstractMultiFieldPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.AbstractMultiFieldPatchBuilder; import java.util.List; import org.apache.commons.lang3.tuple.ImmutableTriple; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/ArrayMergingTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java similarity index 98% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/ArrayMergingTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java index 9cd8e74d952d6..ff721e97c0e1d 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/ArrayMergingTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/ArrayMergingTemplate.java @@ -1,6 +1,6 @@ -package com.linkedin.metadata.models.registry.template; +package com.linkedin.metadata.aspect.patch.template; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java similarity index 71% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java index 029eb688c5291..e9d09085e7eb5 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/AspectTemplateEngine.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java @@ -1,6 +1,18 @@ -package com.linkedin.metadata.models.registry.template; +package com.linkedin.metadata.aspect.patch.template; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.CHART_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DASHBOARD_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATASET_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATA_FLOW_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATA_JOB_INPUT_OUTPUT_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATA_PRODUCT_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.EDITABLE_SCHEMA_METADATA_ASPECT_NAME; +import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.GLOSSARY_TERMS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.Constants.UPSTREAM_LINEAGE_ASPECT_NAME; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.fge.jsonpatch.JsonPatchException; @@ -34,7 +46,8 @@ public class AspectTemplateEngine { DATA_PRODUCT_PROPERTIES_ASPECT_NAME, DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, CHART_INFO_ASPECT_NAME, - DASHBOARD_INFO_ASPECT_NAME) + DASHBOARD_INFO_ASPECT_NAME, + STRUCTURED_PROPERTIES_ASPECT_NAME) .collect(Collectors.toSet()); private final Map> _aspectTemplateMap; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/CompoundKeyTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/CompoundKeyTemplate.java new file mode 100644 index 0000000000000..78cf14c47a0bf --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/CompoundKeyTemplate.java @@ -0,0 +1,23 @@ +package com.linkedin.metadata.aspect.patch.template; + +import static com.linkedin.metadata.aspect.patch.template.TemplateUtil.populateTopLevelKeys; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonpatch.JsonPatchException; +import com.github.fge.jsonpatch.Patch; +import com.linkedin.data.template.RecordTemplate; + +public abstract class CompoundKeyTemplate + implements ArrayMergingTemplate { + + @Override + public T applyPatch(RecordTemplate recordTemplate, Patch jsonPatch) + throws JsonProcessingException, JsonPatchException { + JsonNode transformed = populateTopLevelKeys(preprocessTemplate(recordTemplate), jsonPatch); + JsonNode patched = jsonPatch.apply(transformed); + JsonNode postProcessed = rebaseFields(patched); + return RecordUtils.toRecordTemplate(getTemplateType(), postProcessed.toString()); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/Template.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/Template.java similarity index 69% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/Template.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/Template.java index 0793cacce780f..bd8cd544fb59b 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/Template.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/Template.java @@ -1,6 +1,7 @@ -package com.linkedin.metadata.models.registry.template; +package com.linkedin.metadata.aspect.patch.template; -import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; +import static com.linkedin.metadata.aspect.patch.template.TemplateUtil.OBJECT_MAPPER; +import static com.linkedin.metadata.aspect.patch.template.TemplateUtil.populateTopLevelKeys; import com.datahub.util.RecordUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -19,7 +20,12 @@ public interface Template { * @return specific type for this template * @throws {@link ClassCastException} when recordTemplate is not the correct type for the template */ - T getSubtype(RecordTemplate recordTemplate) throws ClassCastException; + default T getSubtype(RecordTemplate recordTemplate) throws ClassCastException { + if (getTemplateType().isInstance(recordTemplate)) { + return getTemplateType().cast(recordTemplate); + } + throw new ClassCastException("Unable to cast RecordTemplate to " + getTemplateType().getName()); + } /** Get the template clas type */ Class getTemplateType(); @@ -43,10 +49,20 @@ public interface Template { */ default T applyPatch(RecordTemplate recordTemplate, Patch jsonPatch) throws JsonProcessingException, JsonPatchException { - JsonNode transformed = preprocessTemplate(recordTemplate); - JsonNode patched = jsonPatch.apply(transformed); - JsonNode postProcessed = rebaseFields(patched); - return RecordUtils.toRecordTemplate(getTemplateType(), postProcessed.toString()); + + TemplateUtil.validatePatch(jsonPatch); + JsonNode transformed = populateTopLevelKeys(preprocessTemplate(recordTemplate), jsonPatch); + try { + JsonNode patched = jsonPatch.apply(transformed); + JsonNode postProcessed = rebaseFields(patched); + return RecordUtils.toRecordTemplate(getTemplateType(), postProcessed.toString()); + } catch (JsonPatchException e) { + throw new RuntimeException( + String.format( + "Error performing JSON PATCH on aspect %s. Patch: %s Target: %s", + recordTemplate.schema().getName(), jsonPatch, transformed.toString()), + e); + } } /** diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java new file mode 100644 index 0000000000000..d998692f2c388 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java @@ -0,0 +1,97 @@ +package com.linkedin.metadata.aspect.patch.template; + +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.INGESTION_MAX_SERIALIZED_STRING_LENGTH; +import static com.linkedin.metadata.Constants.MAX_JACKSON_STRING_SIZE; + +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.fge.jsonpatch.Patch; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import com.linkedin.util.Pair; +import java.util.ArrayList; +import java.util.List; + +public class TemplateUtil { + + private TemplateUtil() {} + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + int maxSize = + Integer.parseInt( + System.getenv() + .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); + OBJECT_MAPPER + .getFactory() + .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + } + + public static List> getPaths(Patch jsonPatch) { + JsonNode patchNode = OBJECT_MAPPER.valueToTree(jsonPatch); + List> paths = new ArrayList<>(); + patchNode + .elements() + .forEachRemaining( + node -> + paths.add( + Pair.of( + PatchOperationType.valueOf(node.get("op").asText().toUpperCase()), + node.get("path").asText()))); + return paths; + } + + public static void validatePatch(Patch jsonPatch) { + // ensure supported patch operations + JsonNode patchNode = OBJECT_MAPPER.valueToTree(jsonPatch); + patchNode + .elements() + .forEachRemaining( + node -> { + try { + PatchOperationType.valueOf(node.get("op").asText().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Unsupported PATCH operation: `%s` Operation `%s`", + node.get("op").asText(), node), + e); + } + }); + } + + /** + * Necessary step for templates with compound keys due to JsonPatch not allowing non-existent + * paths to be specified + * + * @param transformedNode transformed node to have keys populated + * @return transformed node that has top level keys populated + */ + public static JsonNode populateTopLevelKeys(JsonNode transformedNode, Patch jsonPatch) { + JsonNode transformedNodeClone = transformedNode.deepCopy(); + List> paths = getPaths(jsonPatch); + for (Pair operationPath : paths) { + String[] keys = operationPath.getSecond().split("/"); + JsonNode parent = transformedNodeClone; + + // if not remove, skip last key as we only need to populate top level + int endIdx = + PatchOperationType.REMOVE.equals(operationPath.getFirst()) + ? keys.length + : keys.length - 1; + + // Skip first as it will always be blank due to path starting with / + for (int i = 1; i < endIdx; i++) { + if (parent.get(keys[i]) == null) { + ((ObjectNode) parent).set(keys[i], instance.objectNode()); + } + parent = parent.get(keys[i]); + } + } + + return transformedNodeClone; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/chart/ChartInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/chart/ChartInfoTemplate.java similarity index 92% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/chart/ChartInfoTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/chart/ChartInfoTemplate.java index 654f923e7322d..aabc5b54cfa5c 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/chart/ChartInfoTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/chart/ChartInfoTemplate.java @@ -1,6 +1,6 @@ -package com.linkedin.metadata.models.registry.template.chart; +package com.linkedin.metadata.aspect.patch.template.chart; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.chart.ChartDataSourceTypeArray; @@ -10,7 +10,7 @@ import com.linkedin.common.EdgeArray; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GenericPatchTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GenericPatchTemplate.java new file mode 100644 index 0000000000000..3a3e3c99f25a3 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GenericPatchTemplate.java @@ -0,0 +1,59 @@ +package com.linkedin.metadata.aspect.patch.template.common; + +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonpatch.JsonPatchException; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import javax.annotation.Nonnull; +import lombok.Builder; + +@Builder +public class GenericPatchTemplate extends CompoundKeyTemplate { + + @Nonnull private final GenericJsonPatch genericJsonPatch; + @Nonnull private final Class templateType; + @Nonnull private final T templateDefault; + + @Nonnull + @Override + public Class getTemplateType() { + return templateType; + } + + @Nonnull + @Override + public T getDefault() { + return templateDefault; + } + + @Nonnull + @Override + public JsonNode transformFields(final JsonNode baseNode) { + JsonNode transformedNode = baseNode; + for (Map.Entry> composite : + genericJsonPatch.getArrayPrimaryKeys().entrySet()) { + transformedNode = arrayFieldToMap(transformedNode, composite.getKey(), composite.getValue()); + } + return transformedNode; + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + JsonNode transformedNode = patched; + for (Map.Entry> composite : + genericJsonPatch.getArrayPrimaryKeys().entrySet()) { + transformedNode = + transformedMapToArray(transformedNode, composite.getKey(), composite.getValue()); + } + return transformedNode; + } + + public T applyPatch(RecordTemplate recordTemplate) throws IOException, JsonPatchException { + return super.applyPatch(recordTemplate, genericJsonPatch.getJsonPatch()); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlobalTagsTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlobalTagsTemplate.java similarity index 90% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlobalTagsTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlobalTagsTemplate.java index a98e60c739749..dac5e89edc88e 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlobalTagsTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlobalTagsTemplate.java @@ -1,10 +1,10 @@ -package com.linkedin.metadata.models.registry.template.common; +package com.linkedin.metadata.aspect.patch.template.common; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.common.GlobalTags; import com.linkedin.common.TagAssociationArray; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlossaryTermsTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlossaryTermsTemplate.java similarity index 92% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlossaryTermsTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlossaryTermsTemplate.java index 7ce59916f2073..e6dd1fd523006 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/GlossaryTermsTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/GlossaryTermsTemplate.java @@ -1,7 +1,7 @@ -package com.linkedin.metadata.models.registry.template.common; +package com.linkedin.metadata.aspect.patch.template.common; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -10,7 +10,7 @@ import com.linkedin.common.GlossaryTerms; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/OwnershipTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/OwnershipTemplate.java similarity index 89% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/OwnershipTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/OwnershipTemplate.java index b850ae830b98c..0eaed27ec4cb7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/common/OwnershipTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/OwnershipTemplate.java @@ -1,6 +1,6 @@ -package com.linkedin.metadata.models.registry.template.common; +package com.linkedin.metadata.aspect.patch.template.common; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.common.AuditStamp; @@ -8,7 +8,7 @@ import com.linkedin.common.Ownership; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.CompoundKeyTemplate; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; import java.util.Arrays; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/StructuredPropertiesTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/StructuredPropertiesTemplate.java new file mode 100644 index 0000000000000..df3d682632bca --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/common/StructuredPropertiesTemplate.java @@ -0,0 +1,56 @@ +package com.linkedin.metadata.aspect.patch.template.common; + +import com.fasterxml.jackson.databind.JsonNode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; +import java.util.Collections; +import javax.annotation.Nonnull; + +public class StructuredPropertiesTemplate implements ArrayMergingTemplate { + + private static final String PROPERTIES_FIELD_NAME = "properties"; + private static final String URN_FIELD_NAME = "propertyUrn"; + + // private static final String AUDIT_STAMP_FIELD = "auditStamp"; + // private static final String TIME_FIELD = "time"; + // private static final String ACTOR_FIELD = "actor"; + + @Override + public StructuredProperties getSubtype(RecordTemplate recordTemplate) throws ClassCastException { + if (recordTemplate instanceof StructuredProperties) { + return (StructuredProperties) recordTemplate; + } + throw new ClassCastException("Unable to cast RecordTemplate to StructuredProperties"); + } + + @Override + public Class getTemplateType() { + return StructuredProperties.class; + } + + @Nonnull + @Override + public StructuredProperties getDefault() { + StructuredProperties structuredProperties = new StructuredProperties(); + structuredProperties.setProperties(new StructuredPropertyValueAssignmentArray()); + // .setAuditStamp(new + // AuditStamp().setActor(UrnUtils.getUrn(SYSTEM_ACTOR)).setTime(System.currentTimeMillis())); + return structuredProperties; + } + + @Nonnull + @Override + public JsonNode transformFields(JsonNode baseNode) { + return arrayFieldToMap( + baseNode, PROPERTIES_FIELD_NAME, Collections.singletonList(URN_FIELD_NAME)); + } + + @Nonnull + @Override + public JsonNode rebaseFields(JsonNode patched) { + return transformedMapToArray( + patched, PROPERTIES_FIELD_NAME, Collections.singletonList(URN_FIELD_NAME)); + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dashboard/DashboardInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java similarity index 94% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dashboard/DashboardInfoTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java index eae04b5285adf..85ce06b01c1d7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dashboard/DashboardInfoTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dashboard/DashboardInfoTemplate.java @@ -1,6 +1,6 @@ -package com.linkedin.metadata.models.registry.template.dashboard; +package com.linkedin.metadata.aspect.patch.template.dashboard; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.common.AuditStamp; @@ -11,7 +11,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.dashboard.DashboardInfo; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataflow/DataFlowInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataflow/DataFlowInfoTemplate.java similarity index 89% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataflow/DataFlowInfoTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataflow/DataFlowInfoTemplate.java index 73e837f368f0b..28ee769521995 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataflow/DataFlowInfoTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataflow/DataFlowInfoTemplate.java @@ -1,10 +1,10 @@ -package com.linkedin.metadata.models.registry.template.dataflow; +package com.linkedin.metadata.aspect.patch.template.dataflow; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringMap; import com.linkedin.datajob.DataFlowInfo; -import com.linkedin.metadata.models.registry.template.Template; +import com.linkedin.metadata.aspect.patch.template.Template; import javax.annotation.Nonnull; public class DataFlowInfoTemplate implements Template { diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInfoTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInfoTemplate.java similarity index 89% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInfoTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInfoTemplate.java index bdb306c2d32e4..7cb986da0cba6 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInfoTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInfoTemplate.java @@ -1,10 +1,10 @@ -package com.linkedin.metadata.models.registry.template.datajob; +package com.linkedin.metadata.aspect.patch.template.datajob; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringMap; import com.linkedin.datajob.DataJobInfo; -import com.linkedin.metadata.models.registry.template.Template; +import com.linkedin.metadata.aspect.patch.template.Template; import javax.annotation.Nonnull; public class DataJobInfoTemplate implements Template { diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java similarity index 96% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java index 6761892b1b31b..3d398d97b50c3 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/datajob/DataJobInputOutputTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/datajob/DataJobInputOutputTemplate.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.models.registry.template.datajob; +package com.linkedin.metadata.aspect.patch.template.datajob; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.common.DataJobUrnArray; @@ -8,7 +8,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.datajob.DataJobInputOutput; import com.linkedin.dataset.FineGrainedLineageArray; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataproduct/DataProductPropertiesTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataproduct/DataProductPropertiesTemplate.java similarity index 91% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataproduct/DataProductPropertiesTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataproduct/DataProductPropertiesTemplate.java index 899c51a7c3d7e..9b117114395b1 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataproduct/DataProductPropertiesTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataproduct/DataProductPropertiesTemplate.java @@ -1,10 +1,10 @@ -package com.linkedin.metadata.models.registry.template.dataproduct; +package com.linkedin.metadata.aspect.patch.template.dataproduct; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.data.template.RecordTemplate; import com.linkedin.dataproduct.DataProductAssociationArray; import com.linkedin.dataproduct.DataProductProperties; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/DatasetPropertiesTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/DatasetPropertiesTemplate.java similarity index 91% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/DatasetPropertiesTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/DatasetPropertiesTemplate.java index 991f7f3d4053a..cf76bed2fd3f7 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/DatasetPropertiesTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/DatasetPropertiesTemplate.java @@ -1,11 +1,11 @@ -package com.linkedin.metadata.models.registry.template.dataset; +package com.linkedin.metadata.aspect.patch.template.dataset; import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.data.template.StringMap; import com.linkedin.dataset.DatasetProperties; -import com.linkedin.metadata.models.registry.template.ArrayMergingTemplate; +import com.linkedin.metadata.aspect.patch.template.ArrayMergingTemplate; import java.util.Collections; import javax.annotation.Nonnull; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/EditableSchemaMetadataTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/EditableSchemaMetadataTemplate.java similarity index 92% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/EditableSchemaMetadataTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/EditableSchemaMetadataTemplate.java index 9712a9081d33a..0b3605708e610 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/EditableSchemaMetadataTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/EditableSchemaMetadataTemplate.java @@ -1,15 +1,15 @@ -package com.linkedin.metadata.models.registry.template.dataset; +package com.linkedin.metadata.aspect.patch.template.dataset; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.metadata.models.registry.template.CompoundKeyTemplate; -import com.linkedin.metadata.models.registry.template.common.GlobalTagsTemplate; -import com.linkedin.metadata.models.registry.template.common.GlossaryTermsTemplate; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; +import com.linkedin.metadata.aspect.patch.template.common.GlobalTagsTemplate; +import com.linkedin.metadata.aspect.patch.template.common.GlossaryTermsTemplate; import com.linkedin.schema.EditableSchemaFieldInfoArray; import com.linkedin.schema.EditableSchemaMetadata; import java.util.Collections; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/UpstreamLineageTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java similarity index 96% rename from entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/UpstreamLineageTemplate.java rename to entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java index 81a4065dedb1a..6907181b3f7ff 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/dataset/UpstreamLineageTemplate.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/dataset/UpstreamLineageTemplate.java @@ -1,7 +1,10 @@ -package com.linkedin.metadata.models.registry.template.dataset; +package com.linkedin.metadata.aspect.patch.template.dataset; -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.Constants.*; +import static com.fasterxml.jackson.databind.node.JsonNodeFactory.instance; +import static com.linkedin.metadata.Constants.FINE_GRAINED_LINEAGE_DATASET_TYPE; +import static com.linkedin.metadata.Constants.FINE_GRAINED_LINEAGE_FIELD_SET_TYPE; +import static com.linkedin.metadata.Constants.FINE_GRAINED_LINEAGE_FIELD_TYPE; +import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -13,7 +16,7 @@ import com.linkedin.dataset.FineGrainedLineageArray; import com.linkedin.dataset.UpstreamArray; import com.linkedin.dataset.UpstreamLineage; -import com.linkedin.metadata.models.registry.template.CompoundKeyTemplate; +import com.linkedin.metadata.aspect.patch.template.CompoundKeyTemplate; import java.util.Collections; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java index dd9bbcda8f4af..5f35cb0447e48 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginFactory.java @@ -12,6 +12,7 @@ import io.github.classgraph.ClassInfo; import io.github.classgraph.MethodInfo; import io.github.classgraph.ScanResult; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -27,13 +28,20 @@ @Slf4j public class PluginFactory { + private static final String[] VALIDATOR_PACKAGES = { + "com.linkedin.metadata.aspect.plugins.validation", "com.linkedin.metadata.aspect.validation" + }; + private static final String[] HOOK_PACKAGES = { + "com.linkedin.metadata.aspect.plugins.hooks", "com.linkedin.metadata.aspect.hooks" + }; + public static PluginFactory withCustomClasspath( @Nullable PluginConfiguration pluginConfiguration, @Nonnull List classLoaders) { return new PluginFactory(pluginConfiguration, classLoaders); } public static PluginFactory withConfig(@Nullable PluginConfiguration pluginConfiguration) { - return PluginFactory.withCustomClasspath(pluginConfiguration, List.of()); + return PluginFactory.withCustomClasspath(pluginConfiguration, Collections.emptyList()); } public static PluginFactory empty() { @@ -173,44 +181,35 @@ public EntityRegistryLoadResult.PluginLoadResult getPluginLoadResult() { private List buildAspectPayloadValidators( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( build( AspectPayloadValidator.class, pluginConfiguration.getAspectPayloadValidators(), - "com.linkedin.metadata.aspect.plugins.validation")); + VALIDATOR_PACKAGES)); } private List buildMutationHooks(@Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( - build( - MutationHook.class, - pluginConfiguration.getMutationHooks(), - "com.linkedin.metadata.aspect.plugins.hooks")); + build(MutationHook.class, pluginConfiguration.getMutationHooks(), HOOK_PACKAGES)); } private List buildMCLSideEffects( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( - build( - MCLSideEffect.class, - pluginConfiguration.getMclSideEffects(), - "com.linkedin.metadata.aspect.plugins.hooks")); + build(MCLSideEffect.class, pluginConfiguration.getMclSideEffects(), HOOK_PACKAGES)); } private List buildMCPSideEffects( @Nullable PluginConfiguration pluginConfiguration) { return pluginConfiguration == null - ? List.of() + ? Collections.emptyList() : applyDisable( - build( - MCPSideEffect.class, - pluginConfiguration.getMcpSideEffects(), - "com.linkedin.metadata.aspect.plugins.hooks")); + build(MCPSideEffect.class, pluginConfiguration.getMcpSideEffects(), HOOK_PACKAGES)); } private List build( @@ -226,6 +225,11 @@ private List build( config -> { try { ClassInfo classInfo = classMap.get(config.getClassName()); + if (classInfo == null) { + throw new IllegalStateException( + String.format( + "The following class cannot be loaded: %s", config.getClassName())); + } MethodInfo constructorMethod = classInfo.getConstructorInfo().get(0); return Stream.of( (T) constructorMethod.loadClassAndGetConstructor().newInstance(config)); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java index 03a0473677fb8..d88b05ede8454 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/PluginSpec.java @@ -38,9 +38,11 @@ && isChangeTypeSupported(changeType) protected boolean isEntityAspectSupported( @Nonnull String entityName, @Nonnull String aspectName) { - return (ENTITY_WILDCARD.equals(entityName) - || getConfig().getSupportedEntityAspectNames().stream() - .anyMatch(supported -> supported.getEntityName().equals(entityName))) + return (getConfig().getSupportedEntityAspectNames().stream() + .anyMatch( + supported -> + ENTITY_WILDCARD.equals(supported.getEntityName()) + || supported.getEntityName().equals(entityName))) && isAspectSupported(aspectName); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java index a4d0678c130f3..a2caab7be5f80 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/config/PluginConfiguration.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.aspect.plugins.config; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -11,10 +12,10 @@ @AllArgsConstructor @NoArgsConstructor public class PluginConfiguration { - private List aspectPayloadValidators = List.of(); - private List mutationHooks = List.of(); - private List mclSideEffects = List.of(); - private List mcpSideEffects = List.of(); + private List aspectPayloadValidators = Collections.emptyList(); + private List mutationHooks = Collections.emptyList(); + private List mclSideEffects = Collections.emptyList(); + private List mcpSideEffects = Collections.emptyList(); public static PluginConfiguration EMPTY = new PluginConfiguration(); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java index ef9786f8d711e..a21f3cd2436de 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffect.java @@ -4,7 +4,6 @@ import com.linkedin.metadata.aspect.plugins.PluginSpec; import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; -import com.linkedin.metadata.models.registry.EntityRegistry; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -23,16 +22,12 @@ public MCLSideEffect(AspectPluginConfig aspectPluginConfig) { * @return additional upserts */ public final Stream apply( - @Nonnull List input, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) { + @Nonnull List input, @Nonnull AspectRetriever aspectRetriever) { return input.stream() .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec())) - .flatMap(i -> applyMCLSideEffect(i, entityRegistry, aspectRetriever)); + .flatMap(i -> applyMCLSideEffect(i, aspectRetriever)); } protected abstract Stream applyMCLSideEffect( - @Nonnull MCLBatchItem input, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever); + @Nonnull MCLBatchItem input, @Nonnull AspectRetriever aspectRetriever); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java index fc1d1587d10fb..80cb405201c87 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffect.java @@ -4,7 +4,6 @@ import com.linkedin.metadata.aspect.plugins.PluginSpec; import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; -import com.linkedin.metadata.models.registry.EntityRegistry; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -23,14 +22,12 @@ public MCPSideEffect(AspectPluginConfig aspectPluginConfig) { * @return additional upserts */ public final Stream apply( - List input, - EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) { + List input, @Nonnull AspectRetriever aspectRetriever) { return input.stream() .filter(item -> shouldApply(item.getChangeType(), item.getUrn(), item.getAspectSpec())) - .flatMap(i -> applyMCPSideEffect(i, entityRegistry, aspectRetriever)); + .flatMap(i -> applyMCPSideEffect(i, aspectRetriever)); } protected abstract Stream applyMCPSideEffect( - UpsertItem input, EntityRegistry entityRegistry, @Nonnull AspectRetriever aspectRetriever); + UpsertItem input, @Nonnull AspectRetriever aspectRetriever); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java index 78aa4689472f5..11cd2352025ef 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/plugins/validation/AspectRetriever.java @@ -1,13 +1,38 @@ package com.linkedin.metadata.aspect.plugins.validation; +import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.entity.Aspect; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.r2.RemoteInvocationException; import java.net.URISyntaxException; +import java.util.Collections; +import java.util.Map; +import java.util.Set; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public interface AspectRetriever { - Aspect getLatestAspectObject(@Nonnull final Urn urn, @Nonnull final String aspectName) + @Nullable + default Aspect getLatestAspectObject(@Nonnull final Urn urn, @Nonnull final String aspectName) + throws RemoteInvocationException, URISyntaxException { + return getLatestAspectObjects(ImmutableSet.of(urn), ImmutableSet.of(aspectName)) + .getOrDefault(urn, Collections.emptyMap()) + .get(aspectName); + } + + /** + * Returns for each URN, the map of aspectName to Aspect + * + * @param urns urns to fetch + * @param aspectNames aspect names + * @return urn to aspect name and values + */ + @Nonnull + Map> getLatestAspectObjects(Set urns, Set aspectNames) throws RemoteInvocationException, URISyntaxException; + + @Nonnull + EntityRegistry getEntityRegistry(); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java new file mode 100644 index 0000000000000..5a4635da433ae --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/PropertyDefinitionValidator.java @@ -0,0 +1,91 @@ +package com.linkedin.metadata.aspect.validation; + +import static com.linkedin.structured.PropertyCardinality.*; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class PropertyDefinitionValidator extends AspectPayloadValidator { + + public PropertyDefinitionValidator(AspectPluginConfig aspectPluginConfig) { + super(aspectPluginConfig); + } + + @Override + protected void validateProposedAspect( + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull AspectSpec aspectSpec, + @Nonnull RecordTemplate aspectPayload, + @Nonnull AspectRetriever aspectRetriever) + throws AspectValidationException { + // No-op + } + + @Override + protected void validatePreCommitAspect( + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull AspectSpec aspectSpec, + @Nullable RecordTemplate previousAspect, + @Nonnull RecordTemplate proposedAspect, + AspectRetriever aspectRetriever) + throws AspectValidationException { + validate(previousAspect, proposedAspect); + } + + public static boolean validate( + @Nullable RecordTemplate previousAspect, @Nonnull RecordTemplate proposedAspect) + throws AspectValidationException { + if (previousAspect != null) { + StructuredPropertyDefinition previousDefinition = + (StructuredPropertyDefinition) previousAspect; + StructuredPropertyDefinition newDefinition = (StructuredPropertyDefinition) proposedAspect; + if (!newDefinition.getValueType().equals(previousDefinition.getValueType())) { + throw new AspectValidationException( + "Value type cannot be changed as this is a backwards incompatible change"); + } + if (newDefinition.getCardinality().equals(SINGLE) + && previousDefinition.getCardinality().equals(MULTIPLE)) { + throw new AspectValidationException( + "Property definition cardinality cannot be changed from MULTI to SINGLE"); + } + if (!newDefinition.getQualifiedName().equals(previousDefinition.getQualifiedName())) { + throw new AspectValidationException( + "Cannot change the fully qualified name of a Structured Property"); + } + // Assure new definition has only added allowed values, not removed them + if (newDefinition.getAllowedValues() != null) { + if (!previousDefinition.hasAllowedValues() + || previousDefinition.getAllowedValues() == null) { + throw new AspectValidationException( + "Cannot restrict values that were previously allowed"); + } + Set newAllowedValues = + newDefinition.getAllowedValues().stream() + .map(PropertyValue::getValue) + .collect(Collectors.toSet()); + for (PropertyValue value : previousDefinition.getAllowedValues()) { + if (!newAllowedValues.contains(value.getValue())) { + throw new AspectValidationException( + "Cannot restrict values that were previously allowed"); + } + } + } + } + return true; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java new file mode 100644 index 0000000000000..efd95e0c2e3f1 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/validation/StructuredPropertiesValidator.java @@ -0,0 +1,264 @@ +package com.linkedin.metadata.aspect.validation; + +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.StringArrayMap; +import com.linkedin.entity.Aspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; +import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.LogicalValueType; +import com.linkedin.metadata.models.StructuredPropertyUtils; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PrimitivePropertyValueArray; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** A Validator for StructuredProperties Aspect that is attached to entities like Datasets, etc. */ +@Slf4j +public class StructuredPropertiesValidator extends AspectPayloadValidator { + + private static final Set VALID_VALUE_STORED_AS_STRING = + new HashSet<>( + Arrays.asList( + LogicalValueType.STRING, + LogicalValueType.RICH_TEXT, + LogicalValueType.DATE, + LogicalValueType.URN)); + + public StructuredPropertiesValidator(AspectPluginConfig aspectPluginConfig) { + super(aspectPluginConfig); + } + + public static LogicalValueType getLogicalValueType(Urn valueType) { + String valueTypeId = getValueTypeId(valueType); + if (valueTypeId.equals("string")) { + return LogicalValueType.STRING; + } else if (valueTypeId.equals("date")) { + return LogicalValueType.DATE; + } else if (valueTypeId.equals("number")) { + return LogicalValueType.NUMBER; + } else if (valueTypeId.equals("urn")) { + return LogicalValueType.URN; + } else if (valueTypeId.equals("rich_text")) { + return LogicalValueType.RICH_TEXT; + } + + return LogicalValueType.UNKNOWN; + } + + @Override + protected void validateProposedAspect( + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull AspectSpec aspectSpec, + @Nonnull RecordTemplate aspectPayload, + @Nonnull AspectRetriever aspectRetriever) + throws AspectValidationException { + validate(aspectPayload, aspectRetriever); + } + + public static boolean validate( + @Nonnull RecordTemplate aspectPayload, @Nonnull AspectRetriever aspectRetriever) + throws AspectValidationException { + StructuredProperties structuredProperties = (StructuredProperties) aspectPayload; + log.warn("Validator called with {}", structuredProperties); + Map> structuredPropertiesMap = + structuredProperties.getProperties().stream() + .collect( + Collectors.groupingBy( + x -> x.getPropertyUrn(), + HashMap::new, + Collectors.toCollection(ArrayList::new))); + for (Map.Entry> entry : + structuredPropertiesMap.entrySet()) { + // There should only be one entry per structured property + List values = entry.getValue(); + if (values.size() > 1) { + throw new AspectValidationException( + "Property: " + entry.getKey() + " has multiple entries: " + values); + } + } + + for (StructuredPropertyValueAssignment structuredPropertyValueAssignment : + structuredProperties.getProperties()) { + Urn propertyUrn = structuredPropertyValueAssignment.getPropertyUrn(); + String property = propertyUrn.toString(); + if (!propertyUrn.getEntityType().equals("structuredProperty")) { + throw new IllegalStateException( + "Unexpected entity type. Expected: structuredProperty Found: " + + propertyUrn.getEntityType()); + } + Aspect structuredPropertyDefinitionAspect = null; + try { + structuredPropertyDefinitionAspect = + aspectRetriever.getLatestAspectObject(propertyUrn, "propertyDefinition"); + + if (structuredPropertyDefinitionAspect == null) { + throw new AspectValidationException("Unexpected null value found."); + } + } catch (Exception e) { + log.error("Could not fetch latest aspect. PropertyUrn: {}", propertyUrn, e); + throw new AspectValidationException("Could not fetch latest aspect: " + e.getMessage(), e); + } + + StructuredPropertyDefinition structuredPropertyDefinition = + new StructuredPropertyDefinition(structuredPropertyDefinitionAspect.data()); + log.warn( + "Retrieved property definition for {}. {}", propertyUrn, structuredPropertyDefinition); + if (structuredPropertyDefinition != null) { + PrimitivePropertyValueArray values = structuredPropertyValueAssignment.getValues(); + // Check cardinality + if (structuredPropertyDefinition.getCardinality() == PropertyCardinality.SINGLE) { + if (values.size() > 1) { + throw new AspectValidationException( + "Property: " + + property + + " has cardinality 1, but multiple values were assigned: " + + values); + } + } + // Check values + for (PrimitivePropertyValue value : values) { + validateType(propertyUrn, structuredPropertyDefinition, value); + validateAllowedValues(propertyUrn, structuredPropertyDefinition, value); + } + } + } + + return true; + } + + private static void validateAllowedValues( + Urn propertyUrn, StructuredPropertyDefinition definition, PrimitivePropertyValue value) + throws AspectValidationException { + if (definition.getAllowedValues() != null) { + Set definedValues = + definition.getAllowedValues().stream() + .map(PropertyValue::getValue) + .collect(Collectors.toSet()); + if (definedValues.stream().noneMatch(definedPrimitive -> definedPrimitive.equals(value))) { + throw new AspectValidationException( + String.format( + "Property: %s, value: %s should be one of %s", propertyUrn, value, definedValues)); + } + } + } + + private static void validateType( + Urn propertyUrn, StructuredPropertyDefinition definition, PrimitivePropertyValue value) + throws AspectValidationException { + Urn valueType = definition.getValueType(); + LogicalValueType typeDefinition = getLogicalValueType(valueType); + + // Primitive Type Validation + if (VALID_VALUE_STORED_AS_STRING.contains(typeDefinition)) { + log.debug( + "Property definition demands a string value. {}, {}", value.isString(), value.isDouble()); + if (value.getString() == null) { + throw new AspectValidationException( + "Property: " + propertyUrn.toString() + ", value: " + value + " should be a string"); + } else if (typeDefinition.equals(LogicalValueType.DATE)) { + if (!StructuredPropertyUtils.isValidDate(value)) { + throw new AspectValidationException( + "Property: " + + propertyUrn.toString() + + ", value: " + + value + + " should be a date with format YYYY-MM-DD"); + } + } else if (typeDefinition.equals(LogicalValueType.URN)) { + StringArrayMap valueTypeQualifier = definition.getTypeQualifier(); + Urn typeValue; + try { + typeValue = Urn.createFromString(value.getString()); + } catch (URISyntaxException e) { + throw new AspectValidationException( + "Property: " + propertyUrn.toString() + ", value: " + value + " should be an urn", e); + } + if (valueTypeQualifier != null) { + if (valueTypeQualifier.containsKey("allowedTypes")) { + // Let's get the allowed types and validate that the value is one of those types + StringArray allowedTypes = valueTypeQualifier.get("allowedTypes"); + boolean matchedAny = false; + for (String type : allowedTypes) { + Urn typeUrn = null; + try { + typeUrn = Urn.createFromString(type); + } catch (URISyntaxException e) { + + // we don't expect to have types that we allowed to be written that aren't + // urns + throw new RuntimeException(e); + } + String allowedEntityName = getValueTypeId(typeUrn); + if (typeValue.getEntityType().equals(allowedEntityName)) { + matchedAny = true; + } + } + if (!matchedAny) { + throw new AspectValidationException( + "Property: " + + propertyUrn.toString() + + ", value: " + + value + + " is not of any supported urn types:" + + allowedTypes); + } + } + } + } + } else if (typeDefinition.equals(LogicalValueType.NUMBER)) { + log.debug("Property definition demands a numeric value. {}, {}", value.isString(), value); + try { + Double doubleValue = + value.getDouble() != null ? value.getDouble() : Double.parseDouble(value.getString()); + } catch (NumberFormatException | NullPointerException e) { + throw new AspectValidationException( + "Property: " + propertyUrn.toString() + ", value: " + value + " should be a number"); + } + } else { + throw new AspectValidationException( + "Validation support for type " + definition.getValueType() + " is not yet implemented."); + } + } + + private static String getValueTypeId(@Nonnull final Urn valueType) { + String valueTypeId = valueType.getId(); + if (valueTypeId.startsWith("datahub.")) { + valueTypeId = valueTypeId.split("\\.")[1]; + } + return valueTypeId; + } + + @Override + protected void validatePreCommitAspect( + @Nonnull ChangeType changeType, + @Nonnull Urn entityUrn, + @Nonnull AspectSpec aspectSpec, + @Nullable RecordTemplate previousAspect, + @Nonnull RecordTemplate proposedAspect, + AspectRetriever aspectRetriever) + throws AspectValidationException { + // No-op + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/LogicalValueType.java b/entity-registry/src/main/java/com/linkedin/metadata/models/LogicalValueType.java new file mode 100644 index 0000000000000..1643ce900f748 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/LogicalValueType.java @@ -0,0 +1,10 @@ +package com.linkedin.metadata.models; + +public enum LogicalValueType { + STRING, + RICH_TEXT, + NUMBER, + DATE, + URN, + UNKNOWN +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java new file mode 100644 index 0000000000000..a8711429421f3 --- /dev/null +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java @@ -0,0 +1,45 @@ +package com.linkedin.metadata.models; + +import com.linkedin.structured.PrimitivePropertyValue; +import java.sql.Date; +import java.time.format.DateTimeParseException; + +public class StructuredPropertyUtils { + + private StructuredPropertyUtils() {} + + static final Date MIN_DATE = Date.valueOf("1000-01-01"); + static final Date MAX_DATE = Date.valueOf("9999-12-31"); + + /** + * Sanitizes fully qualified name for use in an ElasticSearch field name Replaces . and " " + * characters + * + * @param fullyQualifiedName The original fully qualified name of the property + * @return The sanitized version that can be used as a field name + */ + public static String sanitizeStructuredPropertyFQN(String fullyQualifiedName) { + String sanitizedName = fullyQualifiedName.replace('.', '_').replace(' ', '_'); + return sanitizedName; + } + + public static Date toDate(PrimitivePropertyValue value) throws DateTimeParseException { + return Date.valueOf(value.getString()); + } + + public static boolean isValidDate(PrimitivePropertyValue value) { + if (value.getString() == null) { + return false; + } + if (value.getString().length() != 10) { + return false; + } + Date date; + try { + date = toDate(value); + } catch (DateTimeParseException e) { + return false; + } + return date.compareTo(MIN_DATE) >= 0 && date.compareTo(MAX_DATE) <= 0; + } +} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java index ce8718c536fbe..41043995a3b77 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.linkedin.data.schema.DataSchema; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.aspect.plugins.PluginFactory; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DataSchemaFactory; @@ -18,7 +19,6 @@ import com.linkedin.metadata.models.registry.config.Entities; import com.linkedin.metadata.models.registry.config.Entity; import com.linkedin.metadata.models.registry.config.Event; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.util.Pair; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -28,11 +28,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -67,7 +69,10 @@ public class ConfigEntityRegistry implements EntityRegistry { public ConfigEntityRegistry(Pair configFileClassPathPair) throws IOException { this( DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()), - DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(), + DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()) + .map(Stream::of) + .orElse(Stream.empty()) + .collect(Collectors.toList()), configFileClassPathPair.getFirst()); } @@ -112,7 +117,7 @@ private static Pair getFileAndClassPath(String entityRegistryRoot) } public ConfigEntityRegistry(InputStream configFileInputStream) { - this(DataSchemaFactory.getInstance(), List.of(), configFileInputStream); + this(DataSchemaFactory.getInstance(), Collections.emptyList(), configFileInputStream); } public ConfigEntityRegistry( diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java index fbc3285579cc0..c2aa1fab6c2c0 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.models.registry; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.aspect.plugins.PluginFactory; import com.linkedin.metadata.aspect.plugins.hooks.MCLSideEffect; import com.linkedin.metadata.aspect.plugins.hooks.MCPSideEffect; @@ -10,7 +11,6 @@ import com.linkedin.metadata.models.DefaultEntitySpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EventSpec; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -39,11 +39,10 @@ default String getIdentifier() { EntitySpec getEntitySpec(@Nonnull final String entityName); /** - * Given an event name, returns an instance of {@link DefaultEventSpec}. + * Given an event name, returns an instance of {@link EventSpec}. * * @param eventName the name of the event to be retrieved - * @return an {@link DefaultEventSpec} corresponding to the entity name provided, null if none - * exists. + * @return an {@link EventSpec} corresponding to the entity name provided, null if none exists. */ @Nullable EventSpec getEventSpec(@Nonnull final String eventName); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java index 285b96b93d1d6..650a1cd41066e 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java @@ -3,13 +3,13 @@ import com.linkedin.data.schema.compatibility.CompatibilityChecker; import com.linkedin.data.schema.compatibility.CompatibilityOptions; import com.linkedin.data.schema.compatibility.CompatibilityResult; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.aspect.plugins.PluginFactory; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.ConfigEntitySpec; import com.linkedin.metadata.models.DefaultEntitySpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EventSpec; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java index c605cfa188fc8..b82b905c50004 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.linkedin.data.schema.DataSchema; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.aspect.plugins.PluginFactory; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DataSchemaFactory; @@ -17,7 +18,6 @@ import com.linkedin.metadata.models.registry.config.Entities; import com.linkedin.metadata.models.registry.config.Entity; import com.linkedin.metadata.models.registry.config.Event; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.util.Pair; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -93,7 +94,10 @@ public PatchEntityRegistry( throws IOException, EntityRegistryException { this( DataSchemaFactory.withCustomClasspath(configFileClassPathPair.getSecond()), - DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()).stream().toList(), + DataSchemaFactory.getClassLoader(configFileClassPathPair.getSecond()) + .map(Stream::of) + .orElse(Stream.empty()) + .collect(Collectors.toList()), configFileClassPathPair.getFirst(), registryName, registryVersion); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java index bb0113abc9ed6..8fefa2fe00ae8 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java @@ -5,25 +5,26 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.UnionTemplate; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; +import com.linkedin.metadata.aspect.patch.template.Template; +import com.linkedin.metadata.aspect.patch.template.chart.ChartInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.common.GlobalTagsTemplate; +import com.linkedin.metadata.aspect.patch.template.common.GlossaryTermsTemplate; +import com.linkedin.metadata.aspect.patch.template.common.OwnershipTemplate; +import com.linkedin.metadata.aspect.patch.template.common.StructuredPropertiesTemplate; +import com.linkedin.metadata.aspect.patch.template.dashboard.DashboardInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.dataflow.DataFlowInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.datajob.DataJobInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.datajob.DataJobInputOutputTemplate; +import com.linkedin.metadata.aspect.patch.template.dataproduct.DataProductPropertiesTemplate; +import com.linkedin.metadata.aspect.patch.template.dataset.DatasetPropertiesTemplate; +import com.linkedin.metadata.aspect.patch.template.dataset.EditableSchemaMetadataTemplate; +import com.linkedin.metadata.aspect.patch.template.dataset.UpstreamLineageTemplate; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DefaultEntitySpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EntitySpecBuilder; import com.linkedin.metadata.models.EventSpec; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; -import com.linkedin.metadata.models.registry.template.Template; -import com.linkedin.metadata.models.registry.template.chart.ChartInfoTemplate; -import com.linkedin.metadata.models.registry.template.common.GlobalTagsTemplate; -import com.linkedin.metadata.models.registry.template.common.GlossaryTermsTemplate; -import com.linkedin.metadata.models.registry.template.common.OwnershipTemplate; -import com.linkedin.metadata.models.registry.template.dashboard.DashboardInfoTemplate; -import com.linkedin.metadata.models.registry.template.dataflow.DataFlowInfoTemplate; -import com.linkedin.metadata.models.registry.template.datajob.DataJobInfoTemplate; -import com.linkedin.metadata.models.registry.template.datajob.DataJobInputOutputTemplate; -import com.linkedin.metadata.models.registry.template.dataproduct.DataProductPropertiesTemplate; -import com.linkedin.metadata.models.registry.template.dataset.DatasetPropertiesTemplate; -import com.linkedin.metadata.models.registry.template.dataset.EditableSchemaMetadataTemplate; -import com.linkedin.metadata.models.registry.template.dataset.UpstreamLineageTemplate; import com.linkedin.metadata.snapshot.Snapshot; import java.util.ArrayList; import java.util.HashMap; @@ -84,6 +85,8 @@ private AspectTemplateEngine populateTemplateEngine(Map aspe aspectSpecTemplateMap.put(CHART_INFO_ASPECT_NAME, new ChartInfoTemplate()); aspectSpecTemplateMap.put(DASHBOARD_INFO_ASPECT_NAME, new DashboardInfoTemplate()); aspectSpecTemplateMap.put(DATA_JOB_INPUT_OUTPUT_ASPECT_NAME, new DataJobInputOutputTemplate()); + aspectSpecTemplateMap.put( + STRUCTURED_PROPERTIES_ASPECT_NAME, new StructuredPropertiesTemplate()); return new AspectTemplateEngine(aspectSpecTemplateMap); } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java index 076387909326b..12a29a7e1757a 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/EntityRegistryLoadResult.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.models.registry.config; +import java.util.Collections; import java.util.Set; import lombok.Builder; import lombok.Data; @@ -23,9 +24,9 @@ public static class PluginLoadResult { private int mcpSideEffectCount; private int mclSideEffectCount; - @Builder.Default private Set validatorClasses = Set.of(); - @Builder.Default private Set mutationHookClasses = Set.of(); - @Builder.Default private Set mcpSideEffectClasses = Set.of(); - @Builder.Default private Set mclSideEffectClasses = Set.of(); + @Builder.Default private Set validatorClasses = Collections.emptySet(); + @Builder.Default private Set mutationHookClasses = Collections.emptySet(); + @Builder.Default private Set mcpSideEffectClasses = Collections.emptySet(); + @Builder.Default private Set mclSideEffectClasses = Collections.emptySet(); } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/CompoundKeyTemplate.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/CompoundKeyTemplate.java deleted file mode 100644 index 44090b3a6d05b..0000000000000 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/CompoundKeyTemplate.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.linkedin.metadata.models.registry.template; - -import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; -import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; - -import com.datahub.util.RecordUtils; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.github.fge.jsonpatch.JsonPatchException; -import com.github.fge.jsonpatch.Patch; -import com.linkedin.data.template.RecordTemplate; -import java.util.List; - -public abstract class CompoundKeyTemplate - implements ArrayMergingTemplate { - - /** - * Necessary step for templates with compound keys due to JsonPatch not allowing non-existent - * paths to be specified - * - * @param transformedNode transformed node to have keys populated - * @return transformed node that has top level keys populated - */ - public JsonNode populateTopLevelKeys(JsonNode transformedNode, Patch jsonPatch) { - JsonNode transformedNodeClone = transformedNode.deepCopy(); - List paths = getPaths(jsonPatch); - for (String path : paths) { - String[] keys = path.split("/"); - // Skip first as it will always be blank due to path starting with /, skip last key as we only - // need to populate top level - JsonNode parent = transformedNodeClone; - for (int i = 1; i < keys.length - 1; i++) { - if (parent.get(keys[i]) == null) { - ((ObjectNode) parent).set(keys[i], instance.objectNode()); - } - parent = parent.get(keys[i]); - } - } - - return transformedNodeClone; - } - - @Override - public T applyPatch(RecordTemplate recordTemplate, Patch jsonPatch) - throws JsonProcessingException, JsonPatchException { - JsonNode transformed = populateTopLevelKeys(preprocessTemplate(recordTemplate), jsonPatch); - JsonNode patched = jsonPatch.apply(transformed); - JsonNode postProcessed = rebaseFields(patched); - return RecordUtils.toRecordTemplate(getTemplateType(), postProcessed.toString()); - } -} diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/util/TemplateUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/util/TemplateUtil.java deleted file mode 100644 index 18d070ec3da45..0000000000000 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/template/util/TemplateUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.linkedin.metadata.models.registry.template.util; - -import static com.linkedin.metadata.Constants.*; - -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonpatch.Patch; -import java.util.ArrayList; -import java.util.List; - -public class TemplateUtil { - - private TemplateUtil() {} - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - static { - int maxSize = - Integer.parseInt( - System.getenv() - .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); - OBJECT_MAPPER - .getFactory() - .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); - } - - public static List getPaths(Patch jsonPatch) { - JsonNode patchNode = OBJECT_MAPPER.valueToTree(jsonPatch); - List paths = new ArrayList<>(); - patchNode - .elements() - .forEachRemaining( - node -> { - paths.add(node.get("path").asText()); - }); - return paths; - } -} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/ChartInfoTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/ChartInfoTemplateTest.java similarity index 92% rename from entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/ChartInfoTemplateTest.java rename to entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/ChartInfoTemplateTest.java index 108936bde2ed5..b2911100519fc 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/ChartInfoTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/ChartInfoTemplateTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.models.registry; +package com.linkedin.metadata.aspect.patch.template; import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; @@ -9,7 +9,7 @@ import com.github.fge.jsonpatch.JsonPatchOperation; import com.linkedin.chart.ChartInfo; import com.linkedin.common.urn.UrnUtils; -import com.linkedin.metadata.models.registry.template.chart.ChartInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.chart.ChartInfoTemplate; import java.util.ArrayList; import java.util.List; import org.testng.Assert; diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/DashboardInfoTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java similarity index 91% rename from entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/DashboardInfoTemplateTest.java rename to entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java index 962ff1d40d873..be15d6976aee6 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/DashboardInfoTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/DashboardInfoTemplateTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.models.registry.patch; +package com.linkedin.metadata.aspect.patch.template; import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; @@ -9,7 +9,7 @@ import com.github.fge.jsonpatch.JsonPatchOperation; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dashboard.DashboardInfo; -import com.linkedin.metadata.models.registry.template.dashboard.DashboardInfoTemplate; +import com.linkedin.metadata.aspect.patch.template.dashboard.DashboardInfoTemplate; import java.util.ArrayList; import java.util.List; import org.testng.Assert; diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java similarity index 99% rename from entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/UpstreamLineageTemplateTest.java rename to entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java index 8f410ae8da085..7d59664513d57 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/patch/UpstreamLineageTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java @@ -1,4 +1,4 @@ -package com.linkedin.metadata.models.registry.patch; +package com.linkedin.metadata.aspect.patch.template; import static com.fasterxml.jackson.databind.node.JsonNodeFactory.*; @@ -16,7 +16,7 @@ import com.linkedin.dataset.FineGrainedLineageDownstreamType; import com.linkedin.dataset.FineGrainedLineageUpstreamType; import com.linkedin.dataset.UpstreamLineage; -import com.linkedin.metadata.models.registry.template.dataset.UpstreamLineageTemplate; +import com.linkedin.metadata.aspect.patch.template.dataset.UpstreamLineageTemplate; import java.util.ArrayList; import java.util.List; import org.testng.Assert; diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java index 8c3f71fcc8019..f801ce7bf1ffe 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/PluginsTest.java @@ -61,17 +61,16 @@ public void testConfigEntityRegistry() throws FileNotFoundException { assertNotNull(eventSpec.getPegasusSchema()); assertEquals( - configEntityRegistry.getAspectPayloadValidators(ChangeType.UPSERT, "*", "status").size(), + configEntityRegistry + .getAspectPayloadValidators(ChangeType.UPSERT, "chart", "status") + .size(), 2); assertEquals( - configEntityRegistry.getAspectPayloadValidators(ChangeType.DELETE, "*", "status").size(), + configEntityRegistry + .getAspectPayloadValidators(ChangeType.DELETE, "chart", "status") + .size(), 0); - assertEquals( - configEntityRegistry.getMCLSideEffects(ChangeType.UPSERT, "chart", "chartInfo").size(), 1); - assertEquals( - configEntityRegistry.getMCLSideEffects(ChangeType.DELETE, "chart", "chartInfo").size(), 0); - assertEquals( configEntityRegistry.getMCPSideEffects(ChangeType.UPSERT, "dataset", "datasetKey").size(), 1); @@ -124,17 +123,16 @@ public void testMergedEntityRegistry() throws EntityRegistryException { assertNotNull(eventSpec.getPegasusSchema()); assertEquals( - mergedEntityRegistry.getAspectPayloadValidators(ChangeType.UPSERT, "*", "status").size(), - 3); + mergedEntityRegistry + .getAspectPayloadValidators(ChangeType.UPSERT, "chart", "status") + .size(), + 2); assertEquals( - mergedEntityRegistry.getAspectPayloadValidators(ChangeType.DELETE, "*", "status").size(), + mergedEntityRegistry + .getAspectPayloadValidators(ChangeType.DELETE, "chart", "status") + .size(), 1); - assertEquals( - mergedEntityRegistry.getMCLSideEffects(ChangeType.UPSERT, "chart", "chartInfo").size(), 2); - assertEquals( - mergedEntityRegistry.getMCLSideEffects(ChangeType.DELETE, "chart", "chartInfo").size(), 1); - assertEquals( mergedEntityRegistry.getMCPSideEffects(ChangeType.UPSERT, "dataset", "datasetKey").size(), 2); diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffectTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffectTest.java index ce904142fecfe..8ee5ff4f99820 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffectTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCLSideEffectTest.java @@ -9,7 +9,6 @@ import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; -import com.linkedin.metadata.models.registry.EntityRegistry; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -60,9 +59,7 @@ public TestMCLSideEffect(AspectPluginConfig aspectPluginConfig) { @Override protected Stream applyMCLSideEffect( - @Nonnull MCLBatchItem input, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) { + @Nonnull MCLBatchItem input, @Nonnull AspectRetriever aspectRetriever) { return Stream.of(input); } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java index ee8f947e0e994..8522e8facf3e0 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/hooks/MCPSideEffectTest.java @@ -9,7 +9,6 @@ import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; -import com.linkedin.metadata.models.registry.EntityRegistry; import java.util.List; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -60,7 +59,7 @@ public TestMCPSideEffect(AspectPluginConfig aspectPluginConfig) { @Override protected Stream applyMCPSideEffect( - UpsertItem input, EntityRegistry entityRegistry, @Nonnull AspectRetriever aspectRetriever) { + UpsertItem input, @Nonnull AspectRetriever aspectRetriever) { return Stream.of(input); } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/validation/ValidatorPluginTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/validation/ValidatorPluginTest.java index 07c99ee8546be..eb132836be465 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/validation/ValidatorPluginTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/plugins/validation/ValidatorPluginTest.java @@ -33,7 +33,7 @@ public void testCustomValidator() { TestEntityProfile.class.getClassLoader().getResourceAsStream(REGISTRY_FILE)); List validators = - configEntityRegistry.getAspectPayloadValidators(ChangeType.UPSERT, "*", "status"); + configEntityRegistry.getAspectPayloadValidators(ChangeType.UPSERT, "chart", "status"); assertEquals( validators, List.of( diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java new file mode 100644 index 0000000000000..96e9fceb4a05d --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/PropertyDefinitionValidatorTest.java @@ -0,0 +1,212 @@ +package com.linkedin.metadata.aspect.validators; + +import static org.testng.Assert.*; + +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.validation.PropertyDefinitionValidator; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PropertyCardinality; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.PropertyValueArray; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.net.URISyntaxException; +import org.testng.annotations.Test; + +public class PropertyDefinitionValidatorTest { + @Test + public void testValidatePreCommitNoPrevious() + throws URISyntaxException, AspectValidationException { + StructuredPropertyDefinition newProperty = new StructuredPropertyDefinition(); + newProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + newProperty.setDisplayName("newProp"); + newProperty.setQualifiedName("prop3"); + newProperty.setCardinality(PropertyCardinality.MULTIPLE); + newProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + assertTrue(PropertyDefinitionValidator.validate(null, newProperty)); + } + + @Test + public void testCanChangeSingleToMultiple() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.SINGLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setCardinality(PropertyCardinality.MULTIPLE); + assertTrue(PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCannotChangeMultipleToSingle() + throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setCardinality(PropertyCardinality.SINGLE); + assertThrows( + AspectValidationException.class, + () -> PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCannotChangeValueType() throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); + assertThrows( + AspectValidationException.class, + () -> PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCanChangeDisplayName() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setDisplayName("newProp"); + assertTrue(PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCannotChangeFullyQualifiedName() + throws URISyntaxException, CloneNotSupportedException { + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + newProperty.setQualifiedName("newProp"); + assertThrows( + AspectValidationException.class, + () -> PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCannotChangeRestrictAllowedValues() + throws URISyntaxException, CloneNotSupportedException { + // No constraint -> constraint case + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + PropertyValue allowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(1.0)).setDescription("hello"); + newProperty.setAllowedValues(new PropertyValueArray(allowedValue)); + assertThrows( + AspectValidationException.class, + () -> PropertyDefinitionValidator.validate(oldProperty, newProperty)); + + // Remove allowed values from constraint case + PropertyValue oldAllowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(3.0)).setDescription("hello"); + oldProperty.setAllowedValues((new PropertyValueArray(allowedValue, oldAllowedValue))); + assertThrows( + AspectValidationException.class, + () -> PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCanExpandAllowedValues() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + // Constraint -> no constraint case + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + PropertyValue allowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(1.0)).setDescription("hello"); + oldProperty.setAllowedValues(new PropertyValueArray(allowedValue)); + assertTrue(PropertyDefinitionValidator.validate(oldProperty, newProperty)); + + // Add allowed values to constraint case + PropertyValue newAllowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(3.0)).setDescription("hello"); + newProperty.setAllowedValues((new PropertyValueArray(allowedValue, newAllowedValue))); + assertTrue(PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } + + @Test + public void testCanChangeAllowedValueDescriptions() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + // Constraint -> no constraint case + StructuredPropertyDefinition oldProperty = new StructuredPropertyDefinition(); + oldProperty.setEntityTypes( + new UrnArray( + Urn.createFromString("urn:li:logicalEntity:dataset"), + Urn.createFromString("urn:li:logicalEntity:chart"), + Urn.createFromString("urn:li:logicalEntity:glossaryTerm"))); + oldProperty.setDisplayName("oldProp"); + oldProperty.setQualifiedName("prop3"); + oldProperty.setCardinality(PropertyCardinality.MULTIPLE); + oldProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + StructuredPropertyDefinition newProperty = oldProperty.copy(); + PropertyValue allowedValue = + new PropertyValue().setValue(PrimitivePropertyValue.create(1.0)).setDescription("hello"); + oldProperty.setAllowedValues(new PropertyValueArray(allowedValue)); + PropertyValue newAllowedValue = + new PropertyValue() + .setValue(PrimitivePropertyValue.create(1.0)) + .setDescription("hello there"); + newProperty.setAllowedValues(new PropertyValueArray(newAllowedValue)); + assertTrue(PropertyDefinitionValidator.validate(oldProperty, newProperty)); + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java new file mode 100644 index 0000000000000..450b299b48b34 --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/validators/StructuredPropertiesValidatorTest.java @@ -0,0 +1,246 @@ +package com.linkedin.metadata.aspect.validators; + +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.Aspect; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; +import com.linkedin.metadata.aspect.plugins.validation.AspectValidationException; +import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PrimitivePropertyValueArray; +import com.linkedin.structured.PropertyValue; +import com.linkedin.structured.PropertyValueArray; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class StructuredPropertiesValidatorTest { + + static class MockAspectRetriever implements AspectRetriever { + StructuredPropertyDefinition _propertyDefinition; + + MockAspectRetriever(StructuredPropertyDefinition defToReturn) { + this._propertyDefinition = defToReturn; + } + + @Nonnull + @Override + public Map> getLatestAspectObjects( + Set urns, Set aspectNames) + throws RemoteInvocationException, URISyntaxException { + return Map.of( + urns.stream().findFirst().get(), + Map.of(aspectNames.stream().findFirst().get(), new Aspect(_propertyDefinition.data()))); + } + + @Nonnull + @Override + public EntityRegistry getEntityRegistry() { + return null; + } + } + + @Test + public void testValidateAspectNumberUpsert() throws URISyntaxException { + StructuredPropertyDefinition numberPropertyDef = + new StructuredPropertyDefinition() + .setValueType(Urn.createFromString("urn:li:type:datahub.number")) + .setAllowedValues( + new PropertyValueArray( + List.of( + new PropertyValue().setValue(PrimitivePropertyValue.create(30.0)), + new PropertyValue().setValue(PrimitivePropertyValue.create(60.0)), + new PropertyValue().setValue(PrimitivePropertyValue.create(90.0))))); + + try { + StructuredPropertyValueAssignment assignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0))); + StructuredProperties numberPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(assignment)); + + boolean isValid = + StructuredPropertiesValidator.validate( + numberPayload, new MockAspectRetriever(numberPropertyDef)); + Assert.assertTrue(isValid); + } catch (AspectValidationException e) { + throw new RuntimeException(e); + } + + try { + StructuredPropertyValueAssignment assignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(0.0))); + StructuredProperties numberPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(assignment)); + + StructuredPropertiesValidator.validate( + numberPayload, new MockAspectRetriever(numberPropertyDef)); + Assert.fail("Should have raised exception for disallowed value 0.0"); + } catch (AspectValidationException e) { + Assert.assertTrue(e.getMessage().contains("{double=0.0} should be one of [{")); + } + + // Assign string value to number property + StructuredPropertyValueAssignment stringAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create("hello"))); + StructuredProperties stringPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(stringAssignment)); + try { + StructuredPropertiesValidator.validate( + stringPayload, new MockAspectRetriever(numberPropertyDef)); + Assert.fail("Should have raised exception for mis-matched types"); + } catch (AspectValidationException e) { + Assert.assertTrue(e.getMessage().contains("should be a number")); + } + } + + @Test + public void testValidateAspectDateUpsert() throws URISyntaxException { + // Assign string value + StructuredPropertyValueAssignment stringAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create("hello"))); + StructuredProperties stringPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(stringAssignment)); + + // Assign invalid date + StructuredPropertyDefinition datePropertyDef = + new StructuredPropertyDefinition() + .setValueType(Urn.createFromString("urn:li:type:datahub.date")); + try { + StructuredPropertiesValidator.validate( + stringPayload, new MockAspectRetriever(datePropertyDef)); + Assert.fail("Should have raised exception for mis-matched types"); + } catch (AspectValidationException e) { + Assert.assertTrue(e.getMessage().contains("should be a date with format")); + } + + // Assign valid date + StructuredPropertyValueAssignment dateAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues( + new PrimitivePropertyValueArray(PrimitivePropertyValue.create("2023-10-24"))); + StructuredProperties datePayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(dateAssignment)); + try { + boolean isValid = + StructuredPropertiesValidator.validate( + datePayload, new MockAspectRetriever(datePropertyDef)); + Assert.assertTrue(isValid); + } catch (AspectValidationException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testValidateAspectStringUpsert() throws URISyntaxException { + // Assign string value + StructuredPropertyValueAssignment stringAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create("hello"))); + StructuredProperties stringPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(stringAssignment)); + + // Assign date + StructuredPropertyValueAssignment dateAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues( + new PrimitivePropertyValueArray(PrimitivePropertyValue.create("2023-10-24"))); + StructuredProperties datePayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(dateAssignment)); + + // Assign number + StructuredPropertyValueAssignment assignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues(new PrimitivePropertyValueArray(PrimitivePropertyValue.create(30.0))); + StructuredProperties numberPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(assignment)); + + StructuredPropertyDefinition stringPropertyDef = + new StructuredPropertyDefinition() + .setValueType(Urn.createFromString("urn:li:type:datahub.string")) + .setAllowedValues( + new PropertyValueArray( + List.of( + new PropertyValue().setValue(PrimitivePropertyValue.create("hello")), + new PropertyValue() + .setValue(PrimitivePropertyValue.create("2023-10-24"))))); + + // Valid strings (both the date value and "hello" are valid) + try { + boolean isValid = + StructuredPropertiesValidator.validate( + stringPayload, new MockAspectRetriever(stringPropertyDef)); + Assert.assertTrue(isValid); + isValid = + StructuredPropertiesValidator.validate( + datePayload, new MockAspectRetriever(stringPropertyDef)); + Assert.assertTrue(isValid); + } catch (AspectValidationException e) { + throw new RuntimeException(e); + } + + // Invalid: assign a number to the string property + try { + StructuredPropertiesValidator.validate( + numberPayload, new MockAspectRetriever(stringPropertyDef)); + Assert.fail("Should have raised exception for mis-matched types"); + } catch (AspectValidationException e) { + Assert.assertTrue(e.getMessage().contains("should be a string")); + } + + // Invalid allowedValue + try { + assignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn( + Urn.createFromString("urn:li:structuredProperty:io.acryl.privacy.retentionTime")) + .setValues( + new PrimitivePropertyValueArray(PrimitivePropertyValue.create("not hello"))); + stringPayload = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(assignment)); + + StructuredPropertiesValidator.validate( + stringPayload, new MockAspectRetriever(stringPropertyDef)); + Assert.fail("Should have raised exception for disallowed value `not hello`"); + } catch (AspectValidationException e) { + Assert.assertTrue(e.getMessage().contains("{string=not hello} should be one of [{")); + } + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java index 2cb48c1b20da9..d9cf8fd2603a8 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/EntitySpecBuilderTest.java @@ -198,7 +198,7 @@ private void validateTestEntityInfo(final AspectSpec testEntityInfo) { .getSearchableAnnotation() .getFieldName()); assertEquals( - SearchableAnnotation.FieldType.KEYWORD, + SearchableAnnotation.FieldType.TEXT, testEntityInfo .getSearchableFieldSpecMap() .get(new PathSpec("customProperties").toString()) diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java index b3eb2af72708c..1a64359008dd8 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java @@ -6,6 +6,7 @@ import com.linkedin.data.schema.ArrayDataSchema; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.DataSchemaFactory; import com.linkedin.metadata.models.DefaultEntitySpec; @@ -17,7 +18,6 @@ import com.linkedin.metadata.models.annotation.EventAnnotation; import com.linkedin.metadata.models.registry.config.EntityRegistryLoadResult; import com.linkedin.metadata.models.registry.config.LoadStatus; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.util.Pair; import java.io.FileNotFoundException; import java.util.ArrayList; diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle index dc9887406b8b4..9505ec57aa858 100644 --- a/ingestion-scheduler/build.gradle +++ b/ingestion-scheduler/build.gradle @@ -1,4 +1,6 @@ -apply plugin: 'java' +plugins { + id 'java' +} dependencies { implementation project(path: ':metadata-models') @@ -7,6 +9,7 @@ dependencies { implementation project(':metadata-service:configuration') implementation externalDependency.slf4jApi + implementation externalDependency.springContext compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok diff --git a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java index f9d22b142cbb9..8174afc20765f 100644 --- a/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java +++ b/ingestion-scheduler/src/test/java/com/datahub/metadata/ingestion/IngestionSchedulerTest.java @@ -11,11 +11,11 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; import com.linkedin.ingestion.DataHubIngestionSourceConfig; import com.linkedin.ingestion.DataHubIngestionSourceInfo; import com.linkedin.ingestion.DataHubIngestionSourceSchedule; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.query.ListResult; import java.util.Collections; @@ -88,7 +88,7 @@ public void setupTest() throws Exception { .thenReturn(Constants.INGESTION_SOURCE_ENTITY_NAME); Mockito.when(entityResponse2.getAspects()).thenReturn(map2); - JavaEntityClient mockClient = Mockito.mock(JavaEntityClient.class); + EntityClient mockClient = Mockito.mock(EntityClient.class); // Set up mocks for ingestion source batch fetching Mockito.when( diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 3d9b533dc8f72..39a17612aa4b3 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -13,6 +13,9 @@ public class Constants { public static final String UNKNOWN_ACTOR = "urn:li:corpuser:UNKNOWN"; // Unknown principal. public static final Long ASPECT_LATEST_VERSION = 0L; public static final String UNKNOWN_DATA_PLATFORM = "urn:li:dataPlatform:unknown"; + public static final String ENTITY_TYPE_URN_PREFIX = "urn:li:entityType:"; + public static final String DATA_TYPE_URN_PREFIX = "urn:li:dataType:"; + public static final String STRUCTURED_PROPERTY_MAPPING_FIELD = "structuredProperties"; // !!!!!!! IMPORTANT !!!!!!! // This effectively sets the max aspect size to 16 MB. Used in deserialization of messages. @@ -73,6 +76,10 @@ public class Constants { public static final String QUERY_ENTITY_NAME = "query"; public static final String DATA_PRODUCT_ENTITY_NAME = "dataProduct"; public static final String OWNERSHIP_TYPE_ENTITY_NAME = "ownershipType"; + public static final String STRUCTURED_PROPERTY_ENTITY_NAME = "structuredProperty"; + public static final String DATA_TYPE_ENTITY_NAME = "dataType"; + public static final String ENTITY_TYPE_ENTITY_NAME = "entityType"; + public static final String FORM_ENTITY_NAME = "form"; /** Aspects */ // Common @@ -125,6 +132,8 @@ public class Constants { public static final String VIEW_PROPERTIES_ASPECT_NAME = "viewProperties"; public static final String DATASET_PROFILE_ASPECT_NAME = "datasetProfile"; + public static final String STRUCTURED_PROPERTIES_ASPECT_NAME = "structuredProperties"; + public static final String FORMS_ASPECT_NAME = "forms"; // Aspect support public static final String FINE_GRAINED_LINEAGE_DATASET_TYPE = "DATASET"; public static final String FINE_GRAINED_LINEAGE_FIELD_SET_TYPE = "FIELD_SET"; @@ -306,6 +315,20 @@ public class Constants { public static final String OWNERSHIP_TYPE_KEY_ASPECT_NAME = "ownershipTypeKey"; public static final String OWNERSHIP_TYPE_INFO_ASPECT_NAME = "ownershipTypeInfo"; + // Structured Property + public static final String STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME = "propertyDefinition"; + + // Form + public static final String FORM_INFO_ASPECT_NAME = "formInfo"; + public static final String FORM_KEY_ASPECT_NAME = "formKey"; + public static final String DYNAMIC_FORM_ASSIGNMENT_ASPECT_NAME = "dynamicFormAssignment"; + + // Data Type + public static final String DATA_TYPE_INFO_ASPECT_NAME = "dataTypeInfo"; + + // Entity Type + public static final String ENTITY_TYPE_INFO_ASPECT_NAME = "entityTypeInfo"; + // Settings public static final String GLOBAL_SETTINGS_ENTITY_NAME = "globalSettings"; public static final String GLOBAL_SETTINGS_INFO_ASPECT_NAME = "globalSettingsInfo"; diff --git a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java index 6b08cdb00e9ab..928876ce71cd5 100644 --- a/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java +++ b/metadata-auth/auth-api/src/main/java/com/datahub/authorization/EntityFieldType.java @@ -28,5 +28,7 @@ public enum EntityFieldType { /** Groups of which the entity (only applies to corpUser) is a member */ GROUP_MEMBERSHIP, /** Data platform instance of resource */ - DATA_PLATFORM_INSTANCE + DATA_PLATFORM_INSTANCE, + /** Tags of the entity */ + TAG, } diff --git a/metadata-events/mxe-avro/build.gradle b/metadata-events/mxe-avro/build.gradle index 3aebc6bb1004d..58e82aff464d5 100644 --- a/metadata-events/mxe-avro/build.gradle +++ b/metadata-events/mxe-avro/build.gradle @@ -1,10 +1,12 @@ +plugins { + id 'java-library' + id 'io.acryl.gradle.plugin.avro' +} + configurations { avsc } -apply plugin: 'io.acryl.gradle.plugin.avro' -apply plugin: 'java-library' - dependencies { api externalDependency.avro implementation(externalDependency.avroCompiler) { diff --git a/metadata-events/mxe-registration/build.gradle b/metadata-events/mxe-registration/build.gradle index 2842dd935c7ee..d4b4d446996fa 100644 --- a/metadata-events/mxe-registration/build.gradle +++ b/metadata-events/mxe-registration/build.gradle @@ -1,4 +1,7 @@ -apply plugin: 'java' +plugins { + id 'java' + id 'pegasus' +} configurations { avroOriginal diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle index 8dc8b71bd1cd8..ab0ea8b649e9d 100644 --- a/metadata-events/mxe-schemas/build.gradle +++ b/metadata-events/mxe-schemas/build.gradle @@ -1,5 +1,7 @@ -apply plugin: 'java-library' -apply plugin: 'pegasus' +plugins { + id 'java-library' + id 'pegasus' +} dependencies { dataModel project(path: ':li-utils', configuration: 'dataTemplate') diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle index 98bfb9127b209..860ced6af2581 100644 --- a/metadata-events/mxe-utils-avro/build.gradle +++ b/metadata-events/mxe-utils-avro/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'pegasus' } dependencies { diff --git a/metadata-ingestion-modules/airflow-plugin/scripts/release.sh b/metadata-ingestion-modules/airflow-plugin/scripts/release.sh index 87157479f37d6..5667e761ea558 100755 --- a/metadata-ingestion-modules/airflow-plugin/scripts/release.sh +++ b/metadata-ingestion-modules/airflow-plugin/scripts/release.sh @@ -13,7 +13,7 @@ MODULE=datahub_airflow_plugin python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"' if [[ ${RELEASE_VERSION:-} ]]; then # Replace version with RELEASE_VERSION env variable - sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/${MODULE}/__init__.py + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py else vim src/${MODULE}/__init__.py fi diff --git a/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml b/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml index cb7fc97a39b9f..632828f42014b 100644 --- a/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml +++ b/metadata-ingestion/docs/sources/datahub/datahub_recipe.yml @@ -1,13 +1,13 @@ pipeline_name: datahub_source_1 datahub_api: - server: "http://localhost:8080" # Migrate data from DataHub instance on localhost:8080 + server: "http://localhost:8080" # Migrate data from DataHub instance on localhost:8080 token: "" source: type: datahub config: include_all_versions: false database_connection: - scheme: "mysql+pymysql" # or "postgresql+psycopg2" for Postgres + scheme: "mysql+pymysql" # or "postgresql+psycopg2" for Postgres host_port: ":" username: "" password: "" @@ -19,12 +19,12 @@ source: enabled: true ignore_old_state: false extractor_config: - set_system_metadata: false # Replicate system metadata + set_system_metadata: false # Replicate system metadata # Here, we write to a DataHub instance # You can also use a different sink, e.g. to write the data to a file instead sink: - type: datahub + type: datahub-rest config: server: "" token: "" diff --git a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml index de6ba8731c878..327246863b0ab 100644 --- a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml +++ b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml @@ -10,6 +10,8 @@ nodes: knowledge_links: - label: Wiki link for classification url: "https://en.wikipedia.org/wiki/Classification" + custom_properties: + is_confidential: true terms: - name: Sensitive description: Sensitive Data diff --git a/metadata-ingestion/examples/forms/forms.yaml b/metadata-ingestion/examples/forms/forms.yaml new file mode 100644 index 0000000000000..80bb7cee08ec3 --- /dev/null +++ b/metadata-ingestion/examples/forms/forms.yaml @@ -0,0 +1,54 @@ +- id: 123456 + # urn: "urn:li:form:123456" # optional if id is provided + type: VERIFICATION + name: "Metadata Initiative 2023" + description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out" + prompts: + - id: "123" + title: "Retention Time" + description: "Apply Retention Time structured property to form" + type: STRUCTURED_PROPERTY + structured_property_id: io.acryl.privacy.retentionTime + required: True # optional, will default to True + - id: "92847" + title: "Replication SLA" + description: "Apply Replication SLA structured property to form" + type: STRUCTURED_PROPERTY + structured_property_urn: urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA + required: True + - id: "76543" + title: "Replication SLA" + description: "Apply Replication SLA structured property to form" + type: FIELDS_STRUCTURED_PROPERTY + structured_property_urn: urn:li:structuredProperty:io.acryl.dataManagement.replicationSLA + required: False + entities: # Either pass a list of urns or a group of filters + # urns: + # - urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD) + # - urn:li:dataset:(urn:li:dataPlatform:snowflake,user.clicks,PROD) + filters: + types: + - dataset + platforms: + - snowflake + - dbt + domains: + - urn:li:domain:b41fbb69-1549-4f30-a463-d75d1bed31c1 + containers: + - urn:li:container:21d4204e13d5b984c58acad468ecdbdd +- urn: "urn:li:form:917364" + # id: 917364 # optional if urn is provided + type: VERIFICATION + name: "Governance Initiative" + prompts: + - id: "123" + title: "Retention Time" + description: "Apply Retention Time structured property to form" + type: STRUCTURED_PROPERTY + structured_property_id: io.acryl.privacy.retentionTime + required: False + - id: "certifier" + title: "Certifier" + type: STRUCTURED_PROPERTY + structured_property_id: io.acryl.dataManagement.certifier + required: True diff --git a/metadata-ingestion/examples/mce_files/test_structured_properties.json b/metadata-ingestion/examples/mce_files/test_structured_properties.json new file mode 100644 index 0000000000000..7771883152d38 --- /dev/null +++ b/metadata-ingestion/examples/mce_files/test_structured_properties.json @@ -0,0 +1,218 @@ +[ + { + "auditHeader": null, + "entityType": "entityType", + "entityUrn": "urn:li:entityType:datahub.dataset", + "changeType": "UPSERT", + "aspectName": "entityTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.dataset\", \"displayName\": \"Dataset\", \"description\": \"An entity type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "entityType", + "entityUrn": "urn:li:entityType:datahub.corpuser", + "changeType": "UPSERT", + "aspectName": "entityTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.corpuser\", \"displayName\": \"User\", \"description\": \"An entity type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "entityType", + "entityUrn": "urn:li:entityType:datahub.corpGroup", + "changeType": "UPSERT", + "aspectName": "entityTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.corpGroup\", \"displayName\": \"Group\", \"description\": \"An entity type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataType", + "entityUrn": "urn:li:dataType:datahub.string", + "changeType": "UPSERT", + "aspectName": "dataTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.string\", \"displayName\": \"String\", \"description\": \"A string type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataType", + "entityUrn": "urn:li:dataType:datahub.float", + "changeType": "UPSERT", + "aspectName": "dataTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.float\", \"displayName\": \"Number\", \"description\": \"A number type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataType", + "entityUrn": "urn:li:dataType:datahub.urn", + "changeType": "UPSERT", + "aspectName": "dataTypeInfo", + "aspect": { + "value": "{\"qualifiedName\": \"datahub.urn\", \"displayName\": \"Urn\", \"description\": \"A entity type.\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property1", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property1\", \"displayName\": \"String Property\", \"valueType\": \"urn:li:dataType:datahub.string\", \"cardinality\": \"SINGLE\", \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property2", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property2\", \"displayName\": \"String Property With Allowed Values\", \"valueType\": \"urn:li:dataType:datahub.string\", \"cardinality\": \"MULTIPLE\", \"allowedValues\": [ { \"value\": { \"string\": \"Test 1\" } }, { \"value\": { \"string\": \"Test 2\" } } ], \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property3", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property3\", \"displayName\": \"Numeric Property\", \"valueType\": \"urn:li:dataType:datahub.float\", \"cardinality\": \"MULTIPLE\", \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property4", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property4\", \"displayName\": \"Numeric Property with Allowed Values\", \"valueType\": \"urn:li:dataType:datahub.float\", \"cardinality\": \"MULTIPLE\", \"allowedValues\": [ { \"value\": { \"double\": 0.12 } }, { \"value\": { \"double\": 1 } } ], \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property5", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property5\", \"displayName\": \"Urn property no type qualifier\", \"valueType\": \"urn:li:dataType:datahub.urn\", \"cardinality\": \"MULTIPLE\", \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property6", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property6\", \"displayName\": \"Urn property with 1 type qualifier (user)\", \"valueType\": \"urn:li:dataType:datahub.urn\", \"typeQualifier\": { \"allowedTypes\": [\"urn:li:entityType:datahub.corpuser\"] }, \"cardinality\": \"MULTIPLE\", \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "structuredProperty", + "entityUrn": "urn:li:structuredProperty:test.Property7", + "changeType": "UPSERT", + "aspectName": "propertyDefinition", + "aspect": { + "value": "{\"qualifiedName\": \"test.Property7\", \"displayName\": \"Urn property with 2 type qualifier (user)\", \"valueType\": \"urn:li:dataType:datahub.urn\", \"typeQualifier\": { \"allowedTypes\": [\"urn:li:entityType:datahub.corpuser\", \"urn:li:entityType:datahub.corpGroup\"] }, \"cardinality\": \"MULTIPLE\", \"entityTypes\": [\"urn:li:entityType:datahub.dataset\"], \"description\": \"My description\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "form", + "entityUrn": "urn:li:form:my-test-form-verification-default-3", + "changeType": "UPSERT", + "aspectName": "formInfo", + "aspect": { + "value": "{\"name\": \"My test form\", \"description\": \"My test description\", \"type\": \"VERIFICATION\", \"prompts\": [{\"id\": \"prompt-1\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" }}, {\"id\": \"prompt-2\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" }}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "form", + "entityUrn": "urn:li:form:my-test-no-verification-3", + "changeType": "UPSERT", + "aspectName": "formInfo", + "aspect": { + "value": "{\"name\": \"My test form without verification\", \"description\": \"My test description\", \"prompts\": [{\"id\": \"prompt-1\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" }}, {\"id\": \"prompt-2\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" }}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "form", + "entityUrn": "urn:li:form:my-test-no-verification-custom-5", + "changeType": "UPSERT", + "aspectName": "formInfo", + "aspect": { + "value": "{\"name\": \"My test form with custom verification\", \"description\": \"My test description\", \"type\": \"VERIFICATION\", \"verification\": { \"type\": \"urn:li:verificationType:my-test\"}, \"prompts\": [{\"id\": \"prompt-1\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"required\": true, \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" } }, {\"id\": \"prompt-2\", \"title\": \"Select your thing\", \"description\": \"Which will you select?\", \"type\": \"STRUCTURED_PROPERTY\", \"required\": true, \"structuredPropertyParams\": { \"urn\": \"urn:li:structuredProperty:test.Property7\" }}]}", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "form", + "entityUrn": "urn:li:form:my-test-no-verification-custom-5", + "changeType": "UPSERT", + "aspectName": "dynamicFormAssignment", + "aspect": { + "value": "{\"filter\": { \"or\": [ { \"and\": [ { \"field\": \"platform\", \"condition\": \"EQUAL\", \"values\": [\"urn:li:dataPlatform:snowflake\"], \"value\": \"\" } ] } ] } }", + "contentType": "application/json" + }, + "systemMetadata": null + }, + { + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "forms", + "aspect": { + "value": "{ \"incompleteForms\":[\n {\n \"incompletePrompts\":[\n \n ],\n \"urn\":\"urn:li:form:my-test-no-verification-custom-4\",\n \"completedPrompts\":[\n {\n \"lastModified\":{\n \"actor\":\"urn:li:corpuser:__datahub_system\",\n \"time\":1697585983115\n },\n \"id\":\"prompt-2\"\n },\n {\n \"id\":\"prompt-1\",\n \"lastModified\":{\n \"actor\":\"urn:li:corpuser:__datahub_system\",\n \"time\":1697585983252\n }\n }\n ]\n },\n {\n \"incompletePrompts\":[\n \n ],\n \"urn\":\"urn:li:form:my-test-no-verification-custom-5\",\n \"completedPrompts\":[\n {\n \"lastModified\":{\n \"actor\":\"urn:li:corpuser:__datahub_system\",\n \"time\":1697645753521\n },\n \"id\":\"prompt-2\"\n },\n {\n \"id\":\"prompt-1\",\n \"lastModified\":{\n \"actor\":\"urn:li:corpuser:__datahub_system\",\n \"time\":1697645754180\n }\n }\n ]\n }\n ],\n \"completedForms\":[\n \n ]}", + "contentType": "application/json" + }, + "systemMetadata": null + } +] \ No newline at end of file diff --git a/metadata-ingestion/examples/structured_properties/README.md b/metadata-ingestion/examples/structured_properties/README.md new file mode 100644 index 0000000000000..0429310be7424 --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/README.md @@ -0,0 +1,51 @@ +# Extended Properties + +## Expected Capabilities + +### structured_properties command + +```yaml +- id: io.acryl.privacy.retentionTime + # urn: urn:li:structuredProperty:<> + # fullyQualifiedName: io.acryl.privacy.retentionTime + type: STRING + cardinality: MULTIPLE + entityTypes: + - dataset # or urn:li:logicalEntity:metamodel.datahub.dataset + - dataflow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowedValues: + - value: 30 days + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 3 months + description: Use this for datasets that drive monthly reporting but contain pii + - value: 2 yrs + description: Use this for non-sensitive data that can be retained for longer +- id: io.acryl.dataManagement.replicationSLA + type: NUMBER + description: "SLA for how long data can be delayed before replicating to the destination cluster" + entityTypes: + - dataset +- id: io.acryl.dataManagement.deprecationDate + type: DATE + entityTypes: + - dataset + - dataFlow + - dataJob +``` + +``` +datahub properties create -f structured_properties.yaml +``` + +``` +datahub properties create --name io.acryl.privacy.retentionTime --type STRING --cardinality MULTIPLE --entity_type DATASET --entity_type DATAFLOW +``` + +### dataset command + +``` +datahub dataset create -f dataset.yaml +``` + +See example in `dataproduct`. diff --git a/metadata-ingestion/examples/structured_properties/click_event.avsc b/metadata-ingestion/examples/structured_properties/click_event.avsc new file mode 100644 index 0000000000000..b277674f8b62f --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/click_event.avsc @@ -0,0 +1,14 @@ +{ + "namespace": "org.acryl", + "type": "record", + "name": "ClickEvent", + "fields": [ + { "name": "ip", "type": "string" }, + { "name": "url", "type": "string" }, + { "name": "time", "type": "long" }, + { "name": "referer", "type": ["string", "null"] }, + { "name": "user_agent", "type": ["string", "null"] }, + { "name": "user_id", "type": ["string", "null"] }, + { "name": "session_id", "type": ["string", "null"] } + ] +} diff --git a/metadata-ingestion/examples/structured_properties/dataset.yaml b/metadata-ingestion/examples/structured_properties/dataset.yaml new file mode 100644 index 0000000000000..557bf0167a51b --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/dataset.yaml @@ -0,0 +1,45 @@ +## This file is used to define a dataset and provide metadata for it +- id: user.clicks + platform: hive + # - urn: urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD) # use urn instead of id and platform + subtype: Table + schema: + file: examples/structured_properties/click_event.avsc + fields: + - id: ip + - urn: urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD),ip) + structured_properties: # structured properties for schema fields/columns go here + io.acryl.dataManagement.deprecationDate: "2023-01-01" + io.acryl.dataManagement.certifier: urn:li:corpuser:john.doe@example.com + io.acryl.dataManagement.replicationSLA: 90 + structured_properties: # dataset level structured properties go here + io.acryl.privacy.retentionTime: 365 + projectNames: + - Tracking + - DataHub +- id: ClickEvent + platform: events + subtype: Topic + description: | + This is a sample event that is generated when a user clicks on a link. + Do not use this event for any purpose other than testing. + properties: + project_name: Tracking + namespace: org.acryl.tracking + version: 1.0.0 + retention: 30 + structured_properties: + io.acryl.dataManagement.certifier: urn:li:corpuser:john.doe@example.com + schema: + file: examples/structured_properties/click_event.avsc + downstreams: + - urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD) +- id: user.clicks + platform: snowflake + schema: + fields: + - id: user_id + structured_properties: + io.acryl.dataManagement.deprecationDate: "2023-01-01" + structured_properties: + io.acryl.dataManagement.replicationSLA: 90 diff --git a/metadata-ingestion/examples/structured_properties/structured_properties.yaml b/metadata-ingestion/examples/structured_properties/structured_properties.yaml new file mode 100644 index 0000000000000..5c7ce47ba3b8a --- /dev/null +++ b/metadata-ingestion/examples/structured_properties/structured_properties.yaml @@ -0,0 +1,68 @@ +- id: io.acryl.privacy.retentionTime + # - urn: urn:li:structuredProperty:io.acryl.privacy.retentionTime # optional if id is provided + qualified_name: io.acryl.privacy.retentionTime # required if urn is provided + type: number + cardinality: MULTIPLE + display_name: Retention Time + entity_types: + - dataset # or urn:li:entityType:datahub.dataset + - dataFlow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowed_values: + - value: 30 + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 90 + description: Use this for datasets that drive monthly reporting but contain pii + - value: 365 + description: Use this for non-sensitive data that can be retained for longer +- id: io.acryl.dataManagement.replicationSLA + type: number + display_name: Replication SLA + description: "SLA for how long data can be delayed before replicating to the destination cluster" + entity_types: + - dataset +- id: io.acryl.dataManagement.deprecationDate + type: date + display_name: Deprecation Date + entity_types: + - dataset + - dataFlow + - dataJob +- id: io.acryl.dataManagement.steward + type: urn + type_qualifier: + allowed_types: # only user and group urns are allowed + - corpuser + - corpGroup + display_name: Steward + entity_types: + - dataset + - dataFlow + - dataJob +- id: io.acryl.dataManagement.certifier + type: urn + display_name: Person Certifying the asset + entity_types: + - dataset + - schemaField +- id: io.acryl.dataManagement.team + type: string + display_name: Management team + entity_types: + - dataset +- id: projectNames + type: string + cardinality: MULTIPLE + display_name: Project names + entity_types: + - dataset + allowed_values: + - value: Tracking + description: test value 1 for project + - value: DataHub + description: test value 2 for project +- id: namespace + type: string + display_name: Namespace + entity_types: + - dataset diff --git a/metadata-ingestion/scripts/docgen.sh b/metadata-ingestion/scripts/docgen.sh index affb87f2e70a9..09fa2be912f61 100755 --- a/metadata-ingestion/scripts/docgen.sh +++ b/metadata-ingestion/scripts/docgen.sh @@ -7,4 +7,4 @@ DOCS_OUT_DIR=$DATAHUB_ROOT/docs/generated/ingestion EXTRA_DOCS_DIR=$DATAHUB_ROOT/metadata-ingestion/docs/sources rm -r $DOCS_OUT_DIR || true -python scripts/docgen.py --out-dir ${DOCS_OUT_DIR} --extra-docs ${EXTRA_DOCS_DIR} $@ +SPARK_VERSION=3.3 python scripts/docgen.py --out-dir ${DOCS_OUT_DIR} --extra-docs ${EXTRA_DOCS_DIR} $@ diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py index 81b26145e620c..610c6d3107916 100644 --- a/metadata-ingestion/scripts/modeldocgen.py +++ b/metadata-ingestion/scripts/modeldocgen.py @@ -493,10 +493,32 @@ def strip_types(field_path: str) -> str: ], ) +@dataclass +class EntityAspectName: + entityName: str + aspectName: str + + +@dataclass +class AspectPluginConfig: + className: str + enabled: bool + supportedOperations: List[str] + supportedEntityAspectNames: List[EntityAspectName] + + +@dataclass +class PluginConfiguration: + aspectPayloadValidators: Optional[List[AspectPluginConfig]] = None + mutationHooks: Optional[List[AspectPluginConfig]] = None + mclSideEffects: Optional[List[AspectPluginConfig]] = None + mcpSideEffects: Optional[List[AspectPluginConfig]] = None + class EntityRegistry(ConfigModel): entities: List[EntityDefinition] events: Optional[List[EventDefinition]] + plugins: Optional[PluginConfiguration] = None def load_registry_file(registry_file: str) -> Dict[str, EntityDefinition]: diff --git a/metadata-ingestion/scripts/release.sh b/metadata-ingestion/scripts/release.sh index eacaf1d920a8d..955eb562089f7 100755 --- a/metadata-ingestion/scripts/release.sh +++ b/metadata-ingestion/scripts/release.sh @@ -11,7 +11,7 @@ fi python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"' if [[ ${RELEASE_VERSION:-} ]]; then # Replace version with RELEASE_VERSION env variable - sed -i.bak "s/__version__ = \"1!0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/datahub/__init__.py + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py else vim src/datahub/__init__.py fi diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 34e8167a997f6..1fb570d76120e 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -149,7 +149,7 @@ # This version of lkml contains a fix for parsing lists in # LookML files with spaces between an item and the following comma. # See https://github.com/joshtemple/lkml/issues/73. - "lkml>=1.3.0b5", + "lkml>=1.3.4", "sql-metadata==2.2.2", *sqllineage_lib, "GitPython>2", diff --git a/metadata-ingestion/src/datahub/api/entities/dataset/__init__.py b/metadata-ingestion/src/datahub/api/entities/dataset/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py b/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py new file mode 100644 index 0000000000000..a1498a6ca961e --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/dataset/dataset.py @@ -0,0 +1,466 @@ +import json +import logging +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple, Union + +from pydantic import BaseModel, Field, validator +from ruamel.yaml import YAML + +from datahub.api.entities.structuredproperties.structuredproperties import ( + AllowedTypes, + StructuredProperties, +) +from datahub.configuration.common import ConfigModel +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataset_urn, + make_schema_field_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields +from datahub.ingestion.graph.client import DataHubGraph, get_default_graph +from datahub.metadata.schema_classes import ( + DatasetPropertiesClass, + MetadataChangeProposalClass, + OtherSchemaClass, + SchemaFieldClass, + SchemaMetadataClass, + StructuredPropertiesClass, + StructuredPropertyValueAssignmentClass, + SubTypesClass, + UpstreamClass, +) +from datahub.specific.dataset import DatasetPatchBuilder +from datahub.utilities.urns.dataset_urn import DatasetUrn +from datahub.utilities.urns.urn import Urn + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class SchemaFieldSpecification(BaseModel): + id: Optional[str] + urn: Optional[str] + structured_properties: Optional[ + Dict[str, Union[str, float, List[Union[str, float]]]] + ] = None + type: Optional[str] + nativeDataType: Optional[str] = None + jsonPath: Union[None, str] = None + nullable: Optional[bool] = None + description: Union[None, str] = None + label: Optional[str] = None + created: Optional[dict] = None + lastModified: Optional[dict] = None + recursive: Optional[bool] = None + globalTags: Optional[dict] = None + glossaryTerms: Optional[dict] = None + isPartOfKey: Optional[bool] = None + isPartitioningKey: Optional[bool] = None + jsonProps: Optional[dict] = None + + def with_structured_properties( + self, + structured_properties: Optional[Dict[str, List[Union[str, float]]]], + ) -> "SchemaFieldSpecification": + self.structured_properties = ( + {k: v for k, v in structured_properties.items()} + if structured_properties + else None + ) + return self + + @classmethod + def from_schema_field( + cls, schema_field: SchemaFieldClass, parent_urn: str + ) -> "SchemaFieldSpecification": + return SchemaFieldSpecification( + id=Dataset._simplify_field_path(schema_field.fieldPath), + urn=make_schema_field_urn( + parent_urn, Dataset._simplify_field_path(schema_field.fieldPath) + ), + type=str(schema_field.type), + nativeDataType=schema_field.nativeDataType, + nullable=schema_field.nullable, + description=schema_field.description, + label=schema_field.label, + created=schema_field.created.__dict__ if schema_field.created else None, + lastModified=schema_field.lastModified.__dict__ + if schema_field.lastModified + else None, + recursive=schema_field.recursive, + globalTags=schema_field.globalTags.__dict__ + if schema_field.globalTags + else None, + glossaryTerms=schema_field.glossaryTerms.__dict__ + if schema_field.glossaryTerms + else None, + isPartitioningKey=schema_field.isPartitioningKey, + jsonProps=json.loads(schema_field.jsonProps) + if schema_field.jsonProps + else None, + ) + + @validator("urn", pre=True, always=True) + def either_id_or_urn_must_be_filled_out(cls, v, values): + if not v and not values.get("id"): + raise ValueError("Either id or urn must be present") + return v + + +class SchemaSpecification(BaseModel): + file: Optional[str] + fields: Optional[List[SchemaFieldSpecification]] + + @validator("file") + def file_must_be_avsc(cls, v): + if v and not v.endswith(".avsc"): + raise ValueError("file must be a .avsc file") + return v + + +class StructuredPropertyValue(ConfigModel): + value: Union[str, float, List[str], List[float]] + created: Optional[str] + lastModified: Optional[str] + + +class Dataset(BaseModel): + id: Optional[str] + platform: Optional[str] + env: str = "PROD" + urn: Optional[str] + description: Optional[str] + name: Optional[str] + schema_metadata: Optional[SchemaSpecification] = Field(alias="schema") + downstreams: Optional[List[str]] + properties: Optional[Dict[str, str]] + subtype: Optional[str] + subtypes: Optional[List[str]] + structured_properties: Optional[ + Dict[str, Union[str, float, List[Union[str, float]]]] + ] = None + + @property + def platform_urn(self) -> str: + if self.platform: + return make_data_platform_urn(self.platform) + else: + assert self.urn is not None # validator should have filled this in + dataset_urn = DatasetUrn.from_string(self.urn) + return str(dataset_urn.get_data_platform_urn()) + + @validator("urn", pre=True, always=True) + def urn_must_be_present(cls, v, values): + if not v: + assert "id" in values, "id must be present if urn is not" + assert "platform" in values, "platform must be present if urn is not" + assert "env" in values, "env must be present if urn is not" + return make_dataset_urn(values["platform"], values["id"], values["env"]) + return v + + @validator("name", pre=True, always=True) + def name_filled_with_id_if_not_present(cls, v, values): + if not v: + assert "id" in values, "id must be present if name is not" + return values["id"] + return v + + @validator("platform") + def platform_must_not_be_urn(cls, v): + if v.startswith("urn:li:dataPlatform:"): + return v[len("urn:li:dataPlatform:") :] + return v + + @classmethod + def from_yaml(cls, file: str) -> Iterable["Dataset"]: + with open(file) as fp: + yaml = YAML(typ="rt") # default, if not specfied, is 'rt' (round-trip) + datasets: Union[dict, List[dict]] = yaml.load(fp) + if isinstance(datasets, dict): + datasets = [datasets] + for dataset_raw in datasets: + dataset = Dataset.parse_obj(dataset_raw) + yield dataset + + def generate_mcp( + self, + ) -> Iterable[Union[MetadataChangeProposalClass, MetadataChangeProposalWrapper]]: + mcp = MetadataChangeProposalWrapper( + entityUrn=self.urn, + aspect=DatasetPropertiesClass( + description=self.description, + name=self.name, + customProperties=self.properties, + ), + ) + yield mcp + + if self.schema_metadata: + if self.schema_metadata.file: + with open(self.schema_metadata.file, "r") as schema_fp: + schema_string = schema_fp.read() + schema_metadata = SchemaMetadataClass( + schemaName=self.name or self.id or self.urn or "", + platform=self.platform_urn, + version=0, + hash="", + platformSchema=OtherSchemaClass(rawSchema=schema_string), + fields=avro_schema_to_mce_fields(schema_string), + ) + mcp = MetadataChangeProposalWrapper( + entityUrn=self.urn, aspect=schema_metadata + ) + yield mcp + + if self.schema_metadata.fields: + for field in self.schema_metadata.fields: + field_urn = field.urn or make_schema_field_urn( + self.urn, field.id # type: ignore[arg-type] + ) + assert field_urn.startswith("urn:li:schemaField:") + if field.structured_properties: + # field_properties_flattened = ( + # Dataset.extract_structured_properties( + # field.structured_properties + # ) + # ) + mcp = MetadataChangeProposalWrapper( + entityUrn=field_urn, + aspect=StructuredPropertiesClass( + properties=[ + StructuredPropertyValueAssignmentClass( + propertyUrn=f"urn:li:structuredProperty:{prop_key}", + values=prop_value + if isinstance(prop_value, list) + else [prop_value], + ) + for prop_key, prop_value in field.structured_properties.items() + ] + ), + ) + yield mcp + + if self.subtype or self.subtypes: + mcp = MetadataChangeProposalWrapper( + entityUrn=self.urn, + aspect=SubTypesClass( + typeNames=[ + s + for s in [self.subtype] + (self.subtypes or []) + if s + ] + ), + ) + yield mcp + + if self.structured_properties: + # structured_properties_flattened = ( + # Dataset.extract_structured_properties( + # self.structured_properties + # ) + # ) + mcp = MetadataChangeProposalWrapper( + entityUrn=self.urn, + aspect=StructuredPropertiesClass( + properties=[ + StructuredPropertyValueAssignmentClass( + propertyUrn=f"urn:li:structuredProperty:{prop_key}", + values=prop_value + if isinstance(prop_value, list) + else [prop_value], + ) + for prop_key, prop_value in self.structured_properties.items() + ] + ), + ) + yield mcp + + if self.downstreams: + for downstream in self.downstreams: + patch_builder = DatasetPatchBuilder(downstream) + assert ( + self.urn is not None + ) # validator should have filled this in + patch_builder.add_upstream_lineage( + UpstreamClass( + dataset=self.urn, + type="COPY", + ) + ) + for patch_event in patch_builder.build(): + yield patch_event + + logger.info(f"Created dataset {self.urn}") + + @staticmethod + def extract_structured_properties( + structured_properties: Dict[str, Union[str, float, List[str], List[float]]] + ) -> List[Tuple[str, Union[str, float]]]: + structured_properties_flattened: List[Tuple[str, Union[str, float]]] = [] + for key, value in structured_properties.items(): + validated_structured_property = Dataset.validate_structured_property( + key, value + ) + if validated_structured_property: + structured_properties_flattened.append(validated_structured_property) + structured_properties_flattened = sorted( + structured_properties_flattened, key=lambda x: x[0] + ) + return structured_properties_flattened + + @staticmethod + def validate_structured_property( + sp_name: str, sp_value: Union[str, float, List[str], List[float]] + ) -> Union[Tuple[str, Union[str, float]], None]: + """ + Validate based on: + 1. Structured property exists/has been created + 2. Structured property value is of the expected type + """ + urn = Urn.make_structured_property_urn(sp_name) + with get_default_graph() as graph: + if graph.exists(urn): + validated_structured_property = StructuredProperties.from_datahub( + graph, urn + ) + allowed_type = Urn.get_data_type_from_urn( + validated_structured_property.type + ) + try: + if not isinstance(sp_value, list): + return Dataset.validate_type(sp_name, sp_value, allowed_type) + else: + for v in sp_value: + return Dataset.validate_type(sp_name, v, allowed_type) + except ValueError: + logger.warning( + f"Property: {sp_name}, value: {sp_value} should be a {allowed_type}." + ) + else: + logger.error( + f"Property {sp_name} does not exist and therefore will not be added to dataset. Please create property before trying again." + ) + return None + + @staticmethod + def validate_type( + sp_name: str, sp_value: Union[str, float], allowed_type: str + ) -> Tuple[str, Union[str, float]]: + if allowed_type == AllowedTypes.NUMBER.value: + return (sp_name, float(sp_value)) + else: + return (sp_name, sp_value) + + @staticmethod + def _simplify_field_path(field_path: str) -> str: + if field_path.startswith("[version=2.0]"): + # v2 field path + field_components = [] + current_field = "" + for c in field_path: + if c == "[": + if current_field: + field_components.append(current_field) + current_field = "" + omit_next = True + elif c == "]": + omit_next = False + elif c == ".": + pass + elif not omit_next: + current_field += c + if current_field: + field_components.append(current_field) + return ".".join(field_components) + else: + return field_path + + @staticmethod + def _schema_from_schema_metadata( + graph: DataHubGraph, urn: str + ) -> Optional[SchemaSpecification]: + schema_metadata: Optional[SchemaMetadataClass] = graph.get_aspect( + urn, SchemaMetadataClass + ) + + if schema_metadata: + schema_specification = SchemaSpecification( + fields=[ + SchemaFieldSpecification.from_schema_field( + field, urn + ).with_structured_properties( + { + sp.propertyUrn: sp.values + for sp in structured_props.properties + } + if structured_props + else None + ) + for field, structured_props in [ + ( + field, + graph.get_aspect( + make_schema_field_urn(urn, field.fieldPath), + StructuredPropertiesClass, + ) + or graph.get_aspect( + make_schema_field_urn( + urn, Dataset._simplify_field_path(field.fieldPath) + ), + StructuredPropertiesClass, + ), + ) + for field in schema_metadata.fields + ] + ] + ) + return schema_specification + else: + return None + + @classmethod + def from_datahub(cls, graph: DataHubGraph, urn: str) -> "Dataset": + dataset_properties: Optional[DatasetPropertiesClass] = graph.get_aspect( + urn, DatasetPropertiesClass + ) + subtypes: Optional[SubTypesClass] = graph.get_aspect(urn, SubTypesClass) + structured_properties: Optional[StructuredPropertiesClass] = graph.get_aspect( + urn, StructuredPropertiesClass + ) + if structured_properties: + structured_properties_map: Dict[str, List[Union[str, float]]] = {} + for sp in structured_properties.properties: + if sp.propertyUrn in structured_properties_map: + assert isinstance(structured_properties_map[sp.propertyUrn], list) + structured_properties_map[sp.propertyUrn].extend(sp.values) # type: ignore[arg-type,union-attr] + else: + structured_properties_map[sp.propertyUrn] = sp.values + + return Dataset( # type: ignore[call-arg] + urn=urn, + description=dataset_properties.description + if dataset_properties and dataset_properties.description + else None, + name=dataset_properties.name + if dataset_properties and dataset_properties.name + else None, + schema=Dataset._schema_from_schema_metadata(graph, urn), + properties=dataset_properties.customProperties + if dataset_properties + else None, + subtypes=[subtype for subtype in subtypes.typeNames] if subtypes else None, + structured_properties=structured_properties_map + if structured_properties + else None, + ) + + def to_yaml( + self, + file: Path, + ) -> None: + with open(file, "w") as fp: + yaml = YAML(typ="rt") # default, if not specfied, is 'rt' (round-trip) + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.default_flow_style = False + yaml.dump(self.dict(exclude_none=True, exclude_unset=True), fp) diff --git a/metadata-ingestion/src/datahub/api/entities/forms/__init__.py b/metadata-ingestion/src/datahub/api/entities/forms/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/api/entities/forms/forms.py b/metadata-ingestion/src/datahub/api/entities/forms/forms.py new file mode 100644 index 0000000000000..cc43779bda409 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/forms/forms.py @@ -0,0 +1,353 @@ +import logging +import uuid +from enum import Enum +from pathlib import Path +from typing import List, Optional, Union + +import yaml +from pydantic import validator +from ruamel.yaml import YAML +from typing_extensions import Literal + +from datahub.api.entities.forms.forms_graphql_constants import ( + CREATE_DYNAMIC_FORM_ASSIGNMENT, + FIELD_FILTER_TEMPLATE, + UPLOAD_ENTITIES_FOR_FORMS, +) +from datahub.configuration.common import ConfigModel +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_group_urn, + make_user_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import DataHubGraph, get_default_graph +from datahub.metadata.schema_classes import ( + FormInfoClass, + FormPromptClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + StructuredPropertyParamsClass, +) +from datahub.utilities.urns.urn import Urn + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class PromptType(Enum): + STRUCTURED_PROPERTY = "STRUCTURED_PROPERTY" + FIELDS_STRUCTURED_PROPERTY = "FIELDS_STRUCTURED_PROPERTY" + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class Prompt(ConfigModel): + id: Optional[str] + title: str + description: Optional[str] + type: str + structured_property_id: Optional[str] + structured_property_urn: Optional[str] + required: Optional[bool] + + @validator("structured_property_urn", pre=True, always=True) + def structured_property_urn_must_be_present(cls, v, values): + if not v and values.get("structured_property_id"): + return Urn.make_structured_property_urn(values["structured_property_id"]) + return v + + +class FormType(Enum): + VERIFICATION = "VERIFICATION" + DOCUMENTATION = "COMPLETION" + + @classmethod + def has_value(cls, value): + return value in cls._value2member_map_ + + +class Filters(ConfigModel): + types: Optional[List[str]] + platforms: Optional[List[str]] + domains: Optional[List[str]] + containers: Optional[List[str]] + + +class Entities(ConfigModel): + urns: Optional[List[str]] + filters: Optional[Filters] + + +class Forms(ConfigModel): + id: Optional[str] + urn: Optional[str] + name: str + description: Optional[str] + prompts: List[Prompt] = [] + type: Optional[str] + version: Optional[Literal[1]] + entities: Optional[Entities] + owners: Optional[List[str]] # can be user IDs or urns + group_owners: Optional[List[str]] # can be group IDs or urns + + @validator("urn", pre=True, always=True) + def urn_must_be_present(cls, v, values): + if not v: + assert values.get("id") is not None, "Form id must be present if urn is not" + return f"urn:li:form:{values['id']}" + return v + + @staticmethod + def create(file: str) -> None: + emitter: DataHubGraph + + with get_default_graph() as emitter: + with open(file, "r") as fp: + forms: List[dict] = yaml.safe_load(fp) + for form_raw in forms: + form = Forms.parse_obj(form_raw) + + try: + if not FormType.has_value(form.type): + logger.error( + f"Form type {form.type} does not exist. Please try again with a valid type." + ) + + mcp = MetadataChangeProposalWrapper( + entityUrn=form.urn, + aspect=FormInfoClass( + name=form.name, + description=form.description, + prompts=form.validate_prompts(emitter), + type=form.type, + ), + ) + emitter.emit_mcp(mcp) + + logger.info(f"Created form {form.urn}") + + if form.owners or form.group_owners: + form.add_owners(emitter) + + if form.entities: + if form.entities.urns: + # Associate specific entities with a form + form.upload_entities_for_form(emitter) + + if form.entities.filters: + # Associate groups of entities with a form based on filters + form.create_form_filters(emitter) + + except Exception as e: + logger.error(e) + return + + def validate_prompts(self, emitter: DataHubGraph) -> List[FormPromptClass]: + prompts = [] + if self.prompts: + for prompt in self.prompts: + if not prompt.id: + prompt.id = str(uuid.uuid4()) + logger.warning( + f"Prompt id not provided. Setting prompt id to {prompt.id}" + ) + if prompt.structured_property_urn: + structured_property_urn = prompt.structured_property_urn + if emitter.exists(structured_property_urn): + prompt.structured_property_urn = structured_property_urn + else: + raise Exception( + f"Structured property {structured_property_urn} does not exist. Unable to create form." + ) + elif ( + prompt.type + in ( + PromptType.STRUCTURED_PROPERTY.value, + PromptType.FIELDS_STRUCTURED_PROPERTY.value, + ) + and not prompt.structured_property_urn + ): + raise Exception( + f"Prompt type is {prompt.type} but no structured properties exist. Unable to create form." + ) + + prompts.append( + FormPromptClass( + id=prompt.id, + title=prompt.title, + description=prompt.description, + type=prompt.type, + structuredPropertyParams=StructuredPropertyParamsClass( + urn=prompt.structured_property_urn + ) + if prompt.structured_property_urn + else None, + required=prompt.required, + ) + ) + else: + logger.warning(f"No prompts exist on form {self.urn}. Is that intended?") + + return prompts + + def upload_entities_for_form(self, emitter: DataHubGraph) -> Union[None, Exception]: + if self.entities and self.entities.urns: + formatted_entity_urns = ", ".join( + ['"{}"'.format(value) for value in self.entities.urns] + ) + query = UPLOAD_ENTITIES_FOR_FORMS.format( + form_urn=self.urn, entity_urns=formatted_entity_urns + ) + result = emitter.execute_graphql(query=query) + if not result: + return Exception(f"Could not bulk upload entities for form {self.urn}.") + + return None + + def create_form_filters(self, emitter: DataHubGraph) -> Union[None, Exception]: + filters_raw = [] + # Loop through each entity and assign a filter for it + if self.entities and self.entities.filters: + filters = self.entities.filters + if filters.types: + filters_raw.append( + Forms.format_form_filter("_entityType", filters.types) + ) + if filters.platforms: + urns = [ + make_data_platform_urn(platform) for platform in filters.platforms + ] + filters_raw.append(Forms.format_form_filter("platform", urns)) + if filters.domains: + urns = [] + for domain in filters.domains: + domain_urn = Forms.validate_domain_urn(domain) + if domain_urn: + urns.append(domain_urn) + filters_raw.append(Forms.format_form_filter("domains", urns)) + if filters.containers: + urns = [] + for container in filters.containers: + container_urn = Forms.validate_container_urn(container) + if container_urn: + urns.append(container_urn) + filters_raw.append(Forms.format_form_filter("container", urns)) + + filters_str = ", ".join(item for item in filters_raw) + result = emitter.execute_graphql( + query=CREATE_DYNAMIC_FORM_ASSIGNMENT.format( + form_urn=self.urn, filters=filters_str + ) + ) + if not result: + return Exception( + f"Could not bulk upload urns or filters for form {self.urn}." + ) + + return None + + def add_owners(self, emitter: DataHubGraph) -> Union[None, Exception]: + owner_urns: List[str] = [] + if self.owners: + owner_urns += Forms.format_owners(self.owners) + if self.group_owners: + owner_urns += Forms.format_group_owners(self.group_owners) + + ownership = OwnershipClass( + owners=[ + OwnerClass(owner=urn, type=OwnershipTypeClass.TECHNICAL_OWNER) + for urn in (owner_urns or []) + ], + ) + + try: + mcp = MetadataChangeProposalWrapper(entityUrn=self.urn, aspect=ownership) + emitter.emit_mcp(mcp) + except Exception as e: + logger.error(e) + + return None + + @staticmethod + def format_form_filter(field: str, urns: List[str]) -> str: + formatted_urns = ", ".join(['"{}"'.format(urn) for urn in urns]) + return FIELD_FILTER_TEMPLATE.format(field=field, values=formatted_urns) + + @staticmethod + def validate_domain_urn(domain: str) -> Union[str, None]: + if domain.startswith("urn:li:domain:"): + return domain + + logger.warning(f"{domain} is not an urn. Unable to create domain filter.") + return None + + @staticmethod + def validate_container_urn(container: str) -> Union[str, None]: + if container.startswith("urn:li:container:"): + return container + + logger.warning(f"{container} is not an urn. Unable to create container filter.") + return None + + @staticmethod + def from_datahub(graph: DataHubGraph, urn: str) -> "Forms": + form: Optional[FormInfoClass] = graph.get_aspect(urn, FormInfoClass) + assert form is not None + prompts = [] + for prompt_raw in form.prompts: + prompts.append( + Prompt( + id=prompt_raw.id, + title=prompt_raw.title, + description=prompt_raw.description, + type=prompt_raw.type, + structured_property_urn=prompt_raw.structuredPropertyParams.urn + if prompt_raw.structuredPropertyParams + else None, + ) + ) + return Forms( + urn=urn, + name=form.name, + description=form.description, + prompts=prompts, + type=form.type, + ) + + @staticmethod + def format_owners(owners: List[str]) -> List[str]: + formatted_owners: List[str] = [] + + for owner in owners: + if owner.startswith("urn:li:"): + formatted_owners.append(owner) + else: + formatted_owners.append(make_user_urn(owner)) + + return formatted_owners + + @staticmethod + def format_group_owners(owners: List[str]) -> List[str]: + formatted_owners: List[str] = [] + + for owner in owners: + if owner.startswith("urn:li:"): + formatted_owners.append(owner) + else: + formatted_owners.append(make_group_urn(owner)) + + return formatted_owners + + def to_yaml( + self, + file: Path, + ) -> None: + with open(file, "w") as fp: + yaml = YAML(typ="rt") # default, if not specfied, is 'rt' (round-trip) + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.default_flow_style = False + yaml.dump(self.dict(), fp) diff --git a/metadata-ingestion/src/datahub/api/entities/forms/forms_graphql_constants.py b/metadata-ingestion/src/datahub/api/entities/forms/forms_graphql_constants.py new file mode 100644 index 0000000000000..c227d8fc05366 --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/forms/forms_graphql_constants.py @@ -0,0 +1,27 @@ +UPLOAD_ENTITIES_FOR_FORMS = """ +mutation batchAssignForm {{ + batchAssignForm( + input: {{ + formUrn: "{form_urn}", + entityUrns: [{entity_urns}] + }} + ) +}} +""" + +FIELD_FILTER_TEMPLATE = ( + """{{ field: "{field}", values: [{values}], condition: EQUAL, negated: false }}""" +) + +CREATE_DYNAMIC_FORM_ASSIGNMENT = """ +mutation createDynamicFormAssignment {{ + createDynamicFormAssignment( + input: {{ + formUrn: "{form_urn}" + orFilters: [{{ + and: [{filters}] + }}] + }} + ) +}} +""" diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/__init__.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py new file mode 100644 index 0000000000000..af9bf3dccac5c --- /dev/null +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -0,0 +1,185 @@ +import logging +from enum import Enum +from pathlib import Path +from typing import List, Optional + +import yaml +from pydantic import validator +from ruamel.yaml import YAML + +from datahub.configuration.common import ConfigModel +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import DataHubGraph, get_default_graph +from datahub.metadata.schema_classes import ( + PropertyValueClass, + StructuredPropertyDefinitionClass, +) +from datahub.utilities.urns.urn import Urn + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class AllowedTypes(Enum): + STRING = "string" + RICH_TEXT = "rich_text" + NUMBER = "number" + DATE = "date" + URN = "urn" + + @staticmethod + def check_allowed_type(value: str) -> bool: + return value in [allowed_type.value for allowed_type in AllowedTypes] + + @staticmethod + def values(): + return ", ".join([allowed_type.value for allowed_type in AllowedTypes]) + + +class AllowedValue(ConfigModel): + value: str + description: Optional[str] + + +class TypeQualifierAllowedTypes(ConfigModel): + allowed_types: List[str] + + @validator("allowed_types") + def validate_allowed_types(cls, v): + validated_entity_type_urns = [] + if v: + with get_default_graph() as graph: + for et in v: + validated_urn = Urn.make_entity_type_urn(et) + if graph.exists(validated_urn): + validated_entity_type_urns.append(validated_urn) + else: + logger.warn( + f"Input {et} is not a valid entity type urn. Skipping." + ) + v = validated_entity_type_urns + if not v: + logger.warn("No allowed_types given within type_qualifier.") + return v + + +class StructuredProperties(ConfigModel): + id: Optional[str] + urn: Optional[str] + qualified_name: Optional[str] + type: str + value_entity_types: Optional[List[str]] + description: Optional[str] + display_name: Optional[str] + entity_types: Optional[List[str]] + cardinality: Optional[str] + allowed_values: Optional[List[AllowedValue]] + type_qualifier: Optional[TypeQualifierAllowedTypes] + + @property + def fqn(self) -> str: + assert self.urn is not None + return ( + self.qualified_name + or self.id + or Urn.create_from_string(self.urn).get_entity_id()[0] + ) + + @validator("urn", pre=True, always=True) + def urn_must_be_present(cls, v, values): + if not v: + assert "id" in values, "id must be present if urn is not" + return f"urn:li:structuredProperty:{values['id']}" + return v + + @staticmethod + def create(file: str) -> None: + emitter: DataHubGraph + + with get_default_graph() as emitter: + with open(file, "r") as fp: + structuredproperties: List[dict] = yaml.safe_load(fp) + for structuredproperty_raw in structuredproperties: + structuredproperty = StructuredProperties.parse_obj( + structuredproperty_raw + ) + if not structuredproperty.type.islower(): + structuredproperty.type = structuredproperty.type.lower() + logger.warn( + f"Structured property type should be lowercase. Updated to {structuredproperty.type}" + ) + if not AllowedTypes.check_allowed_type(structuredproperty.type): + raise ValueError( + f"Type {structuredproperty.type} is not allowed. Allowed types are {AllowedTypes.values()}" + ) + mcp = MetadataChangeProposalWrapper( + entityUrn=structuredproperty.urn, + aspect=StructuredPropertyDefinitionClass( + qualifiedName=structuredproperty.fqn, + valueType=Urn.make_data_type_urn(structuredproperty.type), + displayName=structuredproperty.display_name, + description=structuredproperty.description, + entityTypes=[ + Urn.make_entity_type_urn(entity_type) + for entity_type in structuredproperty.entity_types or [] + ], + cardinality=structuredproperty.cardinality, + allowedValues=[ + PropertyValueClass( + value=v.value, description=v.description + ) + for v in structuredproperty.allowed_values + ] + if structuredproperty.allowed_values + else None, + typeQualifier={ + "allowedTypes": structuredproperty.type_qualifier.allowed_types + } + if structuredproperty.type_qualifier + else None, + ), + ) + emitter.emit_mcp(mcp) + + logger.info(f"Created structured property {structuredproperty.urn}") + + @classmethod + def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties": + + structured_property: Optional[ + StructuredPropertyDefinitionClass + ] = graph.get_aspect(urn, StructuredPropertyDefinitionClass) + assert structured_property is not None + return StructuredProperties( + urn=urn, + qualified_name=structured_property.qualifiedName, + display_name=structured_property.displayName, + type=structured_property.valueType, + description=structured_property.description, + entity_types=structured_property.entityTypes, + cardinality=structured_property.cardinality, + allowed_values=[ + AllowedValue( + value=av.value, + description=av.description, + ) + for av in structured_property.allowedValues or [] + ] + if structured_property.allowedValues is not None + else None, + type_qualifier={ + "allowed_types": structured_property.typeQualifier.get("allowedTypes") + } + if structured_property.typeQualifier + else None, + ) + + def to_yaml( + self, + file: Path, + ) -> None: + with open(file, "w") as fp: + yaml = YAML(typ="rt") # default, if not specfied, is 'rt' (round-trip) + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.default_flow_style = False + yaml.dump(self.dict(), fp) diff --git a/metadata-ingestion/src/datahub/cli/docker_check.py b/metadata-ingestion/src/datahub/cli/docker_check.py index 97b88cbc8b8eb..47b89af6dfd04 100644 --- a/metadata-ingestion/src/datahub/cli/docker_check.py +++ b/metadata-ingestion/src/datahub/cli/docker_check.py @@ -193,6 +193,11 @@ def check_docker_quickstart() -> QuickstartStatus: .labels.get("com.docker.compose.project.config_files") .split(",") ) + + # If using profiles, alternative check + if config_files and "/profiles/" in config_files[0]: + return check_docker_quickstart_profiles(client) + all_containers = set() for config_file in config_files: with open(config_file, "r") as config_file: @@ -234,3 +239,35 @@ def check_docker_quickstart() -> QuickstartStatus: ) return QuickstartStatus(container_statuses) + + +def check_docker_quickstart_profiles(client: docker.DockerClient) -> QuickstartStatus: + container_statuses: List[DockerContainerStatus] = [] + containers = client.containers.list( + all=True, + filters={"label": "io.datahubproject.datahub.component=gms"}, + # We can get race conditions between docker running up / recreating + # containers and our status checks. + ignore_removed=True, + ) + if len(containers) == 0: + return QuickstartStatus([]) + + existing_containers = set() + # Check that the containers are running and healthy. + container: docker.models.containers.Container + for container in containers: + name = container.labels.get("com.docker.compose.service", container.name) + existing_containers.add(name) + status = ContainerStatus.OK + if container.status != "running": + status = ContainerStatus.DIED + elif "Health" in container.attrs["State"]: + if container.attrs["State"]["Health"]["Status"] == "starting": + status = ContainerStatus.STARTING + elif container.attrs["State"]["Health"]["Status"] != "healthy": + status = ContainerStatus.UNHEALTHY + + container_statuses.append(DockerContainerStatus(name, status)) + + return QuickstartStatus(container_statuses) diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 569a836f3ef5c..9c55f52497c0e 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -131,7 +131,7 @@ def run( async def run_pipeline_to_completion(pipeline: Pipeline) -> int: logger.info("Starting metadata ingestion") - with click_spinner.spinner(disable=no_spinner): + with click_spinner.spinner(disable=no_spinner or no_progress): try: pipeline.run() except Exception as e: diff --git a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py index 5d6c65512354a..a52a9dddff127 100644 --- a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py +++ b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py @@ -56,7 +56,6 @@ def _abort_if_non_existent_urn(graph: DataHubGraph, urn: str, operation: str) -> def _print_diff(orig_file, new_file): - with open(orig_file) as fp: orig_lines = fp.readlines() with open(new_file) as fp: @@ -388,7 +387,7 @@ def add_asset(urn: str, asset: str, validate_assets: bool) -> None: graph.emit(mcp) -@dataproduct.command(name="remove_asset", help="Add an asset to a Data Product") +@dataproduct.command(name="remove_asset", help="Remove an asset from a Data Product") @click.option("--urn", required=True, type=str) @click.option("--asset", required=True, type=str) @click.option( diff --git a/metadata-ingestion/src/datahub/cli/specific/dataset_cli.py b/metadata-ingestion/src/datahub/cli/specific/dataset_cli.py new file mode 100644 index 0000000000000..c702d0ec28961 --- /dev/null +++ b/metadata-ingestion/src/datahub/cli/specific/dataset_cli.py @@ -0,0 +1,67 @@ +import json +import logging +from pathlib import Path + +import click +from click_default_group import DefaultGroup + +from datahub.api.entities.dataset.dataset import Dataset +from datahub.ingestion.graph.client import get_default_graph +from datahub.telemetry import telemetry +from datahub.upgrade import upgrade + +logger = logging.getLogger(__name__) + + +@click.group(cls=DefaultGroup, default="upsert") +def dataset() -> None: + """A group of commands to interact with the Dataset entity in DataHub.""" + pass + + +@dataset.command( + name="upsert", +) +@click.option("-f", "--file", required=True, type=click.Path(exists=True)) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def upsert(file: Path) -> None: + """Upsert attributes to a Dataset in DataHub.""" + + with get_default_graph() as graph: + for dataset in Dataset.from_yaml(str(file)): + try: + for mcp in dataset.generate_mcp(): + graph.emit(mcp) + click.secho(f"Update succeeded for urn {dataset.urn}.", fg="green") + except Exception as e: + click.secho( + f"Update failed for id {id}. due to {e}", + fg="red", + ) + + +@dataset.command( + name="get", +) +@click.option("--urn", required=True, type=str) +@click.option("--to-file", required=False, type=str) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def get(urn: str, to_file: str) -> None: + """Get a Dataset from DataHub""" + + if not urn.startswith("urn:li:dataset:"): + urn = f"urn:li:dataset:{urn}" + + with get_default_graph() as graph: + if graph.exists(urn): + dataset: Dataset = Dataset.from_datahub(graph=graph, urn=urn) + click.secho( + f"{json.dumps(dataset.dict(exclude_unset=True, exclude_none=True), indent=2)}" + ) + if to_file: + dataset.to_yaml(Path(to_file)) + click.secho(f"Dataset yaml written to {to_file}", fg="green") + else: + click.secho(f"Dataset {urn} does not exist") diff --git a/metadata-ingestion/src/datahub/cli/specific/forms_cli.py b/metadata-ingestion/src/datahub/cli/specific/forms_cli.py new file mode 100644 index 0000000000000..a494396909b32 --- /dev/null +++ b/metadata-ingestion/src/datahub/cli/specific/forms_cli.py @@ -0,0 +1,53 @@ +import json +import logging +from pathlib import Path + +import click +from click_default_group import DefaultGroup + +from datahub.api.entities.forms.forms import Forms +from datahub.ingestion.graph.client import get_default_graph +from datahub.telemetry import telemetry +from datahub.upgrade import upgrade + +logger = logging.getLogger(__name__) + + +@click.group(cls=DefaultGroup, default="upsert") +def forms() -> None: + """A group of commands to interact with forms in DataHub.""" + pass + + +@forms.command( + name="upsert", +) +@click.option("-f", "--file", required=True, type=click.Path(exists=True)) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def upsert(file: Path) -> None: + """Upsert forms in DataHub.""" + + Forms.create(str(file)) + + +@forms.command( + name="get", +) +@click.option("--urn", required=True, type=str) +@click.option("--to-file", required=False, type=str) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def get(urn: str, to_file: str) -> None: + """Get form from DataHub""" + with get_default_graph() as graph: + if graph.exists(urn): + form: Forms = Forms.from_datahub(graph=graph, urn=urn) + click.secho( + f"{json.dumps(form.dict(exclude_unset=True, exclude_none=True), indent=2)}" + ) + if to_file: + form.to_yaml(Path(to_file)) + click.secho(f"Form yaml written to {to_file}", fg="green") + else: + click.secho(f"Form {urn} does not exist") diff --git a/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py b/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py new file mode 100644 index 0000000000000..4162d44b9b0ea --- /dev/null +++ b/metadata-ingestion/src/datahub/cli/specific/structuredproperties_cli.py @@ -0,0 +1,62 @@ +import json +import logging +from pathlib import Path + +import click +from click_default_group import DefaultGroup + +from datahub.api.entities.structuredproperties.structuredproperties import ( + StructuredProperties, +) +from datahub.ingestion.graph.client import get_default_graph +from datahub.telemetry import telemetry +from datahub.upgrade import upgrade +from datahub.utilities.urns.urn import Urn + +logger = logging.getLogger(__name__) + + +@click.group(cls=DefaultGroup, default="upsert") +def properties() -> None: + """A group of commands to interact with structured properties in DataHub.""" + pass + + +@properties.command( + name="upsert", +) +@click.option("-f", "--file", required=True, type=click.Path(exists=True)) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def upsert(file: Path) -> None: + """Upsert structured properties in DataHub.""" + + StructuredProperties.create(str(file)) + + +@properties.command( + name="get", +) +@click.option("--urn", required=True, type=str) +@click.option("--to-file", required=False, type=str) +@upgrade.check_upgrade +@telemetry.with_telemetry() +def get(urn: str, to_file: str) -> None: + """Get structured properties from DataHub""" + urn = Urn.make_structured_property_urn(urn) + + with get_default_graph() as graph: + if graph.exists(urn): + structuredproperties: StructuredProperties = ( + StructuredProperties.from_datahub(graph=graph, urn=urn) + ) + click.secho( + f"{json.dumps(structuredproperties.dict(exclude_unset=True, exclude_none=True), indent=2)}" + ) + if to_file: + structuredproperties.to_yaml(Path(to_file)) + click.secho( + f"Structured property yaml written to {to_file}", fg="green" + ) + else: + click.secho(f"Structured property {urn} does not exist") diff --git a/metadata-ingestion/src/datahub/emitter/mcp.py b/metadata-ingestion/src/datahub/emitter/mcp.py index d6aa695665e4e..47717f3c1ed19 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp.py +++ b/metadata-ingestion/src/datahub/emitter/mcp.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import TYPE_CHECKING, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, Union from datahub.emitter.aspect import ASPECT_MAP, JSON_CONTENT_TYPE, TIMESERIES_ASPECT_MAP from datahub.emitter.serialization_helper import post_json_transform, pre_json_transform @@ -100,7 +100,7 @@ def __post_init__(self) -> None: @classmethod def construct_many( - cls, entityUrn: str, aspects: List[Optional[_Aspect]] + cls, entityUrn: str, aspects: Sequence[Optional[_Aspect]] ) -> List["MetadataChangeProposalWrapper"]: return [cls(entityUrn=entityUrn, aspect=aspect) for aspect in aspects if aspect] diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py index 0cd37cc939854..1bf090a2e514e 100644 --- a/metadata-ingestion/src/datahub/entrypoints.py +++ b/metadata-ingestion/src/datahub/entrypoints.py @@ -23,7 +23,10 @@ from datahub.cli.put_cli import put from datahub.cli.specific.datacontract_cli import datacontract from datahub.cli.specific.dataproduct_cli import dataproduct +from datahub.cli.specific.dataset_cli import dataset +from datahub.cli.specific.forms_cli import forms from datahub.cli.specific.group_cli import group +from datahub.cli.specific.structuredproperties_cli import properties from datahub.cli.specific.user_cli import user from datahub.cli.state_cli import state from datahub.cli.telemetry import telemetry as telemetry_cli @@ -59,13 +62,6 @@ default=None, help="Enable debug logging.", ) -@click.option( - "--debug-vars/--no-debug-vars", - type=bool, - is_flag=True, - default=False, - help="Show variable values in stack traces. Implies --debug. While we try to avoid printing sensitive information like passwords, this may still happen.", -) @click.version_option( version=datahub_package.nice_version_name(), prog_name=datahub_package.__package_name__, @@ -73,13 +69,7 @@ def datahub( debug: bool, log_file: Optional[str], - debug_vars: bool, ) -> None: - if debug_vars: - # debug_vars implies debug. This option isn't actually used here, but instead - # read directly from the command line arguments in the main entrypoint. - debug = True - debug = debug or get_boolean_env_variable("DATAHUB_DEBUG", False) # Note that we're purposely leaking the context manager here. @@ -144,6 +134,9 @@ def init() -> None: datahub.add_command(user) datahub.add_command(group) datahub.add_command(dataproduct) +datahub.add_command(dataset) +datahub.add_command(properties) +datahub.add_command(forms) datahub.add_command(datacontract) try: diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py index 360ddf1129154..52d2e4a8f56e3 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py @@ -316,10 +316,12 @@ def _get_discriminated_type_from_schema(schema: Dict) -> str: @staticmethod def _get_description_from_any_schema(schema: Dict) -> str: - # we do a redundant `if description in schema` check to guard against the scenario that schema is not a dictionary - description = ( - (schema.get("description") or "") if "description" in schema else "" - ) + description = "" + if "description" in schema: + description = str(schema.get("description")) + elif "const" in schema: + schema_const = schema.get("const") + description = f"Const value: {schema_const}" if JsonSchemaTranslator._INJECT_DEFAULTS_INTO_DESCRIPTION: default = schema.get("default") if default is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 75fba6e9d426b..985c9118f3422 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -125,7 +125,9 @@ @dataclass class DBTSourceReport(StaleEntityRemovalSourceReport): - pass + sql_statements_parsed: int = 0 + sql_parser_detach_ctes_failures: int = 0 + sql_parser_skipped_missing_code: int = 0 class EmitDirective(ConfigEnum): @@ -821,6 +823,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ] test_nodes = [test_node for test_node in nodes if test_node.node_type == "test"] + logger.info(f"Creating dbt metadata for {len(nodes)} nodes") yield from self.create_platform_mces( non_test_nodes, additional_custom_props_filtered, @@ -829,6 +832,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.config.platform_instance, ) + logger.info(f"Updating {self.config.target_platform} metadata") yield from self.create_platform_mces( non_test_nodes, additional_custom_props_filtered, @@ -988,15 +992,22 @@ def _infer_schemas_and_update_cll(self, all_nodes_map: Dict[str, DBTNode]) -> No }, ) except Exception as e: + self.report.sql_parser_detach_ctes_failures += 1 + logger.debug( + f"Failed to detach CTEs from compiled code. {node.dbt_name} will not have column lineage." + ) sql_result = SqlParsingResult.make_from_error(e) else: sql_result = sqlglot_lineage( preprocessed_sql, schema_resolver=schema_resolver ) + self.report.sql_statements_parsed += 1 + else: + self.report.sql_parser_skipped_missing_code += 1 # Save the column lineage. if self.config.include_column_lineage and sql_result: - # We only save the debug info here. We're report errors based on it later, after + # We only save the debug info here. We'll report errors based on it later, after # applying the configured node filters. node.cll_debug_info = sql_result.debug_info diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py index 563b005d7a88d..6fd3c5ba309f9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py @@ -65,7 +65,7 @@ class DBTCoreConfig(DBTCommonConfig): _github_info_deprecated = pydantic_renamed_field("github_info", "git_info") - @validator("aws_connection") + @validator("aws_connection", always=True) def aws_connection_needed_if_s3_uris_present( cls, aws_connection: Optional[AwsConnectionConfig], values: Dict, **kwargs: Any ) -> Optional[AwsConnectionConfig]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py index d7f3dfb9279fb..972eb60ff5b05 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py @@ -13,8 +13,10 @@ make_data_platform_urn, make_dataplatform_instance_urn, make_dataset_urn_with_platform_instance, + make_domain_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import add_domain_to_entity_wu from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, @@ -53,6 +55,7 @@ DataPlatformInstanceClass, DatasetPropertiesClass, ) +from datahub.utilities.registries.domain_registry import DomainRegistry MAX_ITEMS_TO_RETRIEVE = 100 PAGE_SIZE = 100 @@ -68,6 +71,11 @@ class DynamoDBConfig(DatasetSourceConfigMixin, StatefulIngestionConfigBase): aws_access_key_id: str = Field(description="AWS Access Key ID.") aws_secret_access_key: pydantic.SecretStr = Field(description="AWS Secret Key.") + domain: Dict[str, AllowDenyPattern] = Field( + default=dict(), + description="regex patterns for tables to filter to assign domain_key. ", + ) + # This config option allows user to include a list of items from a table when we scan and construct the schema, # the key of this dict is table name and the value is the list of item primary keys in dynamodb format, # if the table use composite key then the value should have partition key and sort key present @@ -155,6 +163,12 @@ def __init__(self, ctx: PipelineContext, config: DynamoDBConfig, platform: str): self.report = DynamoDBSourceReport() self.platform = platform + if self.config.domain: + self.domain_registry = DomainRegistry( + cached_domains=[domain_id for domain_id in self.config.domain], + graph=self.ctx.graph, + ) + @classmethod def create(cls, config_dict: dict, ctx: PipelineContext) -> "DynamoDBSource": config = DynamoDBConfig.parse_obj(config_dict) @@ -234,6 +248,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: aspect=dataset_properties, ).as_workunit() + yield from self._get_domain_wu( + dataset_name=table_name, + entity_urn=dataset_urn, + ) + platform_instance_aspect = DataPlatformInstanceClass( platform=make_data_platform_urn(self.platform), instance=make_dataplatform_instance_urn( @@ -480,3 +499,20 @@ def get_field_type( def get_report(self) -> DynamoDBSourceReport: return self.report + + def _get_domain_wu( + self, dataset_name: str, entity_urn: str + ) -> Iterable[MetadataWorkUnit]: + domain_urn = None + for domain, pattern in self.config.domain.items(): + if pattern.allowed(dataset_name): + domain_urn = make_domain_urn( + self.domain_registry.get_domain_urn(domain) + ) + break + + if domain_urn: + yield from add_domain_to_entity_wu( + entity_urn=entity_urn, + domain_urn=domain_urn, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lkml_patched.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lkml_patched.py new file mode 100644 index 0000000000000..6506682b8ed8d --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lkml_patched.py @@ -0,0 +1,28 @@ +import pathlib +from typing import Union + +import lkml +import lkml.simple +import lkml.tree + +# Patch lkml to support the manifest.lkml files. +# We have to patch both locations because lkml uses a immutable tuple +# instead of a list for this type. +lkml.simple.PLURAL_KEYS = ( + *lkml.simple.PLURAL_KEYS, + "local_dependency", + "remote_dependency", + "constant", + "override_constant", +) +lkml.tree.PLURAL_KEYS = lkml.simple.PLURAL_KEYS + + +def load_lkml(path: Union[str, pathlib.Path]) -> dict: + """Loads a LookML file from disk and returns a dictionary.""" + + # Using this method instead of lkml.load directly ensures + # that our patches to lkml are applied. + + with open(path, "r") as file: + return lkml.load(file) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index 542bf64eb2f49..ab9887c900571 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -608,8 +608,14 @@ def _make_chart_metadata_events( else "" }, ) - chart_snapshot.aspects.append(chart_info) + + if dashboard and dashboard.folder_path is not None: + browse_path = BrowsePathsClass( + paths=[f"/looker/{dashboard.folder_path}/{dashboard.title}"] + ) + chart_snapshot.aspects.append(browse_path) + if dashboard is not None: ownership = self.get_ownership(dashboard) if ownership is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 33079f3fd9ac1..9317605d5b055 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -49,6 +49,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.git.git_import import GitClone +from datahub.ingestion.source.looker.lkml_patched import load_lkml from datahub.ingestion.source.looker.looker_common import ( CORPUSER_DATAHUB, LookerCommonConfig, @@ -98,13 +99,6 @@ _BASE_PROJECT_NAME = "__BASE" -# Patch lkml to support the local_dependency and remote_dependency keywords. -lkml.simple.PLURAL_KEYS = ( - *lkml.simple.PLURAL_KEYS, - "local_dependency", - "remote_dependency", -) - _EXPLORE_FILE_EXTENSION = ".explore.lkml" _VIEW_FILE_EXTENSION = ".view.lkml" _MODEL_FILE_EXTENSION = ".model.lkml" @@ -384,10 +378,9 @@ def from_looker_dict( ] for included_file in explore_files: try: - with open(included_file, "r") as file: - parsed = lkml.load(file) - included_explores = parsed.get("explores", []) - explores.extend(included_explores) + parsed = load_lkml(included_file) + included_explores = parsed.get("explores", []) + explores.extend(included_explores) except Exception as e: reporter.report_warning( path, f"Failed to load {included_file} due to {e}" @@ -514,24 +507,23 @@ def resolve_includes( f"Will be loading {included_file}, traversed here via {traversal_path}" ) try: - with open(included_file, "r") as file: - parsed = lkml.load(file) - seen_so_far.add(included_file) - if "includes" in parsed: # we have more includes to resolve! - resolved.extend( - LookerModel.resolve_includes( - parsed["includes"], - resolved_project_name, - root_project_name, - base_projects_folder, - included_file, - reporter, - seen_so_far, - traversal_path=traversal_path - + "." - + pathlib.Path(included_file).stem, - ) + parsed = load_lkml(included_file) + seen_so_far.add(included_file) + if "includes" in parsed: # we have more includes to resolve! + resolved.extend( + LookerModel.resolve_includes( + parsed["includes"], + resolved_project_name, + root_project_name, + base_projects_folder, + included_file, + reporter, + seen_so_far, + traversal_path=traversal_path + + "." + + pathlib.Path(included_file).stem, ) + ) except Exception as e: reporter.report_warning( path, f"Failed to load {included_file} due to {e}" @@ -648,21 +640,20 @@ def _load_viewfile( self.reporter.report_failure(path, f"failed to load view file: {e}") return None try: - with open(path, "r") as file: - logger.debug(f"Loading viewfile {path}") - parsed = lkml.load(file) - looker_viewfile = LookerViewFile.from_looker_dict( - absolute_file_path=path, - looker_view_file_dict=parsed, - project_name=project_name, - root_project_name=self._root_project_name, - base_projects_folder=self._base_projects_folder, - raw_file_content=raw_file_content, - reporter=reporter, - ) - logger.debug(f"adding viewfile for path {path} to the cache") - self.viewfile_cache[path] = looker_viewfile - return looker_viewfile + logger.debug(f"Loading viewfile {path}") + parsed = load_lkml(path) + looker_viewfile = LookerViewFile.from_looker_dict( + absolute_file_path=path, + looker_view_file_dict=parsed, + project_name=project_name, + root_project_name=self._root_project_name, + base_projects_folder=self._base_projects_folder, + raw_file_content=raw_file_content, + reporter=reporter, + ) + logger.debug(f"adding viewfile for path {path} to the cache") + self.viewfile_cache[path] = looker_viewfile + return looker_viewfile except Exception as e: self.reporter.report_failure(path, f"failed to load view file: {e}") return None @@ -1498,17 +1489,16 @@ def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): ) def _load_model(self, path: str) -> LookerModel: - with open(path, "r") as file: - logger.debug(f"Loading model from file {path}") - parsed = lkml.load(file) - looker_model = LookerModel.from_looker_dict( - parsed, - _BASE_PROJECT_NAME, - self.source_config.project_name, - self.base_projects_folder, - path, - self.reporter, - ) + logger.debug(f"Loading model from file {path}") + parsed = load_lkml(path) + looker_model = LookerModel.from_looker_dict( + parsed, + _BASE_PROJECT_NAME, + self.source_config.project_name, + self.base_projects_folder, + path, + self.reporter, + ) return looker_model def _platform_names_have_2_parts(self, platform: str) -> bool: @@ -1797,8 +1787,7 @@ def get_project_name(self, model_name: str) -> str: def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManifest]: manifest_file = folder / "manifest.lkml" if manifest_file.exists(): - with manifest_file.open() as fp: - manifest_dict = lkml.load(fp) + manifest_dict = load_lkml(manifest_file) manifest = LookerManifest( project_name=manifest_dict.get("project_name"), diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py index 6baa70aa581d6..675c87b13313d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py @@ -71,6 +71,7 @@ class GlossaryNodeConfig(ConfigModel): terms: Optional[List["GlossaryTermConfig"]] nodes: Optional[List["GlossaryNodeConfig"]] knowledge_links: Optional[List[KnowledgeCard]] + custom_properties: Optional[Dict[str, str]] # Private fields. _urn: str @@ -252,6 +253,7 @@ def get_mces_from_node( definition=glossaryNode.description, parentNode=parentNode, name=glossaryNode.name, + customProperties=glossaryNode.custom_properties, ) node_owners = parentOwners if glossaryNode.owners is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/openapi.py b/metadata-ingestion/src/datahub/ingestion/source/openapi.py index ad62ef7362aeb..1b3a6dc4bee58 100755 --- a/metadata-ingestion/src/datahub/ingestion/source/openapi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/openapi.py @@ -46,12 +46,20 @@ class OpenApiConfig(ConfigModel): - name: str = Field(description="") - url: str = Field(description="") - swagger_file: str = Field(description="") - ignore_endpoints: list = Field(default=[], description="") - username: str = Field(default="", description="") - password: str = Field(default="", description="") + name: str = Field(description="Name of ingestion.") + url: str = Field(description="Endpoint URL. e.g. https://example.com") + swagger_file: str = Field( + description="Route for access to the swagger file. e.g. openapi.json" + ) + ignore_endpoints: list = Field( + default=[], description="List of endpoints to ignore during ingestion." + ) + username: str = Field( + default="", description="Username used for basic HTTP authentication." + ) + password: str = Field( + default="", description="Password used for basic HTTP authentication." + ) proxies: Optional[dict] = Field( default=None, description="Eg. " @@ -59,9 +67,16 @@ class OpenApiConfig(ConfigModel): "If authentication is required, add it to the proxy url directly e.g. " "`http://user:pass@10.10.1.10:3128/`.", ) - forced_examples: dict = Field(default={}, description="") - token: Optional[str] = Field(default=None, description="") - get_token: dict = Field(default={}, description="") + forced_examples: dict = Field( + default={}, + description="If no example is provided for a route, it is possible to create one using forced_example.", + ) + token: Optional[str] = Field( + default=None, description="Token for endpoint authentication." + ) + get_token: dict = Field( + default={}, description="Retrieving a token from the endpoint." + ) def get_swagger(self) -> Dict: if self.get_token or self.token is not None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py index 8aeb5421891aa..8b517747307f8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py @@ -16,7 +16,7 @@ class ProcedureDependency: name: str type: str env: str - server: str + server: Optional[str] source: str = "mssql" @@ -34,7 +34,7 @@ def as_property(self) -> Dict[str, str]: @dataclass class MSSQLJob: db: str - platform_instance: str + platform_instance: Optional[str] name: str env: str source: str = "mssql" @@ -42,7 +42,7 @@ class MSSQLJob: @property def formatted_name(self) -> str: - return f"{self.formatted_platform_instance}.{self.name.replace(',', '-')}" + return self.name.replace(",", "-") @property def full_type(self) -> str: @@ -52,10 +52,6 @@ def full_type(self) -> str: def orchestrator(self) -> str: return self.source - @property - def formatted_platform_instance(self) -> str: - return self.platform_instance.replace(".", "/") - @property def cluster(self) -> str: return f"{self.env}" @@ -64,7 +60,7 @@ def cluster(self) -> str: @dataclass class MSSQLProceduresContainer: db: str - platform_instance: str + platform_instance: Optional[str] name: str env: str source: str = "mssql" @@ -72,16 +68,12 @@ class MSSQLProceduresContainer: @property def formatted_name(self) -> str: - return f"{self.formatted_platform_instance}.{self.name.replace(',', '-')}" + return self.name.replace(",", "-") @property def orchestrator(self) -> str: return self.source - @property - def formatted_platform_instance(self) -> str: - return self.platform_instance.replace(".", "/") - @property def cluster(self) -> str: return f"{self.env}" @@ -149,7 +141,7 @@ def full_type(self) -> str: @property def full_name(self) -> str: - return f"{self.formatted_name}.{self.formatted_name}" + return self.formatted_name @dataclass @@ -172,6 +164,9 @@ def urn(self) -> str: flow_id=self.entity.flow.formatted_name, job_id=self.entity.formatted_name, cluster=self.entity.flow.cluster, + platform_instance=self.entity.flow.platform_instance + if self.entity.flow.platform_instance + else None, ) def add_property( @@ -228,6 +223,9 @@ def urn(self) -> str: orchestrator=self.entity.orchestrator, flow_id=self.entity.formatted_name, cluster=self.entity.cluster, + platform_instance=self.entity.platform_instance + if self.entity.platform_instance + else None, ) @property diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index 2442df595d967..56706e6f90d38 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -7,7 +7,6 @@ import sqlalchemy.dialects.mssql # This import verifies that the dependencies are available. -import sqlalchemy_pytds # noqa: F401 from pydantic.fields import Field from sqlalchemy import create_engine, inspect from sqlalchemy.engine.base import Connection @@ -132,10 +131,6 @@ def get_sql_alchemy_url( uri = f"{uri}?{urllib.parse.urlencode(self.uri_args)}" return uri - @property - def host(self): - return self.platform_instance or self.host_port.split(":")[0] - @property def db(self): return self.database @@ -369,7 +364,7 @@ def loop_jobs( name=job_name, env=sql_config.env, db=db_name, - platform_instance=sql_config.host, + platform_instance=sql_config.platform_instance, ) data_flow = MSSQLDataFlow(entity=job) yield from self.construct_flow_workunits(data_flow=data_flow) @@ -404,7 +399,7 @@ def loop_stored_procedures( # noqa: C901 name=procedure_flow_name, env=sql_config.env, db=db_name, - platform_instance=sql_config.host, + platform_instance=sql_config.platform_instance, ) data_flow = MSSQLDataFlow(entity=mssql_default_job) with inspector.engine.connect() as conn: diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 827c630cfa148..18f8e3709a648 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -8,8 +8,18 @@ from pydantic.class_validators import root_validator, validator from pydantic.fields import Field -from datahub.configuration import ConfigModel -from datahub.emitter.mce_builder import DEFAULT_ENV, make_dataset_urn +from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.emitter.mce_builder import ( + make_chart_urn, + make_dashboard_urn, + make_dataset_urn, + make_domain_urn, +) +from datahub.emitter.mcp_builder import add_domain_to_entity_wu from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -49,6 +59,7 @@ DashboardInfoClass, ) from datahub.utilities import config_clean +from datahub.utilities.registries.domain_registry import DomainRegistry logger = logging.getLogger(__name__) @@ -72,7 +83,9 @@ } -class SupersetConfig(StatefulIngestionConfigBase, ConfigModel): +class SupersetConfig( + StatefulIngestionConfigBase, EnvConfigMixin, PlatformInstanceConfigMixin +): # See the Superset /security/login endpoint for details # https://superset.apache.org/docs/rest-api connect_uri: str = Field( @@ -82,6 +95,10 @@ class SupersetConfig(StatefulIngestionConfigBase, ConfigModel): default=None, description="optional URL to use in links (if `connect_uri` is only for ingestion)", ) + domain: Dict[str, AllowDenyPattern] = Field( + default=dict(), + description="regex patterns for tables to filter to assign domain_key. ", + ) username: Optional[str] = Field(default=None, description="Superset username.") password: Optional[str] = Field(default=None, description="Superset password.") @@ -92,10 +109,7 @@ class SupersetConfig(StatefulIngestionConfigBase, ConfigModel): provider: str = Field(default="db", description="Superset provider.") options: Dict = Field(default={}, description="") - env: str = Field( - default=DEFAULT_ENV, - description="Environment to use in namespace when constructing URNs", - ) + # TODO: Check and remove this if no longer needed. # Config database_alias is removed from sql sources. database_alias: Dict[str, str] = Field( @@ -188,6 +202,12 @@ def __init__(self, ctx: PipelineContext, config: SupersetConfig): } ) + if self.config.domain: + self.domain_registry = DomainRegistry( + cached_domains=[domain_id for domain_id in self.config.domain], + graph=self.ctx.graph, + ) + # Test the connection test_response = self.session.get(f"{self.config.connect_uri}/api/v1/dashboard/") if test_response.status_code == 200: @@ -233,7 +253,11 @@ def get_datasource_urn_from_id(self, datasource_id): return None def construct_dashboard_from_api_data(self, dashboard_data): - dashboard_urn = f"urn:li:dashboard:({self.platform},{dashboard_data['id']})" + dashboard_urn = make_dashboard_urn( + platform=self.platform, + name=dashboard_data["id"], + platform_instance=self.config.platform_instance, + ) dashboard_snapshot = DashboardSnapshot( urn=dashboard_urn, aspects=[Status(removed=False)], @@ -262,7 +286,11 @@ def construct_dashboard_from_api_data(self, dashboard_data): if not key.startswith("CHART-"): continue chart_urns.append( - f"urn:li:chart:({self.platform},{value.get('meta', {}).get('chartId', 'unknown')})" + make_chart_urn( + platform=self.platform, + name=value.get("meta", {}).get("chartId", "unknown"), + platform_instance=self.config.platform_instance, + ) ) # Build properties @@ -325,9 +353,17 @@ def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]: ) mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce) + yield from self._get_domain_wu( + title=dashboard_data.get("dashboard_title", ""), + entity_urn=dashboard_snapshot.urn, + ) def construct_chart_from_chart_data(self, chart_data): - chart_urn = f"urn:li:chart:({self.platform},{chart_data['id']})" + chart_urn = make_chart_urn( + platform=self.platform, + name=chart_data["id"], + platform_instance=self.config.platform_instance, + ) chart_snapshot = ChartSnapshot( urn=chart_urn, aspects=[Status(removed=False)], @@ -424,6 +460,10 @@ def emit_chart_mces(self) -> Iterable[MetadataWorkUnit]: mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot) yield MetadataWorkUnit(id=chart_snapshot.urn, mce=mce) + yield from self._get_domain_wu( + title=chart_data.get("slice_name", ""), + entity_urn=chart_snapshot.urn, + ) def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.emit_dashboard_mces() @@ -439,3 +479,18 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: def get_report(self) -> StaleEntityRemovalSourceReport: return self.report + + def _get_domain_wu(self, title: str, entity_urn: str) -> Iterable[MetadataWorkUnit]: + domain_urn = None + for domain, pattern in self.config.domain.items(): + if pattern.allowed(title): + domain_urn = make_domain_urn( + self.domain_registry.get_domain_urn(domain) + ) + break + + if domain_urn: + yield from add_domain_to_entity_wu( + entity_urn=entity_urn, + domain_urn=domain_urn, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/hive_metastore_proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/hive_metastore_proxy.py index 814d86a2f3234..2a98dda1c79c5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/hive_metastore_proxy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/hive_metastore_proxy.py @@ -55,7 +55,6 @@ class HiveMetastoreProxy(Closeable): - # TODO: Support for view lineage using SQL parsing # Why not use hive ingestion source directly here ? # 1. hive ingestion source assumes 2-level namespace heirarchy and currently # there is no other intermediate interface except sqlalchemy inspector diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 1bc47c6307849..7a47b1181ae36 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -1,7 +1,7 @@ import logging import re from concurrent.futures import ThreadPoolExecutor -from typing import Dict, Iterable, List, Optional, Set, Union +from typing import Dict, Iterable, List, Optional, Set, Tuple, Union from urllib.parse import urljoin from datahub.emitter.mce_builder import ( @@ -24,6 +24,7 @@ add_dataset_to_container, gen_containers, ) +from datahub.emitter.sql_parsing_builder import SqlParsingBuilder from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, @@ -67,6 +68,7 @@ DATA_TYPE_REGISTRY, Catalog, Column, + CustomCatalogType, Metastore, Notebook, NotebookId, @@ -104,6 +106,12 @@ from datahub.utilities.file_backed_collections import FileBackedDict from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column from datahub.utilities.registries.domain_registry import DomainRegistry +from datahub.utilities.sqlglot_lineage import ( + SchemaResolver, + SqlParsingResult, + sqlglot_lineage, + view_definition_lineage_helper, +) logger: logging.Logger = logging.getLogger(__name__) @@ -137,6 +145,7 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource): unity_catalog_api_proxy: UnityCatalogApiProxy platform: str = "databricks" platform_instance_name: Optional[str] + sql_parser_schema_resolver: Optional[SchemaResolver] = None def get_report(self) -> UnityCatalogReport: return self.report @@ -179,6 +188,9 @@ def __init__(self, ctx: PipelineContext, config: UnityCatalogSourceConfig): self.table_refs: Set[TableReference] = set() self.view_refs: Set[TableReference] = set() self.notebooks: FileBackedDict[Notebook] = FileBackedDict() + self.view_definitions: FileBackedDict[ + Tuple[TableReference, str] + ] = FileBackedDict() # Global map of tables, for profiling self.tables: FileBackedDict[Table] = FileBackedDict() @@ -191,6 +203,13 @@ def init_hive_metastore_proxy(self): self.config.get_sql_alchemy_url(HIVE_METASTORE), self.config.options ) self.report.hive_metastore_catalog_found = True + + if self.config.include_table_lineage: + self.sql_parser_schema_resolver = SchemaResolver( + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) except Exception as e: logger.debug("Exception", exc_info=True) self.warn( @@ -243,6 +262,8 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.process_metastores() + yield from self.get_view_lineage() + if self.config.include_notebooks: self.report.report_ingestion_stage_start("Notebook lineage") for notebook in self.notebooks.values(): @@ -304,7 +325,6 @@ def process_notebooks(self) -> Iterable[MetadataWorkUnit]: yield from self._gen_notebook_workunits(notebook) def _gen_notebook_workunits(self, notebook: Notebook) -> Iterable[MetadataWorkUnit]: - properties = {"path": notebook.path} if notebook.language: properties["language"] = notebook.language.value @@ -449,6 +469,17 @@ def process_table(self, table: Table, schema: Schema) -> Iterable[MetadataWorkUn table.ref, self.notebooks[str(notebook_id)] ) + # Sql parsing is required only for hive metastore view lineage + if ( + self.sql_parser_schema_resolver + and table.schema.catalog.type == CustomCatalogType.HIVE_METASTORE_CATALOG + ): + self.sql_parser_schema_resolver.add_schema_metadata( + dataset_urn, schema_metadata + ) + if table.view_definition: + self.view_definitions[dataset_urn] = (table.ref, table.view_definition) + yield from [ mcp.as_workunit() for mcp in MetadataChangeProposalWrapper.construct_many( @@ -828,8 +859,74 @@ def _create_schema_field(column: Column) -> List[SchemaFieldClass]: ) ] + def _run_sql_parser( + self, view_ref: TableReference, query: str, schema_resolver: SchemaResolver + ) -> Optional[SqlParsingResult]: + raw_lineage = sqlglot_lineage( + query, + schema_resolver=schema_resolver, + default_db=view_ref.catalog, + default_schema=view_ref.schema, + ) + view_urn = self.gen_dataset_urn(view_ref) + + if raw_lineage.debug_info.table_error: + logger.debug( + f"Failed to parse lineage for view {view_ref}: " + f"{raw_lineage.debug_info.table_error}" + ) + self.report.num_view_definitions_failed_parsing += 1 + self.report.view_definitions_parsing_failures.append( + f"Table-level sql parsing error for view {view_ref}: {raw_lineage.debug_info.table_error}" + ) + return None + + elif raw_lineage.debug_info.column_error: + self.report.num_view_definitions_failed_column_parsing += 1 + self.report.view_definitions_parsing_failures.append( + f"Column-level sql parsing error for view {view_ref}: {raw_lineage.debug_info.column_error}" + ) + else: + self.report.num_view_definitions_parsed += 1 + return view_definition_lineage_helper(raw_lineage, view_urn) + + def get_view_lineage(self) -> Iterable[MetadataWorkUnit]: + if not ( + self.config.include_hive_metastore + and self.config.include_table_lineage + and self.sql_parser_schema_resolver + ): + return + # This is only used for parsing view lineage. Usage, Operations are emitted elsewhere + builder = SqlParsingBuilder( + generate_lineage=True, + generate_usage_statistics=False, + generate_operations=False, + ) + for dataset_name in self.view_definitions.keys(): + view_ref, view_definition = self.view_definitions[dataset_name] + result = self._run_sql_parser( + view_ref, + view_definition, + self.sql_parser_schema_resolver, + ) + if result and result.out_tables: + # This does not yield any workunits but we use + # yield here to execute this method + yield from builder.process_sql_parsing_result( + result=result, + query=view_definition, + is_view_ddl=True, + include_column_lineage=self.config.include_view_column_lineage, + ) + yield from builder.gen_workunits() + def close(self): if self.hive_metastore_proxy: self.hive_metastore_proxy.close() + if self.view_definitions: + self.view_definitions.close() + if self.sql_parser_schema_resolver: + self.sql_parser_schema_resolver.close() super().close() diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py index 254b3d084f2be..e8e25a061a665 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py @@ -77,7 +77,7 @@ def __init__(self): mixedin = mixedin or isinstance(self, mixin) if not mixedin: assert ( - "Class does not implement one of required traits {self.allowed_mixins}" + f"Class does not implement one of required traits {self.allowed_mixins}" ) def _should_process( @@ -135,38 +135,37 @@ def _transform_or_record_mce( if mce.proposedSnapshot: self._record_mce(mce) if isinstance(self, SingleAspectTransformer): - aspect_type = ASPECT_MAP.get(self.aspect_name()) - if aspect_type: - # if we find a type corresponding to the aspect name we look for it in the mce - old_aspect = ( - builder.get_aspect_if_available( + aspect_type = ASPECT_MAP[self.aspect_name()] + + # If we find a type corresponding to the aspect name we look for it in the mce + # It's possible that the aspect is supported by the entity but not in the MCE + # snapshot union. In those cases, we just want to record the urn as seen. + supports_aspect = builder.can_add_aspect(mce, aspect_type) + if supports_aspect: + old_aspect = builder.get_aspect_if_available( + mce, + aspect_type, + ) + if old_aspect is not None: + # TRICKY: If the aspect is not present in the MCE, it might still show up in a + # subsequent MCP. As such, we _only_ mark the urn as processed if we actually + # find the aspect already in the MCE. + + transformed_aspect = self.transform_aspect( + entity_urn=mce.proposedSnapshot.urn, + aspect_name=self.aspect_name(), + aspect=old_aspect, + ) + + # If transformed_aspect is None, this will remove the aspect. + builder.set_aspect( mce, - aspect_type, + aspect_type=aspect_type, + aspect=transformed_aspect, ) - if builder.can_add_aspect(mce, aspect_type) - else None - ) - if old_aspect: - if isinstance(self, LegacyMCETransformer): - # use the transform_one pathway to transform this MCE - envelope.record = self.transform_one(mce) - else: - transformed_aspect = self.transform_aspect( - entity_urn=mce.proposedSnapshot.urn, - aspect_name=self.aspect_name(), - aspect=old_aspect, - ) - builder.set_aspect( - mce, - aspect_type=aspect_type, - aspect=transformed_aspect, - ) - envelope.record = mce + + envelope.record = mce self._mark_processed(mce.proposedSnapshot.urn) - else: - log.warning( - f"Could not locate a snapshot aspect type for aspect {self.aspect_name()}. This can lead to silent drops of messages in transformers." - ) elif isinstance(self, LegacyMCETransformer): # we pass down the full MCE envelope.record = self.transform_one(mce) @@ -202,7 +201,6 @@ def _transform_or_record_mcpw( def _handle_end_of_stream( self, envelope: RecordEnvelope ) -> Iterable[RecordEnvelope]: - if not isinstance(self, SingleAspectTransformer) and not isinstance( self, LegacyMCETransformer ): @@ -265,7 +263,7 @@ def transform( else None, ) if transformed_aspect: - structured_urn = Urn.create_from_string(urn) + structured_urn = Urn.from_string(urn) mcp: MetadataChangeProposalWrapper = ( MetadataChangeProposalWrapper( diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py index 25b18f0806fd6..4b64d38a9b42f 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/extract_dataset_tags.py @@ -34,7 +34,7 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> "ExtractDatasetTags" def _get_tags_to_add(self, entity_urn: str) -> List[TagAssociationClass]: if self.config.extract_tags_from == ExtractTagsOption.URN: - urn = DatasetUrn.create_from_string(entity_urn) + urn = DatasetUrn.from_string(entity_urn) match = re.search(self.config.extract_tags_regex, urn.get_dataset_name()) if match: captured_group = match.group(1) diff --git a/metadata-ingestion/src/datahub/specific/dataset.py b/metadata-ingestion/src/datahub/specific/dataset.py index 62ee4fc57b61b..d3c3de36198e3 100644 --- a/metadata-ingestion/src/datahub/specific/dataset.py +++ b/metadata-ingestion/src/datahub/specific/dataset.py @@ -23,6 +23,7 @@ ) from datahub.specific.custom_properties import CustomPropertiesPatchHelper from datahub.specific.ownership import OwnershipPatchHelper +from datahub.specific.structured_properties import StructuredPropertiesPatchHelper from datahub.utilities.urns.tag_urn import TagUrn from datahub.utilities.urns.urn import Urn @@ -103,6 +104,7 @@ def __init__( self, DatasetProperties.ASPECT_NAME ) self.ownership_patch_helper = OwnershipPatchHelper(self) + self.structured_properties_patch_helper = StructuredPropertiesPatchHelper(self) def add_owner(self, owner: Owner) -> "DatasetPatchBuilder": self.ownership_patch_helper.add_owner(owner) @@ -331,3 +333,33 @@ def set_display_name(self, display_name: str) -> "DatasetPatchBuilder": value=display_name, ) return self + + def set_structured_property( + self, property_name: str, value: Union[str, float, List[Union[str, float]]] + ) -> "DatasetPatchBuilder": + """ + This is a helper method to set a structured property. + @param property_name: the name of the property (either bare or urn form) + @param value: the value of the property (for multi-valued properties, this can be a list) + """ + self.structured_properties_patch_helper.set_property(property_name, value) + return self + + def add_structured_property( + self, property_name: str, value: Union[str, float] + ) -> "DatasetPatchBuilder": + """ + This is a helper method to add a structured property. + @param property_name: the name of the property (either bare or urn form) + @param value: the value of the property (for multi-valued properties, this value will be appended to the list) + """ + self.structured_properties_patch_helper.add_property(property_name, value) + return self + + def remove_structured_property(self, property_name: str) -> "DatasetPatchBuilder": + """ + This is a helper method to remove a structured property. + @param property_name: the name of the property (either bare or urn form) + """ + self.structured_properties_patch_helper.remove_property(property_name) + return self diff --git a/metadata-ingestion/src/datahub/specific/structured_properties.py b/metadata-ingestion/src/datahub/specific/structured_properties.py new file mode 100644 index 0000000000000..6b2592bf1cbba --- /dev/null +++ b/metadata-ingestion/src/datahub/specific/structured_properties.py @@ -0,0 +1,53 @@ +from typing import Generic, List, TypeVar, Union + +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal +from datahub.metadata.schema_classes import StructuredPropertyValueAssignmentClass +from datahub.utilities.urns.structured_properties_urn import ( + make_structured_property_urn, +) + +T = TypeVar("T", bound=MetadataPatchProposal) + + +class StructuredPropertiesPatchHelper(Generic[T]): + def __init__( + self, + parent: T, + aspect_name: str = "structuredProperties", + ) -> None: + self.aspect_name = aspect_name + self._parent = parent + self.aspect_field = "properties" + + def parent(self) -> T: + return self._parent + + def set_property( + self, key: str, value: Union[str, float, List[Union[str, float]]] + ) -> "StructuredPropertiesPatchHelper": + self.remove_property(key) + self.add_property(key, value) + return self + + def remove_property(self, key: str) -> "StructuredPropertiesPatchHelper": + self._parent._add_patch( + self.aspect_name, + "remove", + path=f"/{self.aspect_field}/{make_structured_property_urn(key)}", + value={}, + ) + return self + + def add_property( + self, key: str, value: Union[str, float, List[Union[str, float]]] + ) -> "StructuredPropertiesPatchHelper": + self._parent._add_patch( + self.aspect_name, + "add", + path=f"/{self.aspect_field}/{make_structured_property_urn(key)}", + value=StructuredPropertyValueAssignmentClass( + propertyUrn=make_structured_property_urn(key), + values=value if isinstance(value, list) else [value], + ), + ) + return self diff --git a/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py b/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py index 6ee47f028b7eb..2afb6088072fe 100644 --- a/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py +++ b/metadata-ingestion/src/datahub/utilities/advanced_thread_executor.py @@ -2,6 +2,7 @@ import collections import concurrent.futures +import logging import time from concurrent.futures import Future, ThreadPoolExecutor from threading import BoundedSemaphore @@ -20,6 +21,7 @@ from datahub.ingestion.api.closeable import Closeable +logger = logging.getLogger(__name__) _R = TypeVar("_R") _PARTITION_EXECUTOR_FLUSH_SLEEP_INTERVAL = 0.05 @@ -102,7 +104,19 @@ def _system_done_callback(future: Future) -> None: fn, args, kwargs, user_done_callback = self._pending_by_key[ key ].popleft() - self._submit_nowait(key, fn, args, kwargs, user_done_callback) + + try: + self._submit_nowait(key, fn, args, kwargs, user_done_callback) + except RuntimeError as e: + if self._executor._shutdown: + # If we're in shutdown mode, then we can't submit any more requests. + # That means we'll need to drop requests on the floor, which is to + # be expected in shutdown mode. + # The only reason we'd normally be in shutdown here is during + # Python exit (e.g. KeyboardInterrupt), so this is reasonable. + logger.debug("Dropping request due to shutdown") + else: + raise e else: # If there are no pending requests for this key, mark the key diff --git a/metadata-ingestion/src/datahub/utilities/urn_encoder.py b/metadata-ingestion/src/datahub/utilities/urn_encoder.py index 093c9ade8c152..b39dd04370682 100644 --- a/metadata-ingestion/src/datahub/utilities/urn_encoder.py +++ b/metadata-ingestion/src/datahub/utilities/urn_encoder.py @@ -3,6 +3,7 @@ # NOTE: Frontend relies on encoding these three characters. Specifically, we decode and encode schema fields for column level lineage. # If this changes, make appropriate changes to datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts +# We also rely on encoding these exact three characters when generating schemaField urns in our graphQL layer. Update SchemaFieldUtils if this changes. RESERVED_CHARS = {",", "(", ")"} RESERVED_CHARS_EXTENDED = RESERVED_CHARS.union({"%"}) diff --git a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py index fbde0d6e6d69a..1b50d4b2fe810 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py +++ b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py @@ -207,6 +207,46 @@ def url_encode(urn: str) -> str: # safe='' encodes '/' as '%2F' return urllib.parse.quote(urn, safe="") + @staticmethod + def make_data_type_urn(type: str) -> str: + if type.startswith("urn:li:dataType:"): + return type + else: + if not type.startswith("datahub."): + # we want all data types to be fully qualified within the datahub namespace + type = f"datahub.{type}" + return f"urn:li:dataType:{type}" + + @staticmethod + def get_data_type_from_urn(urn: str) -> str: + if urn.startswith("urn:li:dataType:"): + # urn is formatted like urn:li:dataType:datahub:{dataType}, so extract dataType by + # parsing by . and getting the last element + return urn.split(".")[-1] + return urn + + @staticmethod + def make_entity_type_urn(entity_type: str) -> str: + if entity_type.startswith("urn:li:entityType:"): + return entity_type + else: + if not entity_type.startswith("datahub."): + # we want all entity types to be fully qualified within the datahub namespace + entity_type = f"datahub.{entity_type}" + return f"urn:li:entityType:{entity_type}" + + @staticmethod + def make_structured_property_urn(structured_property: str) -> str: + if not structured_property.startswith("urn:li:structuredProperty:"): + return f"urn:li:structuredProperty:{structured_property}" + return structured_property + + @staticmethod + def make_form_urn(form: str) -> str: + if not form.startswith("urn:li:form:"): + return f"urn:li:form:{form}" + return form + class _SpecificUrn(Urn): ENTITY_TYPE: str = "" diff --git a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py new file mode 100644 index 0000000000000..5bd36a0656d99 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py @@ -0,0 +1,5 @@ +from datahub.metadata.urns import StructuredPropertyUrn # noqa: F401 + + +def make_structured_property_urn(structured_property_id: str) -> str: + return str(StructuredPropertyUrn.create_from_string(structured_property_id)) diff --git a/metadata-ingestion/tests/integration/business-glossary/business_glossary.yml b/metadata-ingestion/tests/integration/business-glossary/business_glossary.yml index da238701e718d..c919dde18b187 100644 --- a/metadata-ingestion/tests/integration/business-glossary/business_glossary.yml +++ b/metadata-ingestion/tests/integration/business-glossary/business_glossary.yml @@ -10,6 +10,8 @@ nodes: knowledge_links: - label: Wiki link for classification url: "https://en.wikipedia.org/wiki/Classification" + custom_properties: + is_confidential: true terms: - name: Sensitive description: Sensitive Data diff --git a/metadata-ingestion/tests/integration/business-glossary/glossary_events_auto_id_golden.json b/metadata-ingestion/tests/integration/business-glossary/glossary_events_auto_id_golden.json index b8cc922f0c1c3..1dce940b44390 100644 --- a/metadata-ingestion/tests/integration/business-glossary/glossary_events_auto_id_golden.json +++ b/metadata-ingestion/tests/integration/business-glossary/glossary_events_auto_id_golden.json @@ -6,6 +6,9 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": { + "is_confidential": "True" + }, "definition": "A set of terms related to Data Classification", "name": "Classification" } @@ -29,7 +32,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -54,7 +58,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -94,7 +99,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -119,7 +125,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -159,7 +166,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -176,7 +184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -216,7 +225,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -226,6 +236,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "All terms related to personal information", "name": "Personal Information" } @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -294,7 +306,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -332,7 +345,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -377,7 +391,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -387,6 +402,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities", "name": "Clients And Accounts" } @@ -410,7 +426,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -458,7 +475,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -496,7 +514,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -506,6 +525,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Common Business KPIs", "name": "KPIs" } @@ -529,7 +549,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +588,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -612,7 +636,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -627,7 +652,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +668,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +684,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -672,7 +700,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -687,7 +716,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -702,7 +732,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -717,7 +748,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -732,7 +764,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -747,7 +780,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -762,7 +796,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/business-glossary/glossary_events_golden.json b/metadata-ingestion/tests/integration/business-glossary/glossary_events_golden.json index e2b525658e36e..af85f6e2a3518 100644 --- a/metadata-ingestion/tests/integration/business-glossary/glossary_events_golden.json +++ b/metadata-ingestion/tests/integration/business-glossary/glossary_events_golden.json @@ -6,6 +6,9 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": { + "is_confidential": "True" + }, "definition": "A set of terms related to Data Classification", "name": "Classification" } @@ -29,7 +32,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -54,7 +58,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -94,7 +99,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -119,7 +125,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -159,7 +166,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -176,7 +184,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -216,7 +225,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -226,6 +236,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "All terms related to personal information", "name": "Personal Information" } @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -294,7 +306,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -332,7 +345,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -377,7 +391,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -387,6 +402,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities", "name": "Clients And Accounts" } @@ -410,7 +426,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -458,7 +475,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -496,7 +514,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -506,6 +525,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Common Business KPIs", "name": "KPIs" } @@ -529,7 +549,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +588,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -612,7 +636,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -627,7 +652,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +668,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +684,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -672,7 +700,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -687,7 +716,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -702,7 +732,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -717,7 +748,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -732,7 +764,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -747,7 +780,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } }, { @@ -762,7 +796,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "datahub-business-glossary-2020_04_14-07_00_00" + "runId": "datahub-business-glossary-2020_04_14-07_00_00", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/lkml_manifest_samples/complex-manifest.lkml b/metadata-ingestion/tests/integration/lookml/lkml_manifest_samples/complex-manifest.lkml new file mode 100644 index 0000000000000..3d2006621dd50 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/lkml_manifest_samples/complex-manifest.lkml @@ -0,0 +1,23 @@ +project_name: "complex-manifest-project" + +constant: CONNECTION_NAME { + value: "choose-connection" + export: override_required +} + +constant: other_variable { + value: "other-variable" + export: override_required +} + +local_dependency: { + project: "looker-hub" +} + +remote_dependency: remote-proj-1 { + override_constant: schema_name {value: "mycorp_prod" } + override_constant: choose-connection {value: "snowflake-conn-main"} +} + +remote_dependency: remote-proj-2 { +} diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 1ed0d05c84263..7d1e8d053a381 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -16,6 +16,7 @@ LookerModel, LookerRefinementResolver, LookMLSourceConfig, + load_lkml, ) from datahub.metadata.schema_classes import ( DatasetSnapshotClass, @@ -852,3 +853,14 @@ def test_same_name_views_different_file_path(pytestconfig, tmp_path, mock_time): output_path=tmp_path / mce_out, golden_path=test_resources_dir / mce_out, ) + + +def test_manifest_parser(pytestconfig: pytest.Config) -> None: + # This mainly tests that we're permissive enough that we don't crash when parsing the manifest file. + # We need the test because we monkeypatch the lkml library. + + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" + manifest_file = test_resources_dir / "lkml_manifest_samples/complex-manifest.lkml" + + manifest = load_lkml(manifest_file) + assert manifest diff --git a/metadata-ingestion/tests/integration/remote/content/business_glossary.yml b/metadata-ingestion/tests/integration/remote/content/business_glossary.yml index 59bea251a24e1..e0bee3eb4468f 100644 --- a/metadata-ingestion/tests/integration/remote/content/business_glossary.yml +++ b/metadata-ingestion/tests/integration/remote/content/business_glossary.yml @@ -10,6 +10,8 @@ nodes: knowledge_links: - label: Wiki link for classification url: "https://en.wikipedia.org/wiki/Classification" + custom_properties: + is_confidential: true terms: - name: Sensitive description: Sensitive Data diff --git a/metadata-ingestion/tests/integration/remote/golden/remote_glossary_golden.json b/metadata-ingestion/tests/integration/remote/golden/remote_glossary_golden.json index 1e1932822aee8..a3adcb7639712 100644 --- a/metadata-ingestion/tests/integration/remote/golden/remote_glossary_golden.json +++ b/metadata-ingestion/tests/integration/remote/golden/remote_glossary_golden.json @@ -6,6 +6,9 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": { + "is_confidential": "True" + }, "definition": "A set of terms related to Data Classification", "name": "Classification" } @@ -29,7 +32,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -54,7 +58,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -94,7 +99,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -119,7 +125,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -159,7 +166,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -176,7 +184,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -216,7 +225,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -226,6 +236,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "All terms related to personal information", "name": "Personal Information" } @@ -249,7 +260,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -294,7 +306,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -332,7 +345,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -377,7 +391,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -387,6 +402,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities", "name": "Clients And Accounts" } @@ -410,7 +426,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -458,7 +475,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -496,7 +514,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -506,6 +525,7 @@ "aspects": [ { "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo": { + "customProperties": {}, "definition": "Common Business KPIs", "name": "KPIs" } @@ -529,7 +549,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -567,7 +588,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -582,7 +604,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -597,7 +620,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -612,7 +636,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -627,7 +652,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -642,7 +668,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -657,7 +684,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -672,7 +700,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -687,7 +716,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -702,7 +732,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -717,7 +748,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -732,7 +764,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -747,7 +780,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } }, { @@ -762,7 +796,8 @@ }, "systemMetadata": { "lastObserved": 1629795600000, - "runId": "remote-4" + "runId": "remote-4", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 66ef9b097c973..4c0c1c6512ec7 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -88,14 +88,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup" + "name": "Weekly Demo Data Backup" } }, "systemMetadata": { @@ -106,24 +106,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { "customProperties": { - "job_id": "3565ea3e-9a3a-4cb0-acd5-213d740479a0", + "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2023-11-27 23:08:29.350000", - "date_modified": "2023-11-27 23:08:29.833000", + "date_created": "2024-01-19 11:45:06.667000", + "date_modified": "2024-01-19 11:45:06.840000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup.localhost.Weekly Demo Data Backup", + "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" } @@ -137,7 +137,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -1932,14 +1932,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.demodata.Foo.stored_procedures" + "name": "demodata.Foo.stored_procedures" } }, "systemMetadata": { @@ -1950,7 +1950,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2023-11-27 23:08:29.077000", - "date_modified": "2023-11-27 23:08:29.077000" + "date_created": "2024-01-19 11:45:06.590000", + "date_modified": "2024-01-19 11:45:06.590000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -1979,7 +1979,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -4381,7 +4381,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -4397,7 +4397,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -4413,7 +4413,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -4429,7 +4429,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -4442,5 +4442,800 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:47217386c89d8b94153f6ee31e7e77ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5eb0d61efa998d1ccd5cbdc6ce4bb4af", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2816b2cb7f90d3dce64125ba89fb1fa8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:20d0f0c94e9796ff44ff32d4d0e19084", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3600d2ebb33b25dac607624d7eae7575", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:280f2e3aefacc346d0ce1590ec337c7d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:cba5c3ca7f028fcf749593be369d3c24", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:58c30fa72f213ca7e12fb04f5a7d150f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9387ddfeb7b57672cabd761ade89c49c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.dbo.ProductsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", + "urn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.ItemsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.PersonsNew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + }, + { + "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", + "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:54727d9fd7deacef27641559125bbc56", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:141b0980dcb08f48544583e47cf48807", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", + "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index c1984828750eb..02c357259c3f5 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -88,14 +88,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup" + "name": "Weekly Demo Data Backup" } }, "systemMetadata": { @@ -106,24 +106,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { "customProperties": { - "job_id": "1f2f14ba-db84-4fa1-910e-7df71bede642", + "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2023-10-27 10:11:55.540000", - "date_modified": "2023-10-27 10:11:55.667000", + "date_created": "2024-01-19 11:45:06.667000", + "date_modified": "2024-01-19 11:45:06.840000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup.localhost.Weekly Demo Data Backup", + "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" } @@ -137,7 +137,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -1932,14 +1932,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.demodata.Foo.stored_procedures" + "name": "demodata.Foo.stored_procedures" } }, "systemMetadata": { @@ -1950,7 +1950,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2023-10-27 10:11:55.460000", - "date_modified": "2023-10-27 10:11:55.460000" + "date_created": "2024-01-19 11:45:06.590000", + "date_modified": "2024-01-19 11:45:06.590000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -1979,7 +1979,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -2324,7 +2324,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2340,7 +2340,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2356,7 +2356,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2372,7 +2372,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2385,5 +2385,415 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 9ce3664eff6a1..02c357259c3f5 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -88,14 +88,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup" + "name": "Weekly Demo Data Backup" } }, "systemMetadata": { @@ -106,24 +106,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { "customProperties": { - "job_id": "3b767c17-c921-4331-93d9-eb0e006045a4", + "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2023-11-23 11:04:47.927000", - "date_modified": "2023-11-23 11:04:48.090000", + "date_created": "2024-01-19 11:45:06.667000", + "date_modified": "2024-01-19 11:45:06.840000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup.localhost.Weekly Demo Data Backup", + "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" } @@ -137,7 +137,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -1932,14 +1932,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.demodata.Foo.stored_procedures" + "name": "demodata.Foo.stored_procedures" } }, "systemMetadata": { @@ -1950,7 +1950,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2023-11-23 11:04:47.857000", - "date_modified": "2023-11-23 11:04:47.857000" + "date_created": "2024-01-19 11:45:06.590000", + "date_modified": "2024-01-19 11:45:06.590000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -1979,7 +1979,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -2324,7 +2324,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2340,7 +2340,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2356,7 +2356,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2372,7 +2372,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2385,5 +2385,415 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 037a341b7d66e..ad15c654e44c9 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -88,14 +88,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup" + "name": "Weekly Demo Data Backup" } }, "systemMetadata": { @@ -106,24 +106,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { "customProperties": { - "job_id": "3b767c17-c921-4331-93d9-eb0e006045a4", + "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2023-11-23 11:04:47.927000", - "date_modified": "2023-11-23 11:04:48.090000", + "date_created": "2024-01-19 11:45:06.667000", + "date_modified": "2024-01-19 11:45:06.840000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", "command": "ALTER DATABASE DemoData SET READ_ONLY" }, "externalUrl": "", - "name": "localhost.Weekly Demo Data Backup.localhost.Weekly Demo Data Backup", + "name": "Weekly Demo Data Backup", "type": { "string": "MSSQL_JOB_STEP" } @@ -137,7 +137,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -1932,14 +1932,14 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "dataFlowInfo", "aspect": { "json": { "customProperties": {}, "externalUrl": "", - "name": "localhost.demodata.Foo.stored_procedures" + "name": "demodata.Foo.stored_procedures" } }, "systemMetadata": { @@ -1950,7 +1950,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2023-11-23 11:04:47.857000", - "date_modified": "2023-11-23 11:04:47.857000" + "date_created": "2024-01-19 11:45:06.590000", + "date_modified": "2024-01-19 11:45:06.590000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -1979,7 +1979,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "dataJobInputOutput", "aspect": { @@ -2324,7 +2324,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2340,7 +2340,7 @@ }, { "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD)", + "entityUrn": "urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2356,7 +2356,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.Weekly Demo Data Backup,PROD),localhost.Weekly Demo Data Backup)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD),Weekly Demo Data Backup)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2372,7 +2372,7 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,localhost.demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2385,5 +2385,415 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + }, + { + "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json index 649212c1041ed..7cc0f84ee5177 100644 --- a/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json +++ b/metadata-ingestion/tests/integration/unity/unity_catalog_mces_golden.json @@ -3463,6 +3463,66 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.bet,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.bet,PROD),betStatusId)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.view1,PROD),betStatusId)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.bet,PROD),channelId)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.view1,PROD),channelId)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.bet,PROD),combination)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:databricks,hive_metastore.bronze_kambi.view1,PROD),combination)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "unity-catalog-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:databricks,system.quickstart_schema.quickstart_table,PROD)", diff --git a/metadata-ingestion/tests/unit/schema/test_json_schema_util.py b/metadata-ingestion/tests/unit/schema/test_json_schema_util.py index 2635363ed8d2e..5e095fc0df8dc 100644 --- a/metadata-ingestion/tests/unit/schema/test_json_schema_util.py +++ b/metadata-ingestion/tests/unit/schema/test_json_schema_util.py @@ -725,6 +725,19 @@ def test_non_str_enums(): assert fields[0].description == 'One of: "baz", 1, null' +def test_const_description_pulled_correctly(): + schema = { + "$id": "test", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": {"bar": {"type": "string", "const": "not_defined"}}, + } + + fields = list(JsonSchemaTranslator.get_fields_from_schema(schema)) + expected_field_paths: List[str] = ["[version=2.0].[type=object].[type=string].bar"] + assert_field_paths_match(fields, expected_field_paths) + assert fields[0].description == "Const value: not_defined" + + def test_anyof_with_properties(): # We expect the event / timestamp fields to be included in both branches of the anyOf. diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_temp_table_shortcut.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_temp_table_shortcut.json new file mode 100644 index 0000000000000..974eddb961d64 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_redshift_temp_table_shortcut.json @@ -0,0 +1,47 @@ +{ + "query_type": "CREATE", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.other_schema.table1,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.my_schema.#my_custom_name,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.my_schema.#my_custom_name,PROD)", + "column": "col1", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.other_schema.table1,PROD)", + "column": "col1" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.my_schema.#my_custom_name,PROD)", + "column": "col2", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.other_schema.table1,PROD)", + "column": "col2" + } + ] + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index eb1ba06669112..42863ab005f07 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -3,7 +3,6 @@ import pytest from datahub.testing.check_sql_parser_result import assert_sql_result -from datahub.utilities.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT RESOURCE_DIR = pathlib.Path(__file__).parent / "goldens" @@ -802,10 +801,6 @@ def test_snowflake_update_hardcoded(): ) -def test_update_from_select(): - assert _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT == {"returning", "this"} - - def test_snowflake_update_from_table(): # Can create these tables with the following SQL: """ @@ -997,3 +992,30 @@ def test_redshift_materialized_view_auto_refresh(): expected_file=RESOURCE_DIR / "test_redshift_materialized_view_auto_refresh.json", ) + + +def test_redshift_temp_table_shortcut(): + # On redshift, tables starting with # are temporary tables. + assert_sql_result( + """ +CREATE TABLE #my_custom_name +distkey (1) +sortkey (1,2) +AS +WITH cte AS ( +SELECT * +FROM other_schema.table1 +) +SELECT * FROM cte +""", + dialect="redshift", + default_db="my_db", + default_schema="my_schema", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:redshift,my_db.other_schema.table1,PROD)": { + "col1": "INTEGER", + "col2": "INTEGER", + }, + }, + expected_file=RESOURCE_DIR / "test_redshift_temp_table_shortcut.json", + ) diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py new file mode 100644 index 0000000000000..b01c512c383cb --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py @@ -0,0 +1,20 @@ +from datahub.utilities.sqlglot_lineage import ( + _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT, + _get_dialect, + _is_dialect_instance, +) + + +def test_update_from_select(): + assert _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT == {"returning", "this"} + + +def test_is_dialect_instance(): + snowflake = _get_dialect("snowflake") + + assert _is_dialect_instance(snowflake, "snowflake") + assert not _is_dialect_instance(snowflake, "bigquery") + + redshift = _get_dialect("redshift") + assert _is_dialect_instance(redshift, ["redshift", "snowflake"]) + assert _is_dialect_instance(redshift, ["postgres", "snowflake"]) diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index b14953d7ce021..873943fd43781 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -1,13 +1,13 @@ plugins { id("com.palantir.git-version") apply false + id 'java' + id 'com.github.johnrengelman.shadow' + id 'jacoco' + id 'signing' + id 'io.codearte.nexus-staging' + id 'maven-publish' } -apply plugin: 'java' -apply plugin: 'com.github.johnrengelman.shadow' -apply plugin: 'jacoco' -apply plugin: 'signing' -apply plugin: 'io.codearte.nexus-staging' -apply plugin: 'maven-publish' -apply plugin: 'org.hidetake.swagger.generator' + apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens @@ -15,8 +15,7 @@ import org.apache.tools.ant.filters.ReplaceTokens jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation dependencies { - implementation project(':metadata-models') - implementation project(path: ':metadata-models', configuration: "dataTemplate") + implementation project(':entity-registry') implementation(externalDependency.kafkaAvroSerializer) { exclude group: "org.apache.avro" } @@ -29,10 +28,7 @@ dependencies { compileOnly externalDependency.httpAsyncClient implementation externalDependency.jacksonDataBind - implementation externalDependency.javaxValidation runtimeOnly externalDependency.jna - implementation externalDependency.springContext - implementation externalDependency.swaggerAnnotations implementation externalDependency.slf4jApi compileOnly externalDependency.lombok @@ -46,8 +42,6 @@ dependencies { testImplementation externalDependency.testContainers testImplementation externalDependency.httpAsyncClient testRuntimeOnly externalDependency.logbackClassic - - swaggerCodegen externalDependency.swaggerCli } task copyAvroSchemas { @@ -81,13 +75,13 @@ shadowJar { // preventing java multi-release JAR leakage // https://github.com/johnrengelman/shadow/issues/729 exclude('module-info.class', 'META-INF/versions/**', - '**/LICENSE', '**/LICENSE.txt', '**/NOTICE', '**/NOTICE.txt') + '**/LICENSE', '**/LICENSE*.txt', '**/NOTICE', '**/NOTICE.txt', 'licenses/**', 'log4j2.*', 'log4j.*') mergeServiceFiles() // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first relocate 'org.springframework', 'datahub.shaded.org.springframework' relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson' - relocate 'org.yaml', 'io.acryl.shaded.org.yaml' // Required for shading snakeyaml + relocate 'org.yaml', 'datahub.shaded.org.yaml' // Required for shading snakeyaml relocate 'net.jcip.annotations', 'datahub.shaded.annotations' relocate 'javassist', 'datahub.shaded.javassist' relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs' @@ -95,6 +89,7 @@ shadowJar { relocate 'antlr', 'datahub.shaded.antlr' relocate 'com.google.common', 'datahub.shaded.com.google.common' relocate 'org.apache.commons', 'datahub.shaded.org.apache.commons' + relocate 'org.apache.maven', 'datahub.shaded.org.apache.maven' relocate 'org.reflections', 'datahub.shaded.org.reflections' relocate 'st4hidden', 'datahub.shaded.st4hidden' relocate 'org.stringtemplate', 'datahub.shaded.org.stringtemplate' @@ -104,7 +99,6 @@ shadowJar { relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine' relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' relocate 'com.google.errorprone', 'datahub.shaded.com.google.errorprone' - relocate 'com.sun.jna', 'datahub.shaded.com.sun.jna' // Below jars added for kafka emitter only relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro' relocate 'com.thoughtworks.paranamer', 'datahub.shaded.com.thoughtworks.paranamer' @@ -121,6 +115,9 @@ shadowJar { relocate 'common.message', 'datahub.shaded.common.message' relocate 'org.glassfish', 'datahub.shaded.org.glassfish' relocate 'ch.randelshofer', 'datahub.shaded.ch.randelshofer' + relocate 'io.github.classgraph', 'datahub.shaded.io.github.classgraph' + relocate 'nonapi.io.github.classgraph', 'datahub.shaded.nonapi.io.github.classgraph' + relocate 'com.github.fge', 'datahub.shaded.com.github.fge' finalizedBy checkShadowJar } @@ -207,28 +204,4 @@ nexusStaging { //required only for projects registered in Sonatype after 2021-02-24 username = System.getenv("NEXUS_USERNAME") password = System.getenv("NEXUS_PASSWORD") -} - -tasks.register('generateOpenApiPojos', GenerateSwaggerCode) { - it.setInputFile(file("${project(':metadata-models').projectDir}/src/generatedJsonSchema/combined/open-api.yaml")) - it.setOutputDir(file("$projectDir/generated")) - it.setLanguage("spring") - it.setComponents(['models']) - it.setTemplateDir(file("$projectDir/src/main/resources/JavaSpring")) - it.setAdditionalProperties(["group-id" : "io.datahubproject", - "dateLibrary" : "java8", - "java8" : "true", - "modelPropertyNaming": "original", - "modelPackage" : "io.datahubproject.openapi.generated"] as Map) - - dependsOn ':metadata-models:generateJsonSchema' -} - -compileJava.dependsOn generateOpenApiPojos -processResources.dependsOn generateOpenApiPojos -sourceSets.main.java.srcDir "${generateOpenApiPojos.outputDir}/src/main/java" -sourceSets.main.resources.srcDir "${generateOpenApiPojos.outputDir}/src/main/resources" - -clean { - project.delete("$projectDir/generated") } \ No newline at end of file diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh index 02a1d06b73acf..e2c9ec16d49f8 100755 --- a/metadata-integration/java/datahub-client/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh @@ -35,7 +35,8 @@ jar -tvf $jarFile |\ grep -v "linux/" |\ grep -v "darwin" |\ grep -v "MetadataChangeProposal.avsc" |\ - grep -v "aix" + grep -v "aix" |\ + grep -v "com/sun/" if [ $? -ne 0 ]; then echo "✅ No unexpected class paths found in ${jarFile}" diff --git a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java index 5bd10245899e4..1107f552012db 100644 --- a/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java +++ b/metadata-integration/java/datahub-client/src/test/java/datahub/client/patch/PatchTest.java @@ -18,20 +18,20 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dataset.DatasetLineageType; +import com.linkedin.metadata.aspect.patch.builder.ChartInfoPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.DashboardInfoPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.DataFlowInfoPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.DataJobInfoPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.DataJobInputOutputPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.EditableSchemaMetadataPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.OwnershipPatchBuilder; +import com.linkedin.metadata.aspect.patch.builder.UpstreamLineagePatchBuilder; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; import datahub.client.file.FileEmitter; import datahub.client.file.FileEmitterConfig; -import datahub.client.patch.chart.ChartInfoPatchBuilder; -import datahub.client.patch.common.OwnershipPatchBuilder; -import datahub.client.patch.dashboard.DashboardInfoPatchBuilder; -import datahub.client.patch.dataflow.DataFlowInfoPatchBuilder; -import datahub.client.patch.datajob.DataJobInfoPatchBuilder; -import datahub.client.patch.datajob.DataJobInputOutputPatchBuilder; -import datahub.client.patch.dataset.DatasetPropertiesPatchBuilder; -import datahub.client.patch.dataset.EditableSchemaMetadataPatchBuilder; -import datahub.client.patch.dataset.UpstreamLineagePatchBuilder; import datahub.client.rest.RestEmitter; import datahub.client.rest.RestEmitterConfig; import java.io.IOException; diff --git a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh index 930e3ab7be9e1..e3aa181c58801 100755 --- a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh @@ -38,7 +38,8 @@ jar -tvf $jarFile |\ grep -v "linux/" |\ grep -v "darwin" |\ grep -v "MetadataChangeProposal.avsc" |\ - grep -v "aix" + grep -v "aix" |\ + grep -v "com/sun/" if [ $? -ne 0 ]; then echo "✅ No unexpected class paths found in ${jarFile}" diff --git a/metadata-integration/java/examples/build.gradle b/metadata-integration/java/examples/build.gradle index ddf574e8c8905..62c80562c7c3b 100644 --- a/metadata-integration/java/examples/build.gradle +++ b/metadata-integration/java/examples/build.gradle @@ -4,7 +4,6 @@ plugins { } dependencies { - implementation externalDependency.slf4jApi compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok @@ -12,8 +11,6 @@ dependencies { implementation externalDependency.typesafeConfig implementation externalDependency.opentracingJdbc - implementation project(path: ':li-utils') - implementation project(path: ':metadata-models') implementation project(path: ':metadata-integration:java:datahub-client', configuration: 'shadow') implementation externalDependency.httpAsyncClient diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java index 4cff55afc92de..e84511083b6d9 100644 --- a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java @@ -3,9 +3,9 @@ import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DataJobInputOutputPatchBuilder; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; -import datahub.client.patch.datajob.DataJobInputOutputPatchBuilder; import datahub.client.rest.RestEmitter; import java.io.IOException; import java.util.concurrent.ExecutionException; diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java index b30cb5166df70..03f0673cd85a4 100644 --- a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java @@ -1,9 +1,9 @@ package io.datahubproject.examples; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; -import datahub.client.patch.dataset.DatasetPropertiesPatchBuilder; import datahub.client.rest.RestEmitter; import java.io.IOException; import java.util.concurrent.ExecutionException; diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java index 0a89e87060698..eb8f700c4b068 100644 --- a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java @@ -1,9 +1,9 @@ package io.datahubproject.examples; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; -import datahub.client.patch.dataset.DatasetPropertiesPatchBuilder; import datahub.client.rest.RestEmitter; import java.io.IOException; import java.util.concurrent.ExecutionException; diff --git a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesReplace.java b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesReplace.java index 053c1f068e048..1586d9b069b24 100644 --- a/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesReplace.java +++ b/metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesReplace.java @@ -1,9 +1,9 @@ package io.datahubproject.examples; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; import com.linkedin.mxe.MetadataChangeProposal; import datahub.client.MetadataWriteResponse; -import datahub.client.patch.dataset.DatasetPropertiesPatchBuilder; import datahub.client.rest.RestEmitter; import java.io.IOException; import java.util.HashMap; diff --git a/metadata-integration/java/spark-lineage/build.gradle b/metadata-integration/java/spark-lineage/build.gradle index c5dd9b5012c29..8d6160631bf45 100644 --- a/metadata-integration/java/spark-lineage/build.gradle +++ b/metadata-integration/java/spark-lineage/build.gradle @@ -102,6 +102,7 @@ shadowJar { // prevent jni conflict with spark exclude '**/libzstd-jni.*' exclude '**/com_github_luben_zstd_*' + exclude '**/log4j*.xml' relocate 'com.fasterxml.jackson', 'datahub.shaded.jackson' relocate 'org.slf4j','datahub.shaded.org.slf4j' @@ -113,6 +114,10 @@ shadowJar { relocate 'io.opentracing','datahub.spark2.shaded.io.opentracing' relocate 'io.netty','datahub.spark2.shaded.io.netty' relocate 'ch.randelshofer', 'datahub.shaded.ch.randelshofer' + relocate 'com.sun', 'datahub.shaded.com.sun' + relocate 'avroutil1', 'datahub.shaded.avroutil1' + relocate 'com.github', 'datahub.shaded.com.github' + relocate 'org.apache.maven', 'datahub.shaded.org.apache.maven' finalizedBy checkShadowJar } diff --git a/metadata-integration/java/spark-lineage/spark-smoke-test/setup_spark_smoke_test.sh b/metadata-integration/java/spark-lineage/spark-smoke-test/setup_spark_smoke_test.sh index 33cac9d562cd8..90a90be768a51 100755 --- a/metadata-integration/java/spark-lineage/spark-smoke-test/setup_spark_smoke_test.sh +++ b/metadata-integration/java/spark-lineage/spark-smoke-test/setup_spark_smoke_test.sh @@ -30,7 +30,7 @@ echo "--------------------------------------------------------------------" cd "${SMOKE_TEST_ROOT_DIR}"/docker #bring up spark cluster -docker-compose -f spark-docker-compose.yml up -d +docker compose -f spark-docker-compose.yml up -d echo "--------------------------------------------------------------------" echo "Executing spark-submit jobs" diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 568b99acdf894..f96517d93fca6 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -1,5 +1,7 @@ -apply plugin: 'java-library' -apply plugin: 'org.hidetake.swagger.generator' +plugins { + id 'java-library' + id 'pegasus' +} configurations { enhance @@ -46,8 +48,8 @@ dependencies { implementation externalDependency.ebeanDdl implementation externalDependency.opentelemetryAnnotations implementation externalDependency.resilience4j - api externalDependency.springContext - implementation externalDependency.swaggerAnnotations + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 implementation(externalDependency.mixpanel) { exclude group: 'org.json', module: 'json' } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java new file mode 100644 index 0000000000000..974406c0be0df --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/EntityClientAspectRetriever.java @@ -0,0 +1,35 @@ +package com.linkedin.metadata.client; + +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.r2.RemoteInvocationException; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Getter; + +@Builder +public class EntityClientAspectRetriever implements AspectRetriever { + @Getter private final EntityRegistry entityRegistry; + private final SystemEntityClient entityClient; + + @Nullable + @Override + public Aspect getLatestAspectObject(@Nonnull Urn urn, @Nonnull String aspectName) + throws RemoteInvocationException, URISyntaxException { + return entityClient.getLatestAspectObject(urn, aspectName); + } + + @Nonnull + @Override + public Map> getLatestAspectObjects( + Set urns, Set aspectNames) throws RemoteInvocationException, URISyntaxException { + return entityClient.getLatestAspects(urns, aspectNames); + } +} diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index 9b3f42a37b45d..0ebe9ed1d1b66 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -4,6 +4,7 @@ import static com.linkedin.metadata.search.utils.SearchUtils.*; import com.datahub.authentication.Authentication; +import com.datahub.plugins.auth.authorization.Authorizer; import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -18,7 +19,6 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.aspect.EnvelopedAspectArray; @@ -31,7 +31,6 @@ import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.IngestResult; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.AutoCompleteResult; @@ -48,6 +47,7 @@ import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.service.RollbackService; import com.linkedin.metadata.shared.ValidationUtils; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.utils.metrics.MetricUtils; @@ -85,15 +85,15 @@ public class JavaEntityClient implements EntityClient { private final Clock _clock = Clock.systemUTC(); - private final EntityService _entityService; + private final EntityService _entityService; private final DeleteEntityService _deleteEntityService; private final EntitySearchService _entitySearchService; private final CachingEntitySearchService _cachingEntitySearchService; private final SearchService _searchService; private final LineageSearchService _lineageSearchService; private final TimeseriesAspectService _timeseriesAspectService; + private final RollbackService rollbackService; private final EventProducer _eventProducer; - private final RestliEntityClient _restliEntityClient; @Nullable public EntityResponse getV2( @@ -713,11 +713,7 @@ public String ingestProposal( Stream.concat(Stream.of(metadataChangeProposal), additionalChanges.stream()); AspectsBatch batch = AspectsBatchImpl.builder() - .mcps( - proposalStream.collect(Collectors.toList()), - auditStamp, - _entityService.getEntityRegistry(), - this) + .mcps(proposalStream.collect(Collectors.toList()), auditStamp, _entityService) .build(); IngestResult one = _entityService.ingestProposal(batch, async).stream().findFirst().get(); @@ -780,9 +776,10 @@ public void producePlatformEvent( } @Override - public void rollbackIngestion(@Nonnull String runId, @Nonnull Authentication authentication) + public void rollbackIngestion( + @Nonnull String runId, @Nonnull Authorizer authorizer, @Nonnull Authentication authentication) throws Exception { - _restliEntityClient.rollbackIngestion(runId, authentication); + rollbackService.rollbackIngestion(runId, false, true, authorizer, authentication); } private void tryIndexRunId(Urn entityUrn, @Nullable SystemMetadata systemMetadata) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java index 31c2846a9c9f3..fa020903c34f0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java @@ -2,18 +2,18 @@ import com.datahub.authentication.Authentication; import com.linkedin.entity.client.EntityClientCache; -import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.service.RollbackService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import javax.annotation.Nonnull; import lombok.Getter; /** Java backed SystemEntityClient */ @@ -24,16 +24,16 @@ public class SystemJavaEntityClient extends JavaEntityClient implements SystemEn private final Authentication systemAuthentication; public SystemJavaEntityClient( - EntityService entityService, + EntityService entityService, DeleteEntityService deleteEntityService, EntitySearchService entitySearchService, CachingEntitySearchService cachingEntitySearchService, SearchService searchService, LineageSearchService lineageSearchService, TimeseriesAspectService timeseriesAspectService, + RollbackService rollbackService, EventProducer eventProducer, - RestliEntityClient restliEntityClient, - Authentication systemAuthentication, + @Nonnull Authentication systemAuthentication, EntityClientCacheConfig cacheConfig) { super( entityService, @@ -43,8 +43,8 @@ public SystemJavaEntityClient( searchService, lineageSearchService, timeseriesAspectService, - eventProducer, - restliEntityClient); + rollbackService, + eventProducer); this.systemAuthentication = systemAuthentication; this.entityClientCache = buildEntityClientCache(SystemJavaEntityClient.class, systemAuthentication, cacheConfig); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index ed69e919a7b24..b3b11d200ec0d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -12,6 +12,7 @@ import static com.linkedin.metadata.Constants.UI_SOURCE; import static com.linkedin.metadata.search.utils.BrowsePathUtils.buildDataPlatformUrn; import static com.linkedin.metadata.search.utils.BrowsePathUtils.getDefaultBrowsePath; +import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToAspectMap; import static com.linkedin.metadata.utils.PegasusUtils.constructMCL; import static com.linkedin.metadata.utils.PegasusUtils.getDataTemplateClassFromSchema; import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; @@ -46,7 +47,6 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.Aspect; @@ -84,6 +84,7 @@ import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; +import com.linkedin.r2.RemoteInvocationException; import com.linkedin.util.Pair; import io.ebean.PagedList; import io.ebean.Transaction; @@ -166,14 +167,12 @@ public class EntityServiceImpl implements EntityService { private final Integer ebeanMaxTransactionRetry; - private SystemEntityClient systemEntityClient; - public EntityServiceImpl( @Nonnull final AspectDao aspectDao, @Nonnull final EventProducer producer, @Nonnull final EntityRegistry entityRegistry, final boolean alwaysEmitChangeLog, - final UpdateIndicesService updateIndicesService, + @Nullable final UpdateIndicesService updateIndicesService, final PreProcessHooks preProcessHooks) { this( aspectDao, @@ -190,9 +189,9 @@ public EntityServiceImpl( @Nonnull final EventProducer producer, @Nonnull final EntityRegistry entityRegistry, final boolean alwaysEmitChangeLog, - final UpdateIndicesService updateIndicesService, + @Nullable final UpdateIndicesService updateIndicesService, final PreProcessHooks preProcessHooks, - final Integer retry) { + @Nullable final Integer retry) { _aspectDao = aspectDao; _producer = producer; @@ -200,21 +199,13 @@ public EntityServiceImpl( _entityToValidAspects = buildEntityToValidAspects(entityRegistry); _alwaysEmitChangeLog = alwaysEmitChangeLog; _updateIndicesService = updateIndicesService; + if (_updateIndicesService != null) { + _updateIndicesService.initializeAspectRetriever(this); + } _preProcessHooks = preProcessHooks; ebeanMaxTransactionRetry = retry != null ? retry : DEFAULT_MAX_TRANSACTION_RETRY; } - @Override - public void setSystemEntityClient(SystemEntityClient systemEntityClient) { - this.systemEntityClient = systemEntityClient; - this._updateIndicesService.setSystemEntityClient(systemEntityClient); - } - - @Override - public SystemEntityClient getSystemEntityClient() { - return this.systemEntityClient; - } - @Override public RecordTemplate getLatestAspect(@Nonnull Urn urn, @Nonnull String aspectName) { log.debug("Invoked getLatestAspect with urn {}, aspect {}", urn, aspectName); @@ -634,7 +625,7 @@ public List ingestAspects( .aspect(pair.getValue()) .systemMetadata(systemMetadata) .auditStamp(auditStamp) - .build(_entityRegistry, systemEntityClient)) + .build(this)) .collect(Collectors.toList()); return ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); } @@ -693,7 +684,7 @@ private List ingestAspectsToLocalDB( // 1. Convert patches to full upserts // 2. Run any entity/aspect level hooks Pair>, List> updatedItems = - aspectsBatch.toUpsertBatchItems(latestAspects, _entityRegistry, systemEntityClient); + aspectsBatch.toUpsertBatchItems(latestAspects, this); // Fetch additional information if needed final Map> updatedLatestAspects; @@ -725,8 +716,7 @@ private List ingestAspectsToLocalDB( previousAspect == null ? null : previousAspect.getRecordTemplate(_entityRegistry), - _entityRegistry, - systemEntityClient); + this); } catch (AspectValidationException e) { throw new RuntimeException(e); } @@ -934,7 +924,7 @@ public RecordTemplate ingestAspectIfNotPresent( .aspect(newValue) .systemMetadata(systemMetadata) .auditStamp(auditStamp) - .build(_entityRegistry, systemEntityClient)) + .build(this)) .build(); List ingested = ingestAspects(aspectsBatch, true, false); @@ -954,10 +944,7 @@ public RecordTemplate ingestAspectIfNotPresent( public IngestResult ingestProposal( MetadataChangeProposal proposal, AuditStamp auditStamp, final boolean async) { return ingestProposal( - AspectsBatchImpl.builder() - .mcps(List.of(proposal), auditStamp, getEntityRegistry(), systemEntityClient) - .build(), - async) + AspectsBatchImpl.builder().mcps(List.of(proposal), auditStamp, this).build(), async) .stream() .findFirst() .get(); @@ -1545,7 +1532,7 @@ protected Map getSnapshotRecords( @Nonnull protected Map> getLatestAspectUnions( @Nonnull final Set urns, @Nonnull final Set aspectNames) { - return getLatestAspects(urns, aspectNames).entrySet().stream() + return this.getLatestAspects(urns, aspectNames).entrySet().stream() .collect( Collectors.toMap( Map.Entry::getKey, @@ -1694,7 +1681,7 @@ private void ingestSnapshotUnion( .aspect(pair.getValue()) .auditStamp(auditStamp) .systemMetadata(systemMetadata) - .build(_entityRegistry, systemEntityClient)) + .build(this)) .collect(Collectors.toList())) .build(); @@ -1796,6 +1783,7 @@ private static Map> buildEntityToValidAspects( } @Override + @Nonnull public EntityRegistry getEntityRegistry() { return _entityRegistry; } @@ -2487,4 +2475,12 @@ private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspec aspectSpec.getRelationshipFieldSpecs(); return relationshipFieldSpecs.stream().anyMatch(RelationshipFieldSpec::isLineageRelationship); } + + @Nonnull + @Override + public Map> getLatestAspectObjects( + Set urns, Set aspectNames) throws RemoteInvocationException, URISyntaxException { + String entityName = urns.stream().findFirst().map(Urn::getEntityType).get(); + return entityResponseToAspectMap(getEntitiesV2(entityName, urns, aspectNames)); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java index 4d3ac9a550553..f353e5142755d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityUtils.java @@ -58,17 +58,11 @@ public static AuditStamp getAuditStamp(Urn actor) { public static void ingestChangeProposals( @Nonnull List changes, - @Nonnull EntityService entityService, + @Nonnull EntityService entityService, @Nonnull Urn actor, @Nonnull Boolean async) { entityService.ingestProposal( - AspectsBatchImpl.builder() - .mcps( - changes, - getAuditStamp(actor), - entityService.getEntityRegistry(), - entityService.getSystemEntityClient()) - .build(), + AspectsBatchImpl.builder().mcps(changes, getAuditStamp(actor), entityService).build(), async); } @@ -85,7 +79,7 @@ public static void ingestChangeProposals( public static RecordTemplate getAspectFromEntity( String entityUrn, String aspectName, - EntityService entityService, + EntityService entityService, RecordTemplate defaultValue) { Urn urn = getUrnFromString(entityUrn); if (urn == null) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java index f1b7d761087b4..4d9d2b3c416b7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraRetentionService.java @@ -17,12 +17,12 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.batch.AspectsBatch; +import com.linkedin.metadata.aspect.batch.UpsertItem; import com.linkedin.metadata.entity.EntityAspect; import com.linkedin.metadata.entity.EntityAspectIdentifier; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.RetentionService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.entity.retention.BulkApplyRetentionArgs; import com.linkedin.metadata.entity.retention.BulkApplyRetentionResult; import com.linkedin.mxe.MetadataChangeProposal; @@ -45,28 +45,22 @@ @Slf4j @RequiredArgsConstructor -public class CassandraRetentionService extends RetentionService { - private final EntityService _entityService; +public class CassandraRetentionService extends RetentionService { + private final EntityService _entityService; private final CqlSession _cqlSession; private final int _batchSize; private final Clock _clock = Clock.systemUTC(); @Override - public EntityService getEntityService() { + public EntityService getEntityService() { return _entityService; } @Override protected AspectsBatch buildAspectsBatch( List mcps, @Nonnull AuditStamp auditStamp) { - return AspectsBatchImpl.builder() - .mcps( - mcps, - auditStamp, - _entityService.getEntityRegistry(), - _entityService.getSystemEntityClient()) - .build(); + return AspectsBatchImpl.builder().mcps(mcps, auditStamp, _entityService).build(); } @Override diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java index d1f54f8a7e6e5..eba550714766b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanRetentionService.java @@ -5,10 +5,10 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.batch.AspectsBatch; +import com.linkedin.metadata.aspect.batch.UpsertItem; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.RetentionService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.entity.retention.BulkApplyRetentionArgs; import com.linkedin.metadata.entity.retention.BulkApplyRetentionResult; import com.linkedin.mxe.MetadataChangeProposal; @@ -40,28 +40,22 @@ @Slf4j @RequiredArgsConstructor -public class EbeanRetentionService extends RetentionService { - private final EntityService _entityService; +public class EbeanRetentionService extends RetentionService { + private final EntityService _entityService; private final Database _server; private final int _batchSize; private final Clock _clock = Clock.systemUTC(); @Override - public EntityService getEntityService() { + public EntityService getEntityService() { return _entityService; } @Override protected AspectsBatch buildAspectsBatch( List mcps, @Nonnull AuditStamp auditStamp) { - return AspectsBatchImpl.builder() - .mcps( - mcps, - auditStamp, - _entityService.getEntityRegistry(), - _entityService.getSystemEntityClient()) - .build(); + return AspectsBatchImpl.builder().mcps(mcps, auditStamp, _entityService).build(); } @Override diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java index 4b75fe73a12e5..80fb4e3e1b940 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java @@ -8,7 +8,6 @@ import com.linkedin.metadata.aspect.batch.SystemAspect; import com.linkedin.metadata.aspect.batch.UpsertItem; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; @@ -33,15 +32,12 @@ public class AspectsBatchImpl implements AspectsBatch { * Convert patches to upserts, apply hooks at the aspect and batch level. * * @param latestAspects latest version in the database - * @param entityRegistry entity registry * @return The new urn/aspectnames and the uniform upserts, possibly expanded/mutated by the * various hooks */ @Override public Pair>, List> toUpsertBatchItems( - final Map> latestAspects, - EntityRegistry entityRegistry, - AspectRetriever aspectRetriever) { + final Map> latestAspects, AspectRetriever aspectRetriever) { LinkedList upsertBatchItems = items.stream() @@ -59,25 +55,27 @@ public Pair>, List> toUpsertBatchItems( // patch to upsert MCPPatchBatchItem patchBatchItem = (MCPPatchBatchItem) item; final RecordTemplate currentValue = - latest != null ? latest.getRecordTemplate(entityRegistry) : null; - upsertItem = - patchBatchItem.applyPatch(entityRegistry, currentValue, aspectRetriever); + latest != null + ? latest.getRecordTemplate(aspectRetriever.getEntityRegistry()) + : null; + upsertItem = patchBatchItem.applyPatch(currentValue, aspectRetriever); } // Apply hooks final SystemMetadata oldSystemMetadata = latest != null ? latest.getSystemMetadata() : null; final RecordTemplate oldAspectValue = - latest != null ? latest.getRecordTemplate(entityRegistry) : null; - upsertItem.applyMutationHooks( - oldAspectValue, oldSystemMetadata, entityRegistry, aspectRetriever); + latest != null + ? latest.getRecordTemplate(aspectRetriever.getEntityRegistry()) + : null; + upsertItem.applyMutationHooks(oldAspectValue, oldSystemMetadata, aspectRetriever); return upsertItem; }) .collect(Collectors.toCollection(LinkedList::new)); LinkedList newItems = - applyMCPSideEffects(upsertBatchItems, entityRegistry, aspectRetriever) + applyMCPSideEffects(upsertBatchItems, aspectRetriever) .collect(Collectors.toCollection(LinkedList::new)); Map> newUrnAspectNames = getNewUrnAspectsMap(getUrnAspectsMap(), newItems); upsertBatchItems.addAll(newItems); @@ -98,20 +96,17 @@ public AspectsBatchImplBuilder one(BatchItem data) { } public AspectsBatchImplBuilder mcps( - List mcps, - AuditStamp auditStamp, - EntityRegistry entityRegistry, - AspectRetriever aspectRetriever) { + List mcps, AuditStamp auditStamp, AspectRetriever aspectRetriever) { this.items = mcps.stream() .map( mcp -> { if (mcp.getChangeType().equals(ChangeType.PATCH)) { return MCPPatchBatchItem.MCPPatchBatchItemBuilder.build( - mcp, auditStamp, entityRegistry); + mcp, auditStamp, aspectRetriever.getEntityRegistry()); } else { return MCPUpsertBatchItem.MCPUpsertBatchItemBuilder.build( - mcp, auditStamp, entityRegistry, aspectRetriever); + mcp, auditStamp, aspectRetriever); } }) .collect(Collectors.toList()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLBatchItemImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLBatchItemImpl.java index f61280bac4b22..6563765657d6d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLBatchItemImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLBatchItemImpl.java @@ -40,18 +40,24 @@ public class MCLBatchItemImpl implements MCLBatchItem { public static class MCLBatchItemImplBuilder { + // Ensure use of other builders + private MCLBatchItemImpl build() { + return null; + } + public MCLBatchItemImpl build( - MetadataChangeLog metadataChangeLog, - EntityRegistry entityRegistry, - AspectRetriever aspectRetriever) { - return MCLBatchItemImpl.builder() - .metadataChangeLog(metadataChangeLog) - .build(entityRegistry, aspectRetriever); + MetadataChangeLog metadataChangeLog, AspectRetriever aspectRetriever) { + return MCLBatchItemImpl.builder().metadataChangeLog(metadataChangeLog).build(aspectRetriever); } - public MCLBatchItemImpl build(EntityRegistry entityRegistry, AspectRetriever aspectRetriever) { + public MCLBatchItemImpl build(AspectRetriever aspectRetriever) { + EntityRegistry entityRegistry = aspectRetriever.getEntityRegistry(); + log.debug("entity type = {}", this.metadataChangeLog.getEntityType()); - entitySpec(entityRegistry.getEntitySpec(this.metadataChangeLog.getEntityType())); + entitySpec( + aspectRetriever + .getEntityRegistry() + .getEntitySpec(this.metadataChangeLog.getEntityType())); aspectSpec(validateAspect(this.metadataChangeLog, this.entitySpec)); Urn urn = this.metadataChangeLog.getEntityUrn(); @@ -75,7 +81,6 @@ public MCLBatchItemImpl build(EntityRegistry entityRegistry, AspectRetriever asp // validate new ValidationUtils.validateRecordTemplate( this.metadataChangeLog.getChangeType(), - entityRegistry, this.entitySpec, this.aspectSpec, urn, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPPatchBatchItem.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPPatchBatchItem.java index 3adf384f3b0ed..be333af2f7539 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPPatchBatchItem.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPPatchBatchItem.java @@ -16,13 +16,13 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.batch.PatchItem; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.entity.EntityUtils; import com.linkedin.metadata.entity.validation.ValidationUtils; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; import com.linkedin.mxe.MetadataChangeProposal; @@ -73,9 +73,7 @@ public ChangeType getChangeType() { } public MCPUpsertBatchItem applyPatch( - EntityRegistry entityRegistry, - RecordTemplate recordTemplate, - AspectRetriever aspectRetriever) { + RecordTemplate recordTemplate, AspectRetriever aspectRetriever) { MCPUpsertBatchItem.MCPUpsertBatchItemBuilder builder = MCPUpsertBatchItem.builder() .urn(getUrn()) @@ -84,7 +82,8 @@ public MCPUpsertBatchItem applyPatch( .auditStamp(auditStamp) .systemMetadata(getSystemMetadata()); - AspectTemplateEngine aspectTemplateEngine = entityRegistry.getAspectTemplateEngine(); + AspectTemplateEngine aspectTemplateEngine = + aspectRetriever.getEntityRegistry().getAspectTemplateEngine(); RecordTemplate currentValue = recordTemplate != null @@ -106,7 +105,7 @@ public MCPUpsertBatchItem applyPatch( throw new RuntimeException(e); } - return builder.build(entityRegistry, aspectRetriever); + return builder.build(aspectRetriever); } public static class MCPPatchBatchItemBuilder { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java index 9d41b141dcd60..89209c44f10c7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCPUpsertBatchItem.java @@ -4,12 +4,14 @@ import static com.linkedin.metadata.entity.AspectUtils.validateAspect; import com.datahub.util.exception.ModelConversionException; +import com.github.fge.jsonpatch.JsonPatchException; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.batch.SystemAspect; import com.linkedin.metadata.aspect.batch.UpsertItem; +import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate; import com.linkedin.metadata.aspect.plugins.hooks.MutationHook; import com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; @@ -19,12 +21,12 @@ import com.linkedin.metadata.entity.validation.ValidationUtils; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; +import java.io.IOException; import java.sql.Timestamp; import java.util.Objects; import javax.annotation.Nonnull; @@ -39,6 +41,31 @@ @Builder(toBuilder = true) public class MCPUpsertBatchItem extends UpsertItem { + public static MCPUpsertBatchItem fromPatch( + @Nonnull Urn urn, + @Nonnull AspectSpec aspectSpec, + @Nullable RecordTemplate recordTemplate, + GenericPatchTemplate genericPatchTemplate, + @Nonnull AuditStamp auditStamp, + AspectRetriever aspectRetriever) { + MCPUpsertBatchItem.MCPUpsertBatchItemBuilder builder = + MCPUpsertBatchItem.builder() + .urn(urn) + .auditStamp(auditStamp) + .aspectName(aspectSpec.getName()); + + RecordTemplate currentValue = + recordTemplate != null ? recordTemplate : genericPatchTemplate.getDefault(); + + try { + builder.aspect(genericPatchTemplate.applyPatch(currentValue)); + } catch (JsonPatchException | IOException e) { + throw new RuntimeException(e); + } + + return builder.build(aspectRetriever); + } + // urn an urn associated with the new aspect @Nonnull private final Urn urn; @@ -66,12 +93,12 @@ public ChangeType getChangeType() { public void applyMutationHooks( @Nullable RecordTemplate oldAspectValue, @Nullable SystemMetadata oldSystemMetadata, - @Nonnull EntityRegistry entityRegistry, @Nonnull AspectRetriever aspectRetriever) { // add audit stamp/system meta if needed for (MutationHook mutationHook : - entityRegistry.getMutationHooks( - getChangeType(), entitySpec.getName(), aspectSpec.getName())) { + aspectRetriever + .getEntityRegistry() + .getMutationHooks(getChangeType(), entitySpec.getName(), aspectSpec.getName())) { mutationHook.applyMutation( getChangeType(), entitySpec, @@ -99,14 +126,14 @@ public SystemAspect toLatestEntityAspect() { @Override public void validatePreCommit( - @Nullable RecordTemplate previous, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) + @Nullable RecordTemplate previous, @Nonnull AspectRetriever aspectRetriever) throws AspectValidationException { for (AspectPayloadValidator validator : - entityRegistry.getAspectPayloadValidators( - getChangeType(), entitySpec.getName(), aspectSpec.getName())) { + aspectRetriever + .getEntityRegistry() + .getAspectPayloadValidators( + getChangeType(), entitySpec.getName(), aspectSpec.getName())) { validator.validatePreCommit( getChangeType(), urn, getAspectSpec(), previous, this.aspect, aspectRetriever); } @@ -125,12 +152,11 @@ public MCPUpsertBatchItemBuilder systemMetadata(SystemMetadata systemMetadata) { } @SneakyThrows - public MCPUpsertBatchItem build( - EntityRegistry entityRegistry, AspectRetriever aspectRetriever) { - EntityUtils.validateUrn(entityRegistry, this.urn); + public MCPUpsertBatchItem build(AspectRetriever aspectRetriever) { + EntityUtils.validateUrn(aspectRetriever.getEntityRegistry(), this.urn); log.debug("entity type = {}", this.urn.getEntityType()); - entitySpec(entityRegistry.getEntitySpec(this.urn.getEntityType())); + entitySpec(aspectRetriever.getEntityRegistry().getEntitySpec(this.urn.getEntityType())); log.debug("entity spec = {}", this.entitySpec); aspectSpec(ValidationUtils.validate(this.entitySpec, this.aspectName)); @@ -138,7 +164,6 @@ public MCPUpsertBatchItem build( ValidationUtils.validateRecordTemplate( ChangeType.UPSERT, - entityRegistry, this.entitySpec, this.aspectSpec, this.urn, @@ -157,17 +182,15 @@ public MCPUpsertBatchItem build( } public static MCPUpsertBatchItem build( - MetadataChangeProposal mcp, - AuditStamp auditStamp, - EntityRegistry entityRegistry, - AspectRetriever aspectRetriever) { + MetadataChangeProposal mcp, AuditStamp auditStamp, AspectRetriever aspectRetriever) { if (!mcp.getChangeType().equals(ChangeType.UPSERT)) { throw new IllegalArgumentException( "Invalid MCP, this class only supports change type of UPSERT."); } log.debug("entity type = {}", mcp.getEntityType()); - EntitySpec entitySpec = entityRegistry.getEntitySpec(mcp.getEntityType()); + EntitySpec entitySpec = + aspectRetriever.getEntityRegistry().getEntitySpec(mcp.getEntityType()); AspectSpec aspectSpec = validateAspect(mcp, entitySpec); if (!isValidChangeType(ChangeType.UPSERT, aspectSpec)) { @@ -191,7 +214,7 @@ public static MCPUpsertBatchItem build( .metadataChangeProposal(mcp) .auditStamp(auditStamp) .aspect(convertToRecordTemplate(mcp, aspectSpec)) - .build(entityRegistry, aspectRetriever); + .build(aspectRetriever); } private MCPUpsertBatchItemBuilder entitySpec(EntitySpec entitySpec) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/validation/ValidationUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/validation/ValidationUtils.java index 97f7aa06340d2..947f0116b587c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/validation/ValidationUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/validation/ValidationUtils.java @@ -67,12 +67,12 @@ public static AspectSpec validate(EntitySpec entitySpec, String aspectName) { public static void validateRecordTemplate( ChangeType changeType, - EntityRegistry entityRegistry, EntitySpec entitySpec, AspectSpec aspectSpec, Urn urn, @Nullable RecordTemplate aspect, @Nonnull AspectRetriever aspectRetriever) { + EntityRegistry entityRegistry = aspectRetriever.getEntityRegistry(); EntityRegistryUrnValidator validator = new EntityRegistryUrnValidator(entityRegistry); validator.setCurrentEntitySpec(entitySpec); Consumer resultFunction = @@ -83,6 +83,7 @@ public static void validateRecordTemplate( + "\n Cause: " + validationResult.getMessages()); }; + RecordTemplateValidator.validate( EntityUtils.buildKeyAspect(entityRegistry, urn), resultFunction, validator); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java index 0d8b7655fddeb..24e272dee7a25 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/dgraph/DgraphGraphService.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.query.filter.Criterion; @@ -17,6 +18,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; import io.dgraph.DgraphClient; import io.dgraph.DgraphProto.Mutation; import io.dgraph.DgraphProto.NQuad; @@ -779,4 +781,21 @@ public void clear() { // setup urn, type and key relationships getSchema(); } + + @Nonnull + @Override + public RelatedEntitiesScrollResult scrollRelatedEntities( + @Nullable List sourceTypes, + @Nonnull Filter sourceEntityFilter, + @Nullable List destinationTypes, + @Nonnull Filter destinationEntityFilter, + @Nonnull List relationshipTypes, + @Nonnull RelationshipFilter relationshipFilter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { + throw new IllegalArgumentException("Not implemented"); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 97cb186ce948c..3051319aa54cf 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -23,6 +23,8 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchAfterWrapper; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.utils.ConcurrencyUtils; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; @@ -81,7 +83,7 @@ public class ESGraphQueryDAO { @Nonnull public static void addFilterToQueryBuilder( - @Nonnull Filter filter, String node, BoolQueryBuilder rootQuery) { + @Nonnull Filter filter, @Nullable String node, BoolQueryBuilder rootQuery) { BoolQueryBuilder orQuery = new BoolQueryBuilder(); for (ConjunctiveCriterion conjunction : filter.getOr()) { final BoolQueryBuilder andQuery = new BoolQueryBuilder(); @@ -93,12 +95,13 @@ public static void addFilterToQueryBuilder( } criterionArray.forEach( criterion -> - andQuery.must( + andQuery.filter( QueryBuilders.termQuery( - node + "." + criterion.getField(), criterion.getValue()))); + (node == null ? "" : node + ".") + criterion.getField(), + criterion.getValue()))); orQuery.should(andQuery); } - rootQuery.must(orQuery); + rootQuery.filter(orQuery); } private SearchResponse executeSearchQuery( @@ -174,9 +177,9 @@ public SearchResponse getSearchResponse( public static BoolQueryBuilder buildQuery( @Nullable final List sourceTypes, - @Nonnull final Filter sourceEntityFilter, + @Nullable final Filter sourceEntityFilter, @Nullable final List destinationTypes, - @Nonnull final Filter destinationEntityFilter, + @Nullable final Filter destinationEntityFilter, @Nonnull final List relationshipTypes, @Nonnull final RelationshipFilter relationshipFilter) { BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); @@ -187,17 +190,22 @@ public static BoolQueryBuilder buildQuery( String sourceNode = relationshipDirection == RelationshipDirection.OUTGOING ? SOURCE : DESTINATION; if (sourceTypes != null && sourceTypes.size() > 0) { - finalQuery.must(QueryBuilders.termsQuery(sourceNode + ".entityType", sourceTypes)); + finalQuery.filter(QueryBuilders.termsQuery(sourceNode + ".entityType", sourceTypes)); + } + if (sourceEntityFilter != null) { + addFilterToQueryBuilder(sourceEntityFilter, sourceNode, finalQuery); } - addFilterToQueryBuilder(sourceEntityFilter, sourceNode, finalQuery); // set destination filter String destinationNode = relationshipDirection == RelationshipDirection.OUTGOING ? DESTINATION : SOURCE; if (destinationTypes != null && destinationTypes.size() > 0) { - finalQuery.must(QueryBuilders.termsQuery(destinationNode + ".entityType", destinationTypes)); + finalQuery.filter( + QueryBuilders.termsQuery(destinationNode + ".entityType", destinationTypes)); + } + if (destinationEntityFilter != null) { + addFilterToQueryBuilder(destinationEntityFilter, destinationNode, finalQuery); } - addFilterToQueryBuilder(destinationEntityFilter, destinationNode, finalQuery); // set relationship filter if (relationshipTypes.size() > 0) { @@ -206,8 +214,14 @@ public static BoolQueryBuilder buildQuery( relationshipType -> relationshipQuery.should( QueryBuilders.termQuery(RELATIONSHIP_TYPE, relationshipType))); - finalQuery.must(relationshipQuery); + finalQuery.filter(relationshipQuery); + } + + // general filter + if (relationshipFilter.getOr() != null) { + addFilterToQueryBuilder(new Filter().setOr(relationshipFilter.getOr()), null, finalQuery); } + return finalQuery; } @@ -659,4 +673,60 @@ public static class LineageResponse { int total; List lineageRelationships; } + + public SearchResponse getSearchResponse( + @Nullable final List sourceTypes, + @Nullable final Filter sourceEntityFilter, + @Nullable final List destinationTypes, + @Nullable final Filter destinationEntityFilter, + @Nonnull final List relationshipTypes, + @Nonnull final RelationshipFilter relationshipFilter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count) { + + BoolQueryBuilder finalQuery = + buildQuery( + sourceTypes, + sourceEntityFilter, + destinationTypes, + destinationEntityFilter, + relationshipTypes, + relationshipFilter); + + return executeScrollSearchQuery(finalQuery, sortCriterion, scrollId, count); + } + + private SearchResponse executeScrollSearchQuery( + @Nonnull final QueryBuilder query, + @Nonnull List sortCriterion, + @Nullable String scrollId, + final int count) { + + Object[] sort = null; + if (scrollId != null) { + SearchAfterWrapper searchAfterWrapper = SearchAfterWrapper.fromScrollId(scrollId); + sort = searchAfterWrapper.getSort(); + } + + SearchRequest searchRequest = new SearchRequest(); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + searchSourceBuilder.size(count); + searchSourceBuilder.query(query); + ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, List.of(), false); + searchRequest.source(searchSourceBuilder); + ESUtils.setSearchAfter(searchSourceBuilder, sort, null, null); + + searchRequest.indices(indexConvention.getIndexName(INDEX_NAME)); + + try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "esQuery").time()) { + MetricUtils.counter(this.getClass(), SEARCH_EXECUTIONS_METRIC).inc(); + return client.search(searchRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + log.error("Search query failed", e); + throw new ESQueryException("Search query failed:", e); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index 6c828c0e7c6ae..67590ffd6e7c1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -11,7 +11,9 @@ import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.graph.LineageRelationshipArray; +import com.linkedin.metadata.graph.RelatedEntities; import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.query.filter.Condition; @@ -22,11 +24,14 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchAfterWrapper; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import com.linkedin.structured.StructuredPropertyDefinition; import io.opentelemetry.extension.annotations.WithSpan; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -35,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -47,6 +53,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.SearchHit; @Slf4j @RequiredArgsConstructor @@ -165,8 +172,6 @@ public RelatedEntitiesResult findRelatedEntities( } final RelationshipDirection relationshipDirection = relationshipFilter.getDirection(); - String destinationNode = - relationshipDirection == RelationshipDirection.OUTGOING ? "destination" : "source"; SearchResponse response = _graphReadDAO.getSearchResponse( @@ -185,28 +190,8 @@ public RelatedEntitiesResult findRelatedEntities( int totalCount = (int) response.getHits().getTotalHits().value; final List relationships = - Arrays.stream(response.getHits().getHits()) - .map( - hit -> { - final String urnStr = - ((HashMap) - hit.getSourceAsMap().getOrDefault(destinationNode, EMPTY_HASH)) - .getOrDefault("urn", null); - final String relationshipType = - (String) hit.getSourceAsMap().get("relationshipType"); - - if (urnStr == null || relationshipType == null) { - log.error( - String.format( - "Found null urn string, relationship type, aspect name or path spec in Elastic index. " - + "urnStr: %s, relationshipType: %s", - urnStr, relationshipType)); - return null; - } - - return new RelatedEntity(relationshipType, urnStr); - }) - .filter(Objects::nonNull) + searchHitsToRelatedEntities(response.getHits().getHits(), relationshipDirection).stream() + .map(RelatedEntities::asRelatedEntity) .collect(Collectors.toList()); return new RelatedEntitiesResult(offset, relationships.size(), totalCount, relationships); @@ -328,6 +313,12 @@ public List buildReindexConfigs() throws IOException { Collections.emptyMap())); } + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException { + return buildReindexConfigs(); + } + @Override public void reindexAll() { configure(); @@ -344,4 +335,88 @@ public void clear() { public boolean supportsMultiHop() { return true; } + + @Nonnull + @Override + public RelatedEntitiesScrollResult scrollRelatedEntities( + @Nullable List sourceTypes, + @Nullable Filter sourceEntityFilter, + @Nullable List destinationTypes, + @Nullable Filter destinationEntityFilter, + @Nonnull List relationshipTypes, + @Nonnull RelationshipFilter relationshipFilter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { + + final RelationshipDirection relationshipDirection = relationshipFilter.getDirection(); + + SearchResponse response = + _graphReadDAO.getSearchResponse( + sourceTypes, + sourceEntityFilter, + destinationTypes, + destinationEntityFilter, + relationshipTypes, + relationshipFilter, + sortCriterion, + scrollId, + count); + + if (response == null) { + return new RelatedEntitiesScrollResult(0, 0, null, ImmutableList.of()); + } + + int totalCount = (int) response.getHits().getTotalHits().value; + final List relationships = + searchHitsToRelatedEntities(response.getHits().getHits(), relationshipDirection); + + SearchHit[] searchHits = response.getHits().getHits(); + // Only return next scroll ID if there are more results, indicated by full size results + String nextScrollId = null; + if (searchHits.length == count) { + Object[] sort = searchHits[searchHits.length - 1].getSortValues(); + nextScrollId = new SearchAfterWrapper(sort, null, 0L).toScrollId(); + } + + return RelatedEntitiesScrollResult.builder() + .entities(relationships) + .pageSize(relationships.size()) + .numResults(totalCount) + .scrollId(nextScrollId) + .build(); + } + + private static List searchHitsToRelatedEntities( + SearchHit[] searchHits, RelationshipDirection relationshipDirection) { + return Arrays.stream(searchHits) + .map( + hit -> { + final String destinationUrnStr = + ((HashMap) + hit.getSourceAsMap().getOrDefault("destination", EMPTY_HASH)) + .getOrDefault("urn", null); + final String sourceUrnStr = + ((HashMap) + hit.getSourceAsMap().getOrDefault("source", EMPTY_HASH)) + .getOrDefault("urn", null); + final String relationshipType = (String) hit.getSourceAsMap().get("relationshipType"); + + if (destinationUrnStr == null || sourceUrnStr == null || relationshipType == null) { + log.error( + String.format( + "Found null urn string, relationship type, aspect name or path spec in Elastic index. " + + "destinationUrnStr: %s, sourceUrnStr: %s, relationshipType: %s", + destinationUrnStr, sourceUrnStr, relationshipType)); + return null; + } + + return new RelatedEntities( + relationshipType, sourceUrnStr, destinationUrnStr, relationshipDirection); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java index c8d3147711eba..a1f73a134ec8e 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java @@ -17,6 +17,7 @@ import com.linkedin.metadata.graph.LineageRelationship; import com.linkedin.metadata.graph.LineageRelationshipArray; import com.linkedin.metadata.graph.RelatedEntitiesResult; +import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.query.filter.Condition; @@ -25,6 +26,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.util.Pair; import io.opentelemetry.extension.annotations.WithSpan; @@ -882,4 +884,21 @@ private boolean isSourceDestReversed( return null; } } + + @Nonnull + @Override + public RelatedEntitiesScrollResult scrollRelatedEntities( + @Nullable List sourceTypes, + @Nonnull Filter sourceEntityFilter, + @Nullable List destinationTypes, + @Nonnull Filter destinationEntityFilter, + @Nonnull List relationshipTypes, + @Nonnull RelationshipFilter relationshipFilter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { + throw new IllegalArgumentException("Not implemented"); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index fd7491fe32ea3..7cba2e0ecc8cb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -18,6 +18,9 @@ import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.search.utils.SearchUtils; import com.linkedin.metadata.shared.ElasticSearchIndexed; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -47,6 +50,12 @@ public List buildReindexConfigs() { return indexBuilders.buildReindexConfigs(); } + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException { + return indexBuilders.buildReindexConfigsWithAllStructProps(properties); + } + @Override public void reindexAll() { configure(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java index 388dcea784cbb..cc6a0f3e3d6f9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ESIndexBuilder.java @@ -1,5 +1,8 @@ package com.linkedin.metadata.search.elasticsearch.indexbuilder; +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder.PROPERTIES; + import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.search.utils.ESUtils; @@ -22,6 +25,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -125,12 +129,20 @@ public ESIndexBuilder( public ReindexConfig buildReindexState( String indexName, Map mappings, Map settings) throws IOException { + return buildReindexState(indexName, mappings, settings, false); + } + + public ReindexConfig buildReindexState( + String indexName, + Map mappings, + Map settings, + boolean copyStructuredPropertyMappings) + throws IOException { ReindexConfig.ReindexConfigBuilder builder = ReindexConfig.builder() .name(indexName) .enableIndexSettingsReindex(enableIndexSettingsReindex) .enableIndexMappingsReindex(enableIndexMappingsReindex) - .targetMappings(mappings) .version(gitVersion.getVersion()); Map baseSettings = new HashMap<>(settings); @@ -148,6 +160,7 @@ public ReindexConfig buildReindexState( // If index doesn't exist, no reindex if (!exists) { + builder.targetMappings(mappings); return builder.build(); } @@ -173,6 +186,35 @@ public ReindexConfig buildReindexState( .getSourceAsMap(); builder.currentMappings(currentMappings); + if (copyStructuredPropertyMappings) { + Map currentStructuredProperties = + (Map) + ((Map) + ((Map) + currentMappings.getOrDefault(PROPERTIES, new TreeMap())) + .getOrDefault(STRUCTURED_PROPERTY_MAPPING_FIELD, new TreeMap())) + .getOrDefault(PROPERTIES, new TreeMap()); + + if (!currentStructuredProperties.isEmpty()) { + HashMap> props = + (HashMap>) + ((Map) mappings.get(PROPERTIES)) + .computeIfAbsent( + STRUCTURED_PROPERTY_MAPPING_FIELD, + (key) -> new HashMap<>(Map.of(PROPERTIES, new HashMap<>()))); + + props.merge( + PROPERTIES, + currentStructuredProperties, + (targetValue, currentValue) -> { + HashMap merged = new HashMap<>(currentValue); + merged.putAll(targetValue); + return merged.isEmpty() ? null : merged; + }); + } + } + + builder.targetMappings(mappings); return builder.build(); } @@ -251,7 +293,7 @@ public void buildIndex(ReindexConfig indexState) throws IOException { * @throws IOException communication issues with ES */ public void applyMappings(ReindexConfig indexState, boolean suppressError) throws IOException { - if (indexState.isPureMappingsAddition()) { + if (indexState.isPureMappingsAddition() || indexState.isPureStructuredProperty()) { log.info("Updating index {} mappings in place.", indexState.name()); PutMappingRequest request = new PutMappingRequest(indexState.name()).source(indexState.targetMappings()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java index 4489c661bb2ed..4322ea90edf1f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/EntityIndexBuilders.java @@ -3,9 +3,12 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import com.linkedin.structured.StructuredPropertyDefinition; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -41,6 +44,24 @@ public List buildReindexConfigs() { entitySpec -> { try { Map mappings = MappingsBuilder.getMappings(entitySpec); + return indexBuilder.buildReindexState( + indexConvention.getIndexName(entitySpec), mappings, settings, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) { + Map settings = settingsBuilder.getSettings(); + return entityRegistry.getEntitySpecs().values().stream() + .map( + entitySpec -> { + try { + Map mappings = MappingsBuilder.getMappings(entitySpec, properties); return indexBuilder.buildReindexState( indexConvention.getIndexName(entitySpec), mappings, settings); } catch (IOException e) { @@ -49,4 +70,31 @@ public List buildReindexConfigs() { }) .collect(Collectors.toList()); } + + /** + * Given a structured property generate all entity index configurations impacted by it, preserving + * existing properties + * + * @param property the new property + * @return index configurations impacted by the new property + */ + public List buildReindexConfigsWithNewStructProp( + StructuredPropertyDefinition property) { + Map settings = settingsBuilder.getSettings(); + return entityRegistry.getEntitySpecs().values().stream() + .map( + entitySpec -> { + try { + Map mappings = + MappingsBuilder.getMappings(entitySpec, List.of(property)); + return indexBuilder.buildReindexState( + indexConvention.getIndexName(entitySpec), mappings, settings, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .filter(Objects::nonNull) + .filter(ReindexConfig::hasNewStructuredProperty) + .collect(Collectors.toList()); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index f85a0dcb06a07..79f530f18a345 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -1,13 +1,21 @@ package com.linkedin.metadata.search.elasticsearch.indexbuilder; +import static com.linkedin.metadata.Constants.ENTITY_TYPE_URN_PREFIX; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD; +import static com.linkedin.metadata.models.StructuredPropertyUtils.sanitizeStructuredPropertyFQN; import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; import com.google.common.collect.ImmutableMap; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.LogicalValueType; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.net.URISyntaxException; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,6 +56,53 @@ public static Map getPartialNgramConfigWithOverrides( private MappingsBuilder() {} + /** + * Builds mappings from entity spec and a collection of structured properties for the entity. + * + * @param entitySpec entity's spec + * @param structuredProperties structured properties for the entity + * @return mappings + */ + public static Map getMappings( + @Nonnull final EntitySpec entitySpec, + Collection structuredProperties) { + Map mappings = getMappings(entitySpec); + + String entityName = entitySpec.getEntityAnnotation().getName(); + Map structuredPropertiesForEntity = + getMappingsForStructuredProperty( + structuredProperties.stream() + .filter( + prop -> { + try { + return prop.getEntityTypes() + .contains(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + entityName)); + } catch (URISyntaxException e) { + return false; + } + }) + .collect(Collectors.toSet())); + + if (!structuredPropertiesForEntity.isEmpty()) { + HashMap> props = + (HashMap>) + ((Map) mappings.get(PROPERTIES)) + .computeIfAbsent( + STRUCTURED_PROPERTY_MAPPING_FIELD, + (key) -> new HashMap<>(Map.of(PROPERTIES, new HashMap<>()))); + + props.merge( + PROPERTIES, + structuredPropertiesForEntity, + (oldValue, newValue) -> { + HashMap merged = new HashMap<>(oldValue); + merged.putAll(newValue); + return merged.isEmpty() ? null : merged; + }); + } + return mappings; + } + public static Map getMappings(@Nonnull final EntitySpec entitySpec) { Map mappings = new HashMap<>(); @@ -89,6 +144,30 @@ private static Map getMappingsForRunId() { return ImmutableMap.builder().put(TYPE, ESUtils.KEYWORD_FIELD_TYPE).build(); } + public static Map getMappingsForStructuredProperty( + Collection properties) { + return properties.stream() + .map( + property -> { + Map mappingForField = new HashMap<>(); + String valueType = property.getValueType().getId(); + if (valueType.equalsIgnoreCase(LogicalValueType.STRING.name())) { + mappingForField = getMappingsForKeyword(); + } else if (valueType.equalsIgnoreCase(LogicalValueType.RICH_TEXT.name())) { + mappingForField = getMappingsForSearchText(FieldType.TEXT_PARTIAL); + } else if (valueType.equalsIgnoreCase(LogicalValueType.DATE.name())) { + mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE); + } else if (valueType.equalsIgnoreCase(LogicalValueType.URN.name())) { + mappingForField = getMappingsForUrn(); + } else if (valueType.equalsIgnoreCase(LogicalValueType.NUMBER.name())) { + mappingForField.put(TYPE, ESUtils.DOUBLE_FIELD_TYPE); + } + return Map.entry( + sanitizeStructuredPropertyFQN(property.getQualifiedName()), mappingForField); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + private static Map getMappingsForField( @Nonnull final SearchableFieldSpec searchableFieldSpec) { FieldType fieldType = searchableFieldSpec.getSearchableAnnotation().getFieldType(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java index e3155c9f943cc..bb6905139f49d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/ReindexConfig.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,6 +66,8 @@ public class ReindexConfig { private final boolean requiresApplyMappings; private final boolean isPureMappingsAddition; private final boolean isSettingsReindex; + private final boolean hasNewStructuredProperty; + private final boolean isPureStructuredProperty; public static ReindexConfigBuilder builder() { return new CalculatedBuilder(); @@ -92,6 +95,14 @@ private ReindexConfigBuilder isSettingsReindexRequired(boolean ignored) { return this; } + private ReindexConfigBuilder hasNewStructuredProperty(boolean ignored) { + return this; + } + + private ReindexConfigBuilder isPureStructuredProperty(boolean ignored) { + return this; + } + // ensure sorted public ReindexConfigBuilder currentMappings(Map currentMappings) { this.currentMappings = sortMap(currentMappings); @@ -141,6 +152,15 @@ public ReindexConfig build() { super.requiresApplyMappings = !mappingsDiff.entriesDiffering().isEmpty() || !mappingsDiff.entriesOnlyOnRight().isEmpty(); + super.isPureStructuredProperty = + mappingsDiff + .entriesDiffering() + .keySet() + .equals(Set.of(STRUCTURED_PROPERTY_MAPPING_FIELD)) + || mappingsDiff + .entriesOnlyOnRight() + .keySet() + .equals(Set.of(STRUCTURED_PROPERTY_MAPPING_FIELD)); super.isPureMappingsAddition = super.requiresApplyMappings && mappingsDiff.entriesDiffering().isEmpty() @@ -157,6 +177,19 @@ public ReindexConfig build() { super.name, mappingsDiff.entriesDiffering()); } + super.hasNewStructuredProperty = + (mappingsDiff.entriesDiffering().containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD) + || mappingsDiff + .entriesOnlyOnRight() + .containsKey(STRUCTURED_PROPERTY_MAPPING_FIELD)) + && getOrDefault( + super.currentMappings, + List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) + .size() + < getOrDefault( + super.targetMappings, + List.of("properties", STRUCTURED_PROPERTY_MAPPING_FIELD, "properties")) + .size(); /* Consider analysis and settings changes */ super.requiresApplySettings = !isSettingsEqual() || !isAnalysisEqual(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index b35c0258d09f0..0eb44edfb11de 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -1,7 +1,7 @@ package com.linkedin.metadata.search.elasticsearch.query; import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; +import static com.linkedin.metadata.aspect.patch.template.TemplateUtil.*; import static com.linkedin.metadata.utils.SearchUtil.*; import com.codahale.metrics.Timer; @@ -303,7 +303,7 @@ public AutoCompleteResult autoComplete( /** * Returns number of documents per field value given the field and filters * - * @param entityName name of the entity, if null, aggregates over all entities + * @param entityNames names of the entities, if null, aggregates over all entities * @param field the field name for aggregate * @param requestParams filters to apply before aggregating * @param limit the number of aggregations to return diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 522c8e510dcf8..bdc0332b040df 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -1,8 +1,10 @@ package com.linkedin.metadata.search.elasticsearch.query.request; +import static com.linkedin.metadata.Constants.*; import static com.linkedin.metadata.utils.SearchUtil.*; import com.linkedin.metadata.config.search.SearchConfiguration; +import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.search.utils.ESUtils; import java.util.ArrayList; @@ -72,8 +74,12 @@ private Set getAllFacetFields(final List annotatio } private boolean isValidAggregate(final String inputFacet) { - Set facets = Set.of(inputFacet.split(AGGREGATION_SEPARATOR_CHAR)); - boolean isValid = !facets.isEmpty() && _allFacetFields.containsAll(facets); + List facets = List.of(inputFacet.split(AGGREGATION_SEPARATOR_CHAR)); + boolean isValid = + !facets.isEmpty() + && ((facets.size() == 1 + && facets.get(0).startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) + || _allFacetFields.containsAll(facets)); if (!isValid) { log.warn( String.format( @@ -89,17 +95,27 @@ private AggregationBuilder facetToAggregationBuilder(final String inputFacet) { AggregationBuilder lastAggBuilder = null; for (int i = facets.size() - 1; i >= 0; i--) { String facet = facets.get(i); + if (facet.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) { + String structPropFqn = facet.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1); + facet = + STRUCTURED_PROPERTY_MAPPING_FIELD + + "." + + StructuredPropertyUtils.sanitizeStructuredPropertyFQN(structPropFqn); + } AggregationBuilder aggBuilder; if (facet.contains(AGGREGATION_SPECIAL_TYPE_DELIMITER)) { List specialTypeFields = List.of(facet.split(AGGREGATION_SPECIAL_TYPE_DELIMITER)); switch (specialTypeFields.get(0)) { - case MISSING_SPECIAL_TYPE -> aggBuilder = - INDEX_VIRTUAL_FIELD.equalsIgnoreCase(specialTypeFields.get(1)) - ? AggregationBuilders.missing(inputFacet).field(getAggregationField("_index")) - : AggregationBuilders.missing(inputFacet) - .field(getAggregationField(specialTypeFields.get(1))); - default -> throw new UnsupportedOperationException( - "Unknown special type: " + specialTypeFields.get(0)); + case MISSING_SPECIAL_TYPE: + aggBuilder = + INDEX_VIRTUAL_FIELD.equalsIgnoreCase(specialTypeFields.get(1)) + ? AggregationBuilders.missing(inputFacet).field(getAggregationField("_index")) + : AggregationBuilders.missing(inputFacet) + .field(getAggregationField(specialTypeFields.get(1))); + break; + default: + throw new UnsupportedOperationException( + "Unknown special type: " + specialTypeFields.get(0)); } } else { aggBuilder = diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchAfterWrapper.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchAfterWrapper.java index 1fe4a74968e42..452e50a6e8d62 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchAfterWrapper.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchAfterWrapper.java @@ -1,6 +1,6 @@ package com.linkedin.metadata.search.elasticsearch.query.request; -import static com.linkedin.metadata.models.registry.template.util.TemplateUtil.*; +import static com.linkedin.metadata.aspect.patch.template.TemplateUtil.*; import java.io.IOException; import java.io.Serializable; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java index 4d51de39c88e3..c5a5ade216bf7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java @@ -93,6 +93,7 @@ public class SearchRequestHandler { private final Set _defaultQueryFieldNames; private final HighlightBuilder _highlights; private final Map _filtersToDisplayName; + private final SearchConfiguration _configs; private final SearchQueryBuilder _searchQueryBuilder; private final AggregationQueryBuilder _aggregationQueryBuilder; @@ -647,10 +648,12 @@ public static Map extractAggregationsFromResponse( if (aggregation == null) { return Collections.emptyMap(); } - if (aggregation instanceof ParsedTerms terms) { - return extractTermAggregations(terms, aggregationName.equals("_entityType")); - } else if (aggregation instanceof ParsedMissing missing) { - return Collections.singletonMap(missing.getName(), missing.getDocCount()); + if (aggregation instanceof ParsedTerms) { + return extractTermAggregations( + (ParsedTerms) aggregation, aggregationName.equals("_entityType")); + } else if (aggregation instanceof ParsedMissing) { + return Collections.singletonMap( + aggregation.getName(), ((ParsedMissing) aggregation).getDocCount()); } throw new UnsupportedOperationException( "Unsupported aggregation type: " + aggregation.getClass().getName()); @@ -668,10 +671,10 @@ private static Map recursivelyAddNestedSubAggs(@Nullable Aggregati if (aggs != null) { for (Map.Entry entry : aggs.getAsMap().entrySet()) { - if (entry.getValue() instanceof ParsedTerms terms) { - recurseTermsAgg(terms, aggResult, false); - } else if (entry.getValue() instanceof ParsedMissing missing) { - recurseMissingAgg(missing, aggResult); + if (entry.getValue() instanceof ParsedTerms) { + recurseTermsAgg((ParsedTerms) entry.getValue(), aggResult, false); + } else if (entry.getValue() instanceof ParsedMissing) { + recurseMissingAgg((ParsedMissing) entry.getValue(), aggResult); } else { throw new UnsupportedOperationException( "Unsupported aggregation type: " + entry.getValue().getClass().getName()); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/features/Features.java b/metadata-io/src/main/java/com/linkedin/metadata/search/features/Features.java index 2a9571b18b726..6cadb39d5970d 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/features/Features.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/features/Features.java @@ -17,7 +17,8 @@ public class Features { public enum Name { SEARCH_BACKEND_SCORE, // Score returned by search backend NUM_ENTITIES_PER_TYPE, // Number of entities per entity type - RANK_WITHIN_TYPE; // Rank within the entity type + RANK_WITHIN_TYPE, + ONLY_MATCH_CUSTOM_PROPERTIES; // Rank within the entity type } public Double getNumericFeature(Name featureName, double defaultValue) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index bfeb993390571..d52a80d685fd5 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -1,5 +1,8 @@ package com.linkedin.metadata.search.transformer; +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.models.StructuredPropertyUtils.sanitizeStructuredPropertyFQN; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; @@ -7,17 +10,26 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.schema.DataSchema; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.entity.Aspect; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; +import com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.LogicalValueType; import com.linkedin.metadata.models.SearchScoreFieldSpec; import com.linkedin.metadata.models.SearchableFieldSpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; import com.linkedin.metadata.models.extractor.FieldExtractor; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; @@ -41,7 +53,7 @@ public class SearchDocumentTransformer { // Maximum customProperties value length private final int maxValueLength; - private SystemEntityClient entityClient; + private AspectRetriever aspectRetriever; private static final String BROWSE_PATH_V2_DELIMITER = "␟"; @@ -77,7 +89,8 @@ public Optional transformAspect( final Urn urn, final RecordTemplate aspect, final AspectSpec aspectSpec, - final Boolean forDelete) { + final Boolean forDelete) + throws RemoteInvocationException, URISyntaxException { final Map> extractedSearchableFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs(), maxValueLength); final Map> extractedSearchScoreFields = @@ -93,6 +106,12 @@ public Optional transformAspect( extractedSearchScoreFields.forEach( (key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); result = Optional.of(searchDocument.toString()); + } else if (STRUCTURED_PROPERTIES_ASPECT_NAME.equals(aspectSpec.getName())) { + final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); + searchDocument.put("urn", urn.toString()); + setStructuredPropertiesSearchValue( + new StructuredProperties(aspect.data()), searchDocument, forDelete); + result = Optional.of(searchDocument.toString()); } return result; @@ -277,4 +296,93 @@ private String getBrowsePathV2Value(@Nonnull final List fieldValues) { } return aggregatedValue; } + + private void setStructuredPropertiesSearchValue( + final StructuredProperties values, final ObjectNode searchDocument, final Boolean forDelete) + throws RemoteInvocationException, URISyntaxException { + Map> propertyMap = + values.getProperties().stream() + .collect( + Collectors.groupingBy( + StructuredPropertyValueAssignment::getPropertyUrn, Collectors.toSet())); + + Map> definitions = + aspectRetriever.getLatestAspectObjects( + propertyMap.keySet(), Set.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)); + + if (definitions.size() < propertyMap.size()) { + String message = + String.format( + "Missing property definitions. %s", + propertyMap.keySet().stream() + .filter(k -> !definitions.containsKey(k)) + .collect(Collectors.toSet())); + log.error(message); + } + + propertyMap + .entrySet() + .forEach( + propertyEntry -> { + StructuredPropertyDefinition definition = + new StructuredPropertyDefinition( + definitions + .get(propertyEntry.getKey()) + .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .data()); + String fieldName = + String.join( + ".", + List.of( + STRUCTURED_PROPERTY_MAPPING_FIELD, + sanitizeStructuredPropertyFQN(definition.getQualifiedName()))); + + if (forDelete) { + searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode()); + } else { + LogicalValueType logicalValueType = + StructuredPropertiesValidator.getLogicalValueType(definition.getValueType()); + + ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); + + propertyEntry + .getValue() + .forEach( + property -> + property + .getValues() + .forEach( + propertyValue -> { + final Optional searchValue; + switch (logicalValueType) { + case UNKNOWN: + log.warn( + "Unable to transform UNKNOWN logical value type."); + searchValue = Optional.empty(); + break; + case NUMBER: + Double doubleValue = + propertyValue.getDouble() != null + ? propertyValue.getDouble() + : Double.valueOf(propertyValue.getString()); + searchValue = + Optional.of( + JsonNodeFactory.instance.numberNode(doubleValue)); + break; + default: + searchValue = + propertyValue.getString().isEmpty() + ? Optional.empty() + : Optional.of( + JsonNodeFactory.instance.textNode( + propertyValue.getString())); + break; + } + searchValue.ifPresent(arrayNode::add); + })); + + searchDocument.set(fieldName, arrayNode); + } + }); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 982b5c8d5f367..aa854149de43a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.search.utils; +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.*; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.KEYWORD_FIELDS; import static com.linkedin.metadata.search.elasticsearch.query.request.SearchFieldConfig.PATH_HIERARCHY_FIELDS; import static com.linkedin.metadata.search.utils.SearchUtils.isUrn; @@ -8,6 +10,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.SearchableFieldSpec; +import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.models.annotation.SearchableAnnotation; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; @@ -97,6 +100,7 @@ public class ESUtils { } }; + // TODO - This has been expanded for has* in another branch public static final Set BOOLEAN_FIELDS = ImmutableSet.of("removed"); /* @@ -203,6 +207,9 @@ public static BoolQueryBuilder buildConjunctiveFilterQuery( public static QueryBuilder getQueryBuilderFromCriterion( @Nonnull final Criterion criterion, boolean isTimeseries) { final String fieldName = toFacetField(criterion.getField()); + if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD)) { + criterion.setField(fieldName); + } /* * Check the field-name for a "sibling" field, or one which should ALWAYS @@ -260,46 +267,69 @@ public static void buildSortOrder( @Nonnull SearchSourceBuilder searchSourceBuilder, @Nullable SortCriterion sortCriterion, List entitySpecs) { - if (sortCriterion == null) { + buildSortOrder( + searchSourceBuilder, + sortCriterion == null ? List.of() : List.of(sortCriterion), + entitySpecs, + true); + } + + /** + * Allow disabling default sort, used when you know uniqueness is present without urn field. For + * example, edge indices where the unique constraint is determined by multiple fields (src urn, + * dst urn, relation type). + * + * @param enableDefaultSort enable/disable default sorting logic + */ + public static void buildSortOrder( + @Nonnull SearchSourceBuilder searchSourceBuilder, + @Nonnull List sortCriterion, + List entitySpecs, + boolean enableDefaultSort) { + if (sortCriterion.isEmpty() && enableDefaultSort) { searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); } else { - Optional fieldTypeForDefault = Optional.empty(); - for (EntitySpec entitySpec : entitySpecs) { - List fieldSpecs = entitySpec.getSearchableFieldSpecs(); - for (SearchableFieldSpec fieldSpec : fieldSpecs) { - SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation(); - if (annotation.getFieldName().equals(sortCriterion.getField()) - || annotation.getFieldNameAliases().contains(sortCriterion.getField())) { - fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType()); + for (SortCriterion sortCriteria : sortCriterion) { + Optional fieldTypeForDefault = Optional.empty(); + for (EntitySpec entitySpec : entitySpecs) { + List fieldSpecs = entitySpec.getSearchableFieldSpecs(); + for (SearchableFieldSpec fieldSpec : fieldSpecs) { + SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation(); + if (annotation.getFieldName().equals(sortCriteria.getField()) + || annotation.getFieldNameAliases().contains(sortCriteria.getField())) { + fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType()); + break; + } + } + if (fieldTypeForDefault.isPresent()) { break; } } - if (fieldTypeForDefault.isPresent()) { - break; + if (fieldTypeForDefault.isEmpty()) { + log.warn( + "Sort criterion field " + + sortCriteria.getField() + + " was not found in any entity spec to be searched"); } - } - if (fieldTypeForDefault.isEmpty()) { - log.warn( - "Sort criterion field " - + sortCriterion.getField() - + " was not found in any entity spec to be searched"); - } - final SortOrder esSortOrder = - (sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) - ? SortOrder.ASC - : SortOrder.DESC; - FieldSortBuilder sortBuilder = - new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder); - if (fieldTypeForDefault.isPresent()) { - String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get()); - if (esFieldtype != null) { - sortBuilder.unmappedType(esFieldtype); + final SortOrder esSortOrder = + (sortCriteria.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) + ? SortOrder.ASC + : SortOrder.DESC; + FieldSortBuilder sortBuilder = + new FieldSortBuilder(sortCriteria.getField()).order(esSortOrder); + if (fieldTypeForDefault.isPresent()) { + String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get()); + if (esFieldtype != null) { + sortBuilder.unmappedType(esFieldtype); + } } + searchSourceBuilder.sort(sortBuilder); } - searchSourceBuilder.sort(sortBuilder); } - if (sortCriterion == null - || !sortCriterion.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)) { + if (enableDefaultSort + && (sortCriterion.isEmpty() + || sortCriterion.stream() + .noneMatch(c -> c.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)))) { searchSourceBuilder.sort( new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC)); } @@ -335,7 +365,15 @@ public static String escapeReservedCharacters(@Nonnull String input) { @Nonnull public static String toFacetField(@Nonnull final String filterField) { - return filterField.replace(ESUtils.KEYWORD_SUFFIX, ""); + String fieldName = filterField; + if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD + ".")) { + fieldName = + STRUCTURED_PROPERTY_MAPPING_FIELD + + "." + + StructuredPropertyUtils.sanitizeStructuredPropertyFQN( + fieldName.substring(STRUCTURED_PROPERTY_MAPPING_FIELD.length() + 1)); + } + return fieldName.replace(ESUtils.KEYWORD_SUFFIX, ""); } @Nonnull diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index 247d542604da7..ee2d794471f6b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -10,15 +10,16 @@ import com.linkedin.common.InputField; import com.linkedin.common.InputFields; import com.linkedin.common.Status; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RecordTemplate; import com.linkedin.dataset.FineGrainedLineage; import com.linkedin.dataset.UpstreamLineage; -import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.batch.MCLBatchItem; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.entity.ebean.batch.MCLBatchItemImpl; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.GraphIndexUtils; @@ -43,6 +44,7 @@ import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; +import com.linkedin.structured.StructuredPropertyDefinition; import com.linkedin.util.Pair; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -70,11 +72,11 @@ public class UpdateIndicesService { private final EntitySearchService _entitySearchService; private final TimeseriesAspectService _timeseriesAspectService; private final SystemMetadataService _systemMetadataService; - private final EntityRegistry _entityRegistry; private final SearchDocumentTransformer _searchDocumentTransformer; private final EntityIndexBuilders _entityIndexBuilders; - private SystemEntityClient systemEntityClient; + private AspectRetriever aspectRetriever; + private EntityRegistry _entityRegistry; @Value("${featureFlags.graphServiceDiffModeEnabled:true}") private boolean _graphDiffMode; @@ -82,6 +84,12 @@ public class UpdateIndicesService { @Value("${featureFlags.searchServiceDiffModeEnabled:true}") private boolean _searchDiffMode; + @Value("${structuredProperties.enabled}") + private boolean _structuredPropertiesHookEnabled; + + @Value("${structuredProperties.writeEnabled}") + private boolean _structuredPropertiesWriteEnabled; + private static final Set UPDATE_CHANGE_TYPES = ImmutableSet.of(ChangeType.UPSERT, ChangeType.RESTATE, ChangeType.PATCH); @@ -100,33 +108,29 @@ public UpdateIndicesService( EntitySearchService entitySearchService, TimeseriesAspectService timeseriesAspectService, SystemMetadataService systemMetadataService, - EntityRegistry entityRegistry, SearchDocumentTransformer searchDocumentTransformer, EntityIndexBuilders entityIndexBuilders) { _graphService = graphService; _entitySearchService = entitySearchService; _timeseriesAspectService = timeseriesAspectService; _systemMetadataService = systemMetadataService; - _entityRegistry = entityRegistry; _searchDocumentTransformer = searchDocumentTransformer; _entityIndexBuilders = entityIndexBuilders; } public void handleChangeEvent(@Nonnull final MetadataChangeLog event) { try { - MCLBatchItemImpl batch = - MCLBatchItemImpl.builder().build(event, _entityRegistry, systemEntityClient); + MCLBatchItemImpl batch = MCLBatchItemImpl.builder().build(event, aspectRetriever); Stream sideEffects = _entityRegistry .getMCLSideEffects( event.getChangeType(), event.getEntityType(), event.getAspectName()) .stream() - .flatMap( - mclSideEffect -> - mclSideEffect.apply(List.of(batch), _entityRegistry, systemEntityClient)); + .flatMap(mclSideEffect -> mclSideEffect.apply(List.of(batch), aspectRetriever)); - for (MCLBatchItem mclBatchItem : Stream.concat(Stream.of(batch), sideEffects).toList()) { + for (MCLBatchItem mclBatchItem : + Stream.concat(Stream.of(batch), sideEffects).collect(Collectors.toList())) { MetadataChangeLog hookEvent = mclBatchItem.getMetadataChangeLog(); if (UPDATE_CHANGE_TYPES.contains(hookEvent.getChangeType())) { handleUpdateChangeEvent(mclBatchItem); @@ -173,11 +177,14 @@ private void handleUpdateChangeEvent(@Nonnull final MCLBatchItem event) throws I updateSystemMetadata(event.getSystemMetadata(), urn, aspectSpec, aspect); } - // Step 1. For all aspects, attempt to update Search + // Step 1. Handle StructuredProperties Index Mapping changes + updateIndexMappings(entitySpec, aspectSpec, aspect, previousAspect); + + // Step 2. For all aspects, attempt to update Search updateSearchService( entitySpec.getName(), urn, aspectSpec, aspect, event.getSystemMetadata(), previousAspect); - // Step 2. For all aspects, attempt to update Graph + // Step 3. For all aspects, attempt to update Graph SystemMetadata systemMetadata = event.getSystemMetadata(); if (_graphDiffMode && !(_graphService instanceof DgraphGraphService) @@ -190,6 +197,46 @@ private void handleUpdateChangeEvent(@Nonnull final MCLBatchItem event) throws I } } + public void updateIndexMappings( + EntitySpec entitySpec, + AspectSpec aspectSpec, + RecordTemplate newValue, + RecordTemplate oldValue) + throws IOException { + if (_structuredPropertiesHookEnabled + && STRUCTURED_PROPERTY_ENTITY_NAME.equals(entitySpec.getName()) + && STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME.equals(aspectSpec.getName())) { + + UrnArray oldEntityTypes = + Optional.ofNullable(oldValue) + .map( + recordTemplate -> + new StructuredPropertyDefinition(recordTemplate.data()).getEntityTypes()) + .orElse(new UrnArray()); + + StructuredPropertyDefinition newDefinition = + new StructuredPropertyDefinition(newValue.data()); + newDefinition.getEntityTypes().removeAll(oldEntityTypes); + + if (newDefinition.getEntityTypes().size() > 0) { + _entityIndexBuilders + .buildReindexConfigsWithNewStructProp(newDefinition) + .forEach( + reindexState -> { + try { + log.info( + "Applying new structured property {} to index {}", + newDefinition, + reindexState.name()); + _entityIndexBuilders.getIndexBuilder().applyMappings(reindexState, false); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } + } + /** * This very important method processes {@link MetadataChangeLog} deletion events to cleanup the * Metadata Graph when an aspect or entity is removed. @@ -617,13 +664,13 @@ private EntitySpec getEventEntitySpec(@Nonnull final MetadataChangeLog event) { } /** - * Allow internal use of the system entity client. Solves recursive dependencies between the - * UpdateIndicesService and the SystemJavaEntityClient + * Solves recursive dependencies between the UpdateIndicesService and EntityService * - * @param systemEntityClient system entity client + * @param aspectRetriever aspect Retriever */ - public void setSystemEntityClient(SystemEntityClient systemEntityClient) { - this.systemEntityClient = systemEntityClient; - _searchDocumentTransformer.setEntityClient(systemEntityClient); + public void initializeAspectRetriever(AspectRetriever aspectRetriever) { + this.aspectRetriever = aspectRetriever; + this._entityRegistry = aspectRetriever.getEntityRegistry(); + this._searchDocumentTransformer.setAspectRetriever(aspectRetriever); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java index 9aa0cdca99f68..e894558e3d1af 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/shared/ElasticSearchIndexed.java @@ -1,7 +1,9 @@ package com.linkedin.metadata.shared; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; +import com.linkedin.structured.StructuredPropertyDefinition; import java.io.IOException; +import java.util.Collection; import java.util.List; public interface ElasticSearchIndexed { @@ -12,6 +14,15 @@ public interface ElasticSearchIndexed { */ List buildReindexConfigs() throws IOException; + /** + * The index configurations for the given service with StructuredProperties applied. + * + * @param properties The structured properties to apply to the index mappings + * @return List of reindex configurations + */ + List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException; + /** * Mirrors the service's functions which are expected to build/reindex as needed based on the * reindex configurations above diff --git a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java index 6fbe7cfe882ce..36eab7b69e6a1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/systemmetadata/ElasticSearchSystemMetadataService.java @@ -13,12 +13,14 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.mxe.SystemMetadata; +import com.linkedin.structured.StructuredPropertyDefinition; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -245,6 +247,12 @@ public List buildReindexConfigs() throws IOException { Collections.emptyMap())); } + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException { + return buildReindexConfigs(); + } + @Override public void reindexAll() { configure(); diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java index f9ab86d41335d..a2b36b7d8ddb8 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/ElasticSearchTimeseriesAspectService.java @@ -20,12 +20,15 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; +import com.linkedin.metadata.search.elasticsearch.query.request.SearchAfterWrapper; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.search.utils.QueryUtils; import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.timeseries.BatchWriteOperationsOptions; +import com.linkedin.metadata.timeseries.GenericTimeseriesDocument; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.timeseries.TimeseriesScrollResult; import com.linkedin.metadata.timeseries.elastic.indexbuilder.MappingsBuilder; import com.linkedin.metadata.timeseries.elastic.indexbuilder.TimeseriesAspectIndexBuilders; import com.linkedin.metadata.timeseries.elastic.query.ESAggregatedStatsDAO; @@ -33,6 +36,7 @@ import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.SystemMetadata; +import com.linkedin.structured.StructuredPropertyDefinition; import com.linkedin.timeseries.AggregationSpec; import com.linkedin.timeseries.DeleteAspectValuesResult; import com.linkedin.timeseries.GenericTable; @@ -43,9 +47,11 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -86,8 +92,6 @@ public class ElasticSearchTimeseriesAspectService .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); } - private static final String TIMESTAMP_FIELD = "timestampMillis"; - private static final String EVENT_FIELD = "event"; private static final Integer DEFAULT_LIMIT = 10000; private final IndexConvention _indexConvention; @@ -118,7 +122,7 @@ public ElasticSearchTimeseriesAspectService( private static EnvelopedAspect parseDocument(@Nonnull SearchHit doc) { Map docFields = doc.getSourceAsMap(); EnvelopedAspect envelopedAspect = new EnvelopedAspect(); - Object event = docFields.get(EVENT_FIELD); + Object event = docFields.get(MappingsBuilder.EVENT_FIELD); GenericAspect genericAspect; try { genericAspect = @@ -147,6 +151,61 @@ private static EnvelopedAspect parseDocument(@Nonnull SearchHit doc) { return envelopedAspect; } + private static Set commonFields = + Set.of( + MappingsBuilder.URN_FIELD, + MappingsBuilder.RUN_ID_FIELD, + MappingsBuilder.EVENT_GRANULARITY, + MappingsBuilder.IS_EXPLODED_FIELD, + MappingsBuilder.MESSAGE_ID_FIELD, + MappingsBuilder.PARTITION_SPEC_PARTITION, + MappingsBuilder.PARTITION_SPEC, + MappingsBuilder.SYSTEM_METADATA_FIELD, + MappingsBuilder.TIMESTAMP_MILLIS_FIELD, + MappingsBuilder.TIMESTAMP_FIELD, + MappingsBuilder.EVENT_FIELD); + + private static Pair toEnvAspectGenericDocument( + @Nonnull SearchHit doc) { + EnvelopedAspect envelopedAspect = null; + + Map documentFieldMap = doc.getSourceAsMap(); + + GenericTimeseriesDocument.GenericTimeseriesDocumentBuilder builder = + GenericTimeseriesDocument.builder() + .urn((String) documentFieldMap.get(MappingsBuilder.URN_FIELD)) + .timestampMillis((Long) documentFieldMap.get(MappingsBuilder.TIMESTAMP_MILLIS_FIELD)) + .timestamp((Long) documentFieldMap.get(MappingsBuilder.TIMESTAMP_FIELD)); + + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.RUN_ID_FIELD)) + .ifPresent(d -> builder.runId((String) d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.EVENT_GRANULARITY)) + .ifPresent(d -> builder.eventGranularity((String) d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.IS_EXPLODED_FIELD)) + .ifPresent(d -> builder.isExploded((Boolean) d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.MESSAGE_ID_FIELD)) + .ifPresent(d -> builder.messageId((String) d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.PARTITION_SPEC_PARTITION)) + .ifPresent(d -> builder.partition((String) d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.PARTITION_SPEC)) + .ifPresent(d -> builder.partitionSpec(d)); + Optional.ofNullable(documentFieldMap.get(MappingsBuilder.SYSTEM_METADATA_FIELD)) + .ifPresent(d -> builder.systemMetadata(d)); + + if (documentFieldMap.get(MappingsBuilder.EVENT_FIELD) != null) { + envelopedAspect = parseDocument(doc); + builder.event(documentFieldMap.get(MappingsBuilder.EVENT_FIELD)); + } else { + // If no event, the event is any non-common field + builder.event( + documentFieldMap.entrySet().stream() + .filter(entry -> !commonFields.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + return Pair.of(envelopedAspect, builder.build()); + } + @Override public void configure() { _indexBuilders.reindexAll(); @@ -157,6 +216,12 @@ public List buildReindexConfigs() { return _indexBuilders.buildReindexConfigs(); } + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException { + return _indexBuilders.buildReindexConfigsWithAllStructProps(properties); + } + public String reindexAsync( String index, @Nullable QueryBuilder filterQuery, BatchWriteOperationsOptions options) throws Exception { @@ -256,7 +321,7 @@ public List getAspectValues( if (startTimeMillis != null) { Criterion startTimeCriterion = new Criterion() - .setField(TIMESTAMP_FIELD) + .setField(MappingsBuilder.TIMESTAMP_MILLIS_FIELD) .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO) .setValue(startTimeMillis.toString()); filterQueryBuilder.must(ESUtils.getQueryBuilderFromCriterion(startTimeCriterion, true)); @@ -264,7 +329,7 @@ public List getAspectValues( if (endTimeMillis != null) { Criterion endTimeCriterion = new Criterion() - .setField(TIMESTAMP_FIELD) + .setField(MappingsBuilder.TIMESTAMP_MILLIS_FIELD) .setCondition(Condition.LESS_THAN_OR_EQUAL_TO) .setValue(endTimeMillis.toString()); filterQueryBuilder.must(ESUtils.getQueryBuilderFromCriterion(endTimeCriterion, true)); @@ -421,4 +486,88 @@ public DeleteAspectValuesResult rollbackTimeseriesAspects(@Nonnull String runId) return rollbackResult; } + + @Nonnull + @Override + public TimeseriesScrollResult scrollAspects( + @Nonnull String entityName, + @Nonnull String aspectName, + @Nullable Filter filter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { + final BoolQueryBuilder filterQueryBuilder = + QueryBuilders.boolQuery().filter(ESUtils.buildFilterQuery(filter, true)); + + if (startTimeMillis != null) { + Criterion startTimeCriterion = + new Criterion() + .setField(MappingsBuilder.TIMESTAMP_MILLIS_FIELD) + .setCondition(Condition.GREATER_THAN_OR_EQUAL_TO) + .setValue(startTimeMillis.toString()); + filterQueryBuilder.filter(ESUtils.getQueryBuilderFromCriterion(startTimeCriterion, true)); + } + if (endTimeMillis != null) { + Criterion endTimeCriterion = + new Criterion() + .setField(MappingsBuilder.TIMESTAMP_MILLIS_FIELD) + .setCondition(Condition.LESS_THAN_OR_EQUAL_TO) + .setValue(endTimeMillis.toString()); + filterQueryBuilder.filter(ESUtils.getQueryBuilderFromCriterion(endTimeCriterion, true)); + } + + SearchResponse response = + executeScrollSearchQuery( + entityName, aspectName, filterQueryBuilder, sortCriterion, scrollId, count); + int totalCount = (int) response.getHits().getTotalHits().value; + + List> resultPairs = + Arrays.stream(response.getHits().getHits()) + .map(ElasticSearchTimeseriesAspectService::toEnvAspectGenericDocument) + .collect(Collectors.toList()); + + return TimeseriesScrollResult.builder() + .numResults(totalCount) + .pageSize(response.getHits().getHits().length) + .events(resultPairs.stream().map(Pair::getFirst).collect(Collectors.toList())) + .documents(resultPairs.stream().map(Pair::getSecond).collect(Collectors.toList())) + .build(); + } + + private SearchResponse executeScrollSearchQuery( + @Nonnull final String entityNname, + @Nonnull final String aspectName, + @Nonnull final QueryBuilder query, + @Nonnull List sortCriterion, + @Nullable String scrollId, + final int count) { + + Object[] sort = null; + if (scrollId != null) { + SearchAfterWrapper searchAfterWrapper = SearchAfterWrapper.fromScrollId(scrollId); + sort = searchAfterWrapper.getSort(); + } + + SearchRequest searchRequest = new SearchRequest(); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + + searchSourceBuilder.size(count); + searchSourceBuilder.query(query); + ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, List.of(), false); + searchRequest.source(searchSourceBuilder); + ESUtils.setSearchAfter(searchSourceBuilder, sort, null, null); + + searchRequest.indices(_indexConvention.getTimeseriesAspectIndexName(entityNname, aspectName)); + + try (Timer.Context ignored = + MetricUtils.timer(this.getClass(), "scrollAspects_search").time()) { + return _searchClient.search(searchRequest, RequestOptions.DEFAULT); + } catch (Exception e) { + log.error("Search query failed", e); + throw new ESQueryException("Search query failed:", e); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java index 564bcb2a242cb..6437bbc390d82 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/timeseries/elastic/indexbuilder/TimeseriesAspectIndexBuilders.java @@ -7,8 +7,10 @@ import com.linkedin.metadata.shared.ElasticSearchIndexed; import com.linkedin.metadata.timeseries.BatchWriteOperationsOptions; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import com.linkedin.structured.StructuredPropertyDefinition; import com.linkedin.util.Pair; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -91,4 +93,10 @@ public List buildReindexConfigs() { }) .collect(Collectors.toList()); } + + @Override + public List buildReindexConfigsWithAllStructProps( + Collection properties) throws IOException { + return buildReindexConfigs(); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java index 252ac2d633b98..451b732722498 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectIngestionUtils.java @@ -41,7 +41,7 @@ public static Map ingestCorpUserKeyAspects( .aspect(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(entityService.getEntityRegistry(), entityService.getSystemEntityClient())); + .build(entityService)); } entityService.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); return aspects; @@ -71,7 +71,7 @@ public static Map ingestCorpUserInfoAspects( .aspect(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(entityService.getEntityRegistry(), entityService.getSystemEntityClient())); + .build(entityService)); } entityService.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); return aspects; @@ -102,7 +102,7 @@ public static Map ingestChartInfoAspects( .aspect(aspect) .auditStamp(AspectGenerationUtils.createAuditStamp()) .systemMetadata(AspectGenerationUtils.createSystemMetadata()) - .build(entityService.getEntityRegistry(), entityService.getSystemEntityClient())); + .build(entityService)); } entityService.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); return aspects; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java index fba11f24f4c44..5a4443904e260 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java @@ -6,7 +6,6 @@ import com.codahale.metrics.Counter; import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.metadata.entity.DeleteEntityService; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.event.EventProducer; @@ -14,6 +13,7 @@ import com.linkedin.metadata.search.LineageSearchService; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.service.RollbackService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.utils.metrics.MetricUtils; import java.util.function.Supplier; @@ -32,8 +32,8 @@ public class JavaEntityClientTest { private LineageSearchService _lineageSearchService; private TimeseriesAspectService _timeseriesAspectService; private EventProducer _eventProducer; - private RestliEntityClient _restliEntityClient; private MockedStatic _metricUtils; + private RollbackService rollbackService; private Counter _counter; @BeforeMethod @@ -45,8 +45,8 @@ public void setupTest() { _searchService = mock(SearchService.class); _lineageSearchService = mock(LineageSearchService.class); _timeseriesAspectService = mock(TimeseriesAspectService.class); + rollbackService = mock(RollbackService.class); _eventProducer = mock(EventProducer.class); - _restliEntityClient = mock(RestliEntityClient.class); _metricUtils = mockStatic(MetricUtils.class); _counter = mock(Counter.class); when(MetricUtils.counter(any(), any())).thenReturn(_counter); @@ -66,8 +66,8 @@ private JavaEntityClient getJavaEntityClient() { _searchService, _lineageSearchService, _timeseriesAspectService, - _eventProducer, - _restliEntityClient); + rollbackService, + _eventProducer); } @Test diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 45e992576676d..c45306e5f022b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -124,21 +124,21 @@ public void testIngestListLatestAspects() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn2) .aspectName(aspectName) .aspect(writeAspect2) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn3) .aspectName(aspectName) .aspect(writeAspect3) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // List aspects @@ -193,21 +193,21 @@ public void testIngestListUrns() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn2) .aspectName(aspectName) .aspect(writeAspect2) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn3) .aspectName(aspectName) .aspect(writeAspect3) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // List aspects urns @@ -451,13 +451,7 @@ public void run() { auditStamp.setActor(Urn.createFromString(Constants.DATAHUB_ACTOR)); auditStamp.setTime(System.currentTimeMillis()); AspectsBatchImpl batch = - AspectsBatchImpl.builder() - .mcps( - mcps, - auditStamp, - entityService.getEntityRegistry(), - entityService.getSystemEntityClient()) - .build(); + AspectsBatchImpl.builder().mcps(mcps, auditStamp, entityService).build(); entityService.ingestProposal(batch, false); } } catch (InterruptedException | URISyntaxException ie) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index ee21b56cea7c0..db749f3575a06 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -12,6 +12,7 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.Status; +import com.linkedin.common.UrnArray; import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; import com.linkedin.common.urn.TupleKey; @@ -29,6 +30,7 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.AspectGenerationUtils; @@ -58,6 +60,12 @@ import com.linkedin.retention.DataHubRetentionConfig; import com.linkedin.retention.Retention; import com.linkedin.retention.VersionBasedRetention; +import com.linkedin.structured.PrimitivePropertyValue; +import com.linkedin.structured.PrimitivePropertyValueArray; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; import com.linkedin.util.Pair; import jakarta.annotation.Nonnull; import java.util.ArrayList; @@ -67,6 +75,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.junit.Assert; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -847,28 +857,28 @@ public void testRollbackAspect() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn2) .aspectName(aspectName) .aspect(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn3) .aspectName(aspectName) .aspect(writeAspect3) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn1) .aspectName(aspectName) .aspect(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // this should no-op since this run has been overwritten @@ -926,21 +936,21 @@ public void testRollbackKey() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn1) .aspectName(keyAspectName) .aspect(writeKey1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn1) .aspectName(aspectName) .aspect(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // this should no-op since the key should have been written in the furst run @@ -1006,35 +1016,35 @@ public void testRollbackUrn() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn1) .aspectName(keyAspectName) .aspect(writeKey1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn2) .aspectName(aspectName) .aspect(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn3) .aspectName(aspectName) .aspect(writeAspect3) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn1) .aspectName(aspectName) .aspect(writeAspect1Overwrite) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // this should no-op since the key should have been written in the furst run @@ -1073,7 +1083,7 @@ public void testIngestGetLatestAspect() throws AssertionError { .aspect(writeAspect1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #1 @@ -1104,7 +1114,7 @@ public void testIngestGetLatestAspect() throws AssertionError { .aspect(writeAspect2) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata2) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #2 @@ -1150,7 +1160,7 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { .aspect(writeAspect1) .auditStamp(TEST_AUDIT_STAMP) .systemMetadata(metadata1) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #1 @@ -1170,7 +1180,7 @@ public void testIngestGetLatestEnvelopedAspect() throws Exception { .aspect(writeAspect2) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #2 @@ -1215,7 +1225,7 @@ public void testIngestSameAspect() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #1 @@ -1246,7 +1256,7 @@ public void testIngestSameAspect() throws AssertionError { .aspect(writeAspect2) .systemMetadata(metadata2) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); // Validate retrieval of CorpUserInfo Aspect #2 @@ -1299,42 +1309,42 @@ public void testRetention() throws AssertionError { .aspect(writeAspect1) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName) .aspect(writeAspect1a) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName) .aspect(writeAspect1b) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName2) .aspect(writeAspect2) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName2) .aspect(writeAspect2a) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName2) .aspect(writeAspect2b) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); assertEquals(_entityServiceImpl.getAspect(entityUrn, aspectName, 1), writeAspect1); @@ -1366,14 +1376,14 @@ public void testRetention() throws AssertionError { .aspect(writeAspect1c) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient()), + .build(_entityServiceImpl), MCPUpsertBatchItem.builder() .urn(entityUrn) .aspectName(aspectName2) .aspect(writeAspect2c) .systemMetadata(metadata1) .auditStamp(TEST_AUDIT_STAMP) - .build(_testEntityRegistry, _entityServiceImpl.getSystemEntityClient())); + .build(_entityServiceImpl)); _entityServiceImpl.ingestAspects(AspectsBatchImpl.builder().items(items).build(), true, true); assertNull(_entityServiceImpl.getAspect(entityUrn, aspectName, 1)); @@ -1637,6 +1647,172 @@ public void testUIPreProcessedProposal() throws Exception { assertEquals(UI_SOURCE, captor.getValue().getSystemMetadata().getProperties().get(APP_SOURCE)); } + @Test + public void testStructuredPropertyIngestProposal() throws Exception { + String urnStr = "urn:li:dataset:(urn:li:dataPlatform:looker,sample_dataset_unique,PROD)"; + Urn entityUrn = UrnUtils.getUrn(urnStr); + + // Ingest one structured property definition + String definitionAspectName = "propertyDefinition"; + Urn firstPropertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:firstStructuredProperty"); + MetadataChangeProposal gmce = new MetadataChangeProposal(); + gmce.setEntityUrn(firstPropertyUrn); + gmce.setChangeType(ChangeType.UPSERT); + gmce.setEntityType("structuredProperty"); + gmce.setAspectName(definitionAspectName); + StructuredPropertyDefinition structuredPropertyDefinition = + new StructuredPropertyDefinition() + .setQualifiedName("firstStructuredProperty") + .setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "string")) + .setEntityTypes(new UrnArray(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"))); + JacksonDataTemplateCodec dataTemplateCodec = new JacksonDataTemplateCodec(); + byte[] definitionSerialized = + dataTemplateCodec.dataTemplateToBytes(structuredPropertyDefinition); + GenericAspect genericAspect = new GenericAspect(); + genericAspect.setValue(ByteString.unsafeWrap(definitionSerialized)); + genericAspect.setContentType("application/json"); + gmce.setAspect(genericAspect); + _entityServiceImpl.ingestProposal(gmce, TEST_AUDIT_STAMP, false); + ArgumentCaptor captor = ArgumentCaptor.forClass(MetadataChangeLog.class); + verify(_mockProducer, times(1)) + .produceMetadataChangeLog(Mockito.eq(firstPropertyUrn), Mockito.any(), captor.capture()); + assertEquals( + _entityServiceImpl.getAspect(firstPropertyUrn, definitionAspectName, 0), + structuredPropertyDefinition); + Urn secondPropertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:secondStructuredProperty"); + assertNull(_entityServiceImpl.getAspect(secondPropertyUrn, definitionAspectName, 0)); + assertEquals( + _entityServiceImpl.getAspect(firstPropertyUrn, definitionAspectName, 0), + structuredPropertyDefinition); + Set defs = + _aspectDao + .streamAspects( + STRUCTURED_PROPERTY_ENTITY_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .map( + entityAspect -> + EntityUtils.toAspectRecord( + STRUCTURED_PROPERTY_ENTITY_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + entityAspect.getMetadata(), + _testEntityRegistry)) + .map(recordTemplate -> (StructuredPropertyDefinition) recordTemplate) + .collect(Collectors.toSet()); + assertEquals(defs.size(), 1); + assertEquals(defs, Set.of(structuredPropertyDefinition)); + + SystemEntityClient mockSystemEntityClient = Mockito.mock(SystemEntityClient.class); + Mockito.when( + mockSystemEntityClient.getLatestAspectObject(firstPropertyUrn, "propertyDefinition")) + .thenReturn(new com.linkedin.entity.Aspect(structuredPropertyDefinition.data())); + + // Add a value for that property + PrimitivePropertyValueArray propertyValues = new PrimitivePropertyValueArray(); + propertyValues.add(PrimitivePropertyValue.create("hello")); + StructuredPropertyValueAssignment assignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn(firstPropertyUrn) + .setValues(propertyValues); + StructuredProperties structuredProperties = + new StructuredProperties() + .setProperties(new StructuredPropertyValueAssignmentArray(assignment)); + MetadataChangeProposal asgnMce = new MetadataChangeProposal(); + asgnMce.setEntityUrn(entityUrn); + asgnMce.setChangeType(ChangeType.UPSERT); + asgnMce.setEntityType("dataset"); + asgnMce.setAspectName("structuredProperties"); + JacksonDataTemplateCodec asgnTemplateCodec = new JacksonDataTemplateCodec(); + byte[] asgnSerialized = asgnTemplateCodec.dataTemplateToBytes(structuredProperties); + GenericAspect asgnGenericAspect = new GenericAspect(); + asgnGenericAspect.setValue(ByteString.unsafeWrap(asgnSerialized)); + asgnGenericAspect.setContentType("application/json"); + asgnMce.setAspect(asgnGenericAspect); + _entityServiceImpl.ingestProposal(asgnMce, TEST_AUDIT_STAMP, false); + assertEquals( + _entityServiceImpl.getAspect(entityUrn, "structuredProperties", 0), structuredProperties); + + // Ingest second structured property definition + MetadataChangeProposal gmce2 = new MetadataChangeProposal(); + gmce2.setEntityUrn(secondPropertyUrn); + gmce2.setChangeType(ChangeType.UPSERT); + gmce2.setEntityType("structuredProperty"); + gmce2.setAspectName(definitionAspectName); + StructuredPropertyDefinition secondDefinition = + new StructuredPropertyDefinition() + .setQualifiedName("secondStructuredProperty") + .setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "number")) + .setEntityTypes(new UrnArray(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"))); + JacksonDataTemplateCodec secondDataTemplate = new JacksonDataTemplateCodec(); + byte[] secondDefinitionSerialized = secondDataTemplate.dataTemplateToBytes(secondDefinition); + GenericAspect secondGenericAspect = new GenericAspect(); + secondGenericAspect.setValue(ByteString.unsafeWrap(secondDefinitionSerialized)); + secondGenericAspect.setContentType("application/json"); + gmce2.setAspect(secondGenericAspect); + _entityServiceImpl.ingestProposal(gmce2, TEST_AUDIT_STAMP, false); + ArgumentCaptor secondCaptor = + ArgumentCaptor.forClass(MetadataChangeLog.class); + verify(_mockProducer, times(1)) + .produceMetadataChangeLog( + Mockito.eq(secondPropertyUrn), Mockito.any(), secondCaptor.capture()); + assertEquals( + _entityServiceImpl.getAspect(firstPropertyUrn, definitionAspectName, 0), + structuredPropertyDefinition); + assertEquals( + _entityServiceImpl.getAspect(secondPropertyUrn, definitionAspectName, 0), secondDefinition); + defs = + _aspectDao + .streamAspects( + STRUCTURED_PROPERTY_ENTITY_NAME, STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .map( + entityAspect -> + EntityUtils.toAspectRecord( + STRUCTURED_PROPERTY_ENTITY_NAME, + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + entityAspect.getMetadata(), + _testEntityRegistry)) + .map(recordTemplate -> (StructuredPropertyDefinition) recordTemplate) + .collect(Collectors.toSet()); + assertEquals(defs.size(), 2); + assertEquals(defs, Set.of(secondDefinition, structuredPropertyDefinition)); + + Mockito.when( + mockSystemEntityClient.getLatestAspectObject(secondPropertyUrn, "propertyDefinition")) + .thenReturn(new com.linkedin.entity.Aspect(secondDefinition.data())); + + // Get existing value for first structured property + assertEquals( + _entityServiceImpl.getAspect(entityUrn, "structuredProperties", 0), structuredProperties); + + // Add a value for second property + propertyValues = new PrimitivePropertyValueArray(); + propertyValues.add(PrimitivePropertyValue.create(15.0)); + StructuredPropertyValueAssignment secondAssignment = + new StructuredPropertyValueAssignment() + .setPropertyUrn(secondPropertyUrn) + .setValues(propertyValues); + StructuredProperties secondPropertyArr = + new StructuredProperties() + .setProperties( + new StructuredPropertyValueAssignmentArray(assignment, secondAssignment)); + MetadataChangeProposal asgn2Mce = new MetadataChangeProposal(); + asgn2Mce.setEntityUrn(entityUrn); + asgn2Mce.setChangeType(ChangeType.UPSERT); + asgn2Mce.setEntityType("dataset"); + asgn2Mce.setAspectName("structuredProperties"); + JacksonDataTemplateCodec asgnTemplateCodec2 = new JacksonDataTemplateCodec(); + byte[] asgnSerialized2 = asgnTemplateCodec2.dataTemplateToBytes(secondPropertyArr); + GenericAspect asgnGenericAspect2 = new GenericAspect(); + asgnGenericAspect2.setValue(ByteString.unsafeWrap(asgnSerialized2)); + asgnGenericAspect2.setContentType("application/json"); + asgn2Mce.setAspect(asgnGenericAspect2); + _entityServiceImpl.ingestProposal(asgn2Mce, TEST_AUDIT_STAMP, false); + StructuredProperties expectedProperties = + new StructuredProperties() + .setProperties( + new StructuredPropertyValueAssignmentArray(assignment, secondAssignment)); + assertEquals( + _entityServiceImpl.getAspect(entityUrn, "structuredProperties", 0), expectedProperties); + } + @Nonnull protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email) throws Exception { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/TestEntityRegistry.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/TestEntityRegistry.java index 680d4079851eb..15852e0cbe35b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/TestEntityRegistry.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/TestEntityRegistry.java @@ -1,11 +1,11 @@ package com.linkedin.metadata.entity; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EntitySpecBuilder; import com.linkedin.metadata.models.EventSpec; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.metadata.snapshot.Snapshot; import java.util.Collections; import java.util.HashMap; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java index 2f8fba0083aa7..bd500cd469100 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java @@ -337,26 +337,26 @@ public void testTimestampLineage() throws Exception { // Without timestamps EntityLineageResult upstreamResult = getUpstreamLineage(datasetTwoUrn, null, null); EntityLineageResult downstreamResult = getDownstreamLineage(datasetTwoUrn, null, null); - Assert.assertEquals(new Integer(1), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(3), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(3), downstreamResult.getTotal()); // Timestamp before upstreamResult = getUpstreamLineage(datasetTwoUrn, 0L, initialTime - 10); downstreamResult = getDownstreamLineage(datasetTwoUrn, 0L, initialTime - 10); - Assert.assertEquals(new Integer(0), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(1), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(0), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), downstreamResult.getTotal()); // Timestamp after upstreamResult = getUpstreamLineage(datasetTwoUrn, initialTime + 10, initialTime + 100); downstreamResult = getDownstreamLineage(datasetTwoUrn, initialTime + 10, initialTime + 100); - Assert.assertEquals(new Integer(0), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(1), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(0), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), downstreamResult.getTotal()); // Timestamp included upstreamResult = getUpstreamLineage(datasetTwoUrn, initialTime - 10, initialTime + 10); downstreamResult = getDownstreamLineage(datasetTwoUrn, initialTime - 10, initialTime + 10); - Assert.assertEquals(new Integer(1), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(3), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(3), downstreamResult.getTotal()); // Update only one of the downstream edges Long updatedTime = 2000L; @@ -387,20 +387,20 @@ public void testTimestampLineage() throws Exception { // Without timestamps upstreamResult = getUpstreamLineage(datasetTwoUrn, null, null); downstreamResult = getDownstreamLineage(datasetTwoUrn, null, null); - Assert.assertEquals(new Integer(1), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(3), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(3), downstreamResult.getTotal()); // Window includes initial time and updated time upstreamResult = getUpstreamLineage(datasetTwoUrn, initialTime - 10, updatedTime + 10); downstreamResult = getDownstreamLineage(datasetTwoUrn, initialTime - 10, updatedTime + 10); - Assert.assertEquals(new Integer(1), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(3), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(3), downstreamResult.getTotal()); // Window includes updated time but not initial time upstreamResult = getUpstreamLineage(datasetTwoUrn, initialTime + 10, updatedTime + 10); downstreamResult = getDownstreamLineage(datasetTwoUrn, initialTime + 10, updatedTime + 10); - Assert.assertEquals(new Integer(1), upstreamResult.getTotal()); - Assert.assertEquals(new Integer(2), downstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(1), upstreamResult.getTotal()); + Assert.assertEquals(Integer.valueOf(2), downstreamResult.getTotal()); } /** diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java index fba9d5359d29f..d2aef982750bd 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java @@ -7,7 +7,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.EntityType; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.MatchedFieldArray; import com.linkedin.metadata.search.SearchEntityArray; diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java index 2c395875a1d6b..a54e8aa1c9191 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/IndexBuilderTestBase.java @@ -1,14 +1,19 @@ package com.linkedin.metadata.search.indexbuilder; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_MAPPING_FIELD; import static org.testng.Assert.*; import com.google.common.collect.ImmutableMap; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder; +import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.systemmetadata.SystemMetadataMappingsBuilder; import com.linkedin.metadata.version.GitVersion; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -295,4 +300,117 @@ public void testSettingsNoReindex() throws Exception { wipe(); } } + + @Test + public void testCopyStructuredPropertyMappings() throws Exception { + GitVersion gitVersion = new GitVersion("0.0.0-test", "123456", Optional.empty()); + ESIndexBuilder enabledMappingReindex = + new ESIndexBuilder( + getSearchClient(), + 1, + 0, + 0, + 0, + Map.of(), + false, + true, + new ElasticSearchConfiguration(), + gitVersion); + + ReindexConfig reindexConfigNoIndexBefore = + enabledMappingReindex.buildReindexState( + TEST_INDEX_NAME, SystemMetadataMappingsBuilder.getMappings(), Map.of()); + assertNull(reindexConfigNoIndexBefore.currentMappings()); + assertEquals( + reindexConfigNoIndexBefore.targetMappings(), SystemMetadataMappingsBuilder.getMappings()); + assertFalse(reindexConfigNoIndexBefore.requiresApplyMappings()); + assertFalse(reindexConfigNoIndexBefore.isPureMappingsAddition()); + + // Create index + enabledMappingReindex.buildIndex( + TEST_INDEX_NAME, SystemMetadataMappingsBuilder.getMappings(), Map.of()); + + // Test build reindex config with no structured properties added + ReindexConfig reindexConfigNoChange = + enabledMappingReindex.buildReindexState( + TEST_INDEX_NAME, SystemMetadataMappingsBuilder.getMappings(), Map.of()); + assertEquals( + reindexConfigNoChange.currentMappings(), SystemMetadataMappingsBuilder.getMappings()); + assertEquals( + reindexConfigNoChange.targetMappings(), SystemMetadataMappingsBuilder.getMappings()); + assertFalse(reindexConfigNoIndexBefore.requiresApplyMappings()); + assertFalse(reindexConfigNoIndexBefore.isPureMappingsAddition()); + + // Test add new field to the mappings + Map targetMappingsNewField = + new HashMap<>(SystemMetadataMappingsBuilder.getMappings()); + ((Map) targetMappingsNewField.get("properties")) + .put("myNewField", Map.of(SettingsBuilder.TYPE, SettingsBuilder.KEYWORD)); + + // Test build reindex config for new fields with no structured properties added + ReindexConfig reindexConfigNewField = + enabledMappingReindex.buildReindexState(TEST_INDEX_NAME, targetMappingsNewField, Map.of()); + assertEquals( + reindexConfigNewField.currentMappings(), SystemMetadataMappingsBuilder.getMappings()); + assertEquals(reindexConfigNewField.targetMappings(), targetMappingsNewField); + assertTrue(reindexConfigNewField.requiresApplyMappings()); + assertTrue(reindexConfigNewField.isPureMappingsAddition()); + + // Add structured properties to index + Map mappingsWithStructuredProperties = + new HashMap<>(SystemMetadataMappingsBuilder.getMappings()); + ((Map) mappingsWithStructuredProperties.get("properties")) + .put( + STRUCTURED_PROPERTY_MAPPING_FIELD + ".myStringProp", + Map.of(SettingsBuilder.TYPE, SettingsBuilder.KEYWORD)); + ((Map) mappingsWithStructuredProperties.get("properties")) + .put( + STRUCTURED_PROPERTY_MAPPING_FIELD + ".myNumberProp", + Map.of(SettingsBuilder.TYPE, ESUtils.DOUBLE_FIELD_TYPE)); + + enabledMappingReindex.buildIndex(TEST_INDEX_NAME, mappingsWithStructuredProperties, Map.of()); + + // Test build reindex config with structured properties not copied + ReindexConfig reindexConfigNoCopy = + enabledMappingReindex.buildReindexState( + TEST_INDEX_NAME, SystemMetadataMappingsBuilder.getMappings(), Map.of()); + Map expectedMappingsStructPropsNested = + new HashMap<>(SystemMetadataMappingsBuilder.getMappings()); + ((Map) expectedMappingsStructPropsNested.get("properties")) + .put( + "structuredProperties", + Map.of( + "properties", + Map.of( + "myNumberProp", + Map.of(SettingsBuilder.TYPE, ESUtils.DOUBLE_FIELD_TYPE), + "myStringProp", + Map.of(SettingsBuilder.TYPE, SettingsBuilder.KEYWORD)))); + assertEquals(reindexConfigNoCopy.currentMappings(), expectedMappingsStructPropsNested); + assertEquals(reindexConfigNoCopy.targetMappings(), SystemMetadataMappingsBuilder.getMappings()); + assertFalse(reindexConfigNoCopy.isPureMappingsAddition()); + + // Test build reindex config with structured properties copied + ReindexConfig reindexConfigCopy = + enabledMappingReindex.buildReindexState( + TEST_INDEX_NAME, SystemMetadataMappingsBuilder.getMappings(), Map.of(), true); + assertEquals(reindexConfigCopy.currentMappings(), expectedMappingsStructPropsNested); + assertEquals(reindexConfigCopy.targetMappings(), expectedMappingsStructPropsNested); + assertFalse(reindexConfigCopy.requiresApplyMappings()); + assertFalse(reindexConfigCopy.isPureMappingsAddition()); + + // Test build reindex config with new field added and structured properties copied + ReindexConfig reindexConfigCopyAndNewField = + enabledMappingReindex.buildReindexState( + TEST_INDEX_NAME, targetMappingsNewField, Map.of(), true); + assertEquals(reindexConfigCopyAndNewField.currentMappings(), expectedMappingsStructPropsNested); + Map targetMappingsNewFieldAndStructProps = + new HashMap<>(expectedMappingsStructPropsNested); + ((Map) targetMappingsNewFieldAndStructProps.get("properties")) + .put("myNewField", Map.of(SettingsBuilder.TYPE, SettingsBuilder.KEYWORD)); + assertEquals( + reindexConfigCopyAndNewField.targetMappings(), targetMappingsNewFieldAndStructProps); + assertTrue(reindexConfigCopyAndNewField.requiresApplyMappings()); + assertTrue(reindexConfigCopyAndNewField.isPureMappingsAddition()); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java index 02bd186ccc183..6df31b35fecde 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/indexbuilder/MappingsBuilderTest.java @@ -1,11 +1,16 @@ package com.linkedin.metadata.search.indexbuilder; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; +import static com.linkedin.metadata.Constants.*; +import static org.testng.Assert.*; import com.google.common.collect.ImmutableMap; +import com.linkedin.common.UrnArray; +import com.linkedin.common.urn.Urn; import com.linkedin.metadata.TestEntitySpecBuilder; import com.linkedin.metadata.search.elasticsearch.indexbuilder.MappingsBuilder; +import com.linkedin.structured.StructuredPropertyDefinition; +import java.net.URISyntaxException; +import java.util.List; import java.util.Map; import org.testng.annotations.Test; @@ -54,14 +59,6 @@ public void testMappingsBuilder() { Map keyPart3FieldSubfields = (Map) keyPart3Field.get("fields"); assertEquals(keyPart3FieldSubfields.size(), 1); assertTrue(keyPart3FieldSubfields.containsKey("keyword")); - Map customPropertiesField = - (Map) properties.get("customProperties"); - assertEquals(customPropertiesField.get("type"), "keyword"); - assertEquals(customPropertiesField.get("normalizer"), "keyword_normalizer"); - Map customPropertiesFieldSubfields = - (Map) customPropertiesField.get("fields"); - assertEquals(customPropertiesFieldSubfields.size(), 1); - assertTrue(customPropertiesFieldSubfields.containsKey("keyword")); // TEXT Map nestedArrayStringField = (Map) properties.get("nestedArrayStringField"); @@ -81,6 +78,15 @@ public void testMappingsBuilder() { assertEquals(nestedArrayArrayFieldSubfields.size(), 2); assertTrue(nestedArrayArrayFieldSubfields.containsKey("delimited")); assertTrue(nestedArrayArrayFieldSubfields.containsKey("keyword")); + Map customPropertiesField = + (Map) properties.get("customProperties"); + assertEquals(customPropertiesField.get("type"), "keyword"); + assertEquals(customPropertiesField.get("normalizer"), "keyword_normalizer"); + Map customPropertiesFieldSubfields = + (Map) customPropertiesField.get("fields"); + assertEquals(customPropertiesFieldSubfields.size(), 2); + assertTrue(customPropertiesFieldSubfields.containsKey("delimited")); + assertTrue(customPropertiesFieldSubfields.containsKey("keyword")); // TEXT with addToFilters Map textField = (Map) properties.get("textFieldOverride"); @@ -153,4 +159,115 @@ public void testMappingsBuilder() { Map doubleField = (Map) properties.get("doubleField"); assertEquals(doubleField.get("type"), "double"); } + + @Test + public void testGetMappingsWithStructuredProperty() throws URISyntaxException { + // Baseline comparison: Mappings with no structured props + Map resultWithoutStructuredProps = + MappingsBuilder.getMappings(TestEntitySpecBuilder.getSpec()); + + // Test that a structured property that does not apply to the entity does not alter the mappings + StructuredPropertyDefinition structPropNotForThisEntity = + new StructuredPropertyDefinition() + .setQualifiedName("propNotForThis") + .setDisplayName("propNotForThis") + .setEntityTypes(new UrnArray(Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map resultWithOnlyUnrelatedStructuredProp = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), List.of(structPropNotForThisEntity)); + assertEquals(resultWithOnlyUnrelatedStructuredProp, resultWithoutStructuredProps); + + // Test that a structured property that does apply to this entity is included in the mappings + String fqnOfRelatedProp = "propForThis"; + StructuredPropertyDefinition structPropForThisEntity = + new StructuredPropertyDefinition() + .setQualifiedName(fqnOfRelatedProp) + .setDisplayName("propForThis") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map resultWithOnlyRelatedStructuredProp = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), List.of(structPropForThisEntity)); + assertNotEquals(resultWithOnlyRelatedStructuredProp, resultWithoutStructuredProps); + Map fieldsBefore = + (Map) resultWithoutStructuredProps.get("properties"); + Map fieldsAfter = + (Map) resultWithOnlyRelatedStructuredProp.get("properties"); + assertEquals(fieldsAfter.size(), fieldsBefore.size() + 1); + + Map structProps = (Map) fieldsAfter.get("structuredProperties"); + fieldsAfter = (Map) structProps.get("properties"); + + String newField = + fieldsAfter.keySet().stream() + .filter(field -> !fieldsBefore.containsKey(field)) + .findFirst() + .get(); + assertEquals(newField, fqnOfRelatedProp); + assertEquals( + fieldsAfter.get(newField), + Map.of( + "normalizer", + "keyword_normalizer", + "type", + "keyword", + "fields", + Map.of("keyword", Map.of("type", "keyword")))); + + // Test that only structured properties that apply are included + Map resultWithBothStructuredProps = + MappingsBuilder.getMappings( + TestEntitySpecBuilder.getSpec(), + List.of(structPropForThisEntity, structPropNotForThisEntity)); + assertEquals(resultWithBothStructuredProps, resultWithOnlyRelatedStructuredProp); + } + + @Test + public void testGetMappingsForStructuredProperty() throws URISyntaxException { + StructuredPropertyDefinition testStructProp = + new StructuredPropertyDefinition() + .setQualifiedName("testProp") + .setDisplayName("exampleProp") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + Map structuredPropertyFieldMappings = + MappingsBuilder.getMappingsForStructuredProperty(List.of(testStructProp)); + assertEquals(structuredPropertyFieldMappings.size(), 1); + String keyInMap = structuredPropertyFieldMappings.keySet().stream().findFirst().get(); + assertEquals(keyInMap, "testProp"); + Object mappings = structuredPropertyFieldMappings.get(keyInMap); + assertEquals( + mappings, + Map.of( + "type", + "keyword", + "normalizer", + "keyword_normalizer", + "fields", + Map.of("keyword", Map.of("type", "keyword")))); + + StructuredPropertyDefinition propWithNumericType = + new StructuredPropertyDefinition() + .setQualifiedName("testPropNumber") + .setDisplayName("examplePropNumber") + .setEntityTypes( + new UrnArray( + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "dataset"), + Urn.createFromString(ENTITY_TYPE_URN_PREFIX + "testEntity"))) + .setValueType(Urn.createFromString("urn:li:logicalType:NUMBER")); + Map structuredPropertyFieldMappingsNumber = + MappingsBuilder.getMappingsForStructuredProperty(List.of(propWithNumericType)); + assertEquals(structuredPropertyFieldMappingsNumber.size(), 1); + keyInMap = structuredPropertyFieldMappingsNumber.keySet().stream().findFirst().get(); + assertEquals("testPropNumber", keyInMap); + mappings = structuredPropertyFieldMappingsNumber.get(keyInMap); + assertEquals(Map.of("type", "double"), mappings); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 6269827104faf..9e8855622ced4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -13,6 +13,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.testng.Assert; import org.testng.annotations.Test; @@ -20,7 +21,6 @@ public class AggregationQueryBuilderTest { @Test public void testGetDefaultAggregationsHasFields() { - SearchableAnnotation annotation = new SearchableAnnotation( "test", @@ -82,7 +82,6 @@ public void testGetDefaultAggregationsFields() { @Test public void testGetSpecificAggregationsHasFields() { - SearchableAnnotation annotation1 = new SearchableAnnotation( "test1", @@ -135,6 +134,100 @@ public void testGetSpecificAggregationsHasFields() { Assert.assertEquals(aggs.size(), 0); } + @Test + public void testAggregateOverStructuredProperty() { + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = new AggregationQueryBuilder(config, List.of()); + + List aggs = + builder.getAggregations(List.of("structuredProperties.ab.fgh.ten")); + Assert.assertEquals(aggs.size(), 1); + AggregationBuilder aggBuilder = aggs.get(0); + Assert.assertTrue(aggBuilder instanceof TermsAggregationBuilder); + TermsAggregationBuilder agg = (TermsAggregationBuilder) aggBuilder; + // Check that field name is sanitized to correct field name + Assert.assertEquals(agg.field(), "structuredProperties.ab_fgh_ten"); + + // Two structured properties + aggs = + builder.getAggregations( + List.of("structuredProperties.ab.fgh.ten", "structuredProperties.hello")); + Assert.assertEquals(aggs.size(), 2); + Assert.assertEquals( + aggs.stream() + .map(aggr -> ((TermsAggregationBuilder) aggr).field()) + .collect(Collectors.toSet()), + Set.of("structuredProperties.ab_fgh_ten", "structuredProperties.hello")); + } + + @Test + public void testAggregateOverFieldsAndStructProp() { + SearchableAnnotation annotation1 = + new SearchableAnnotation( + "test1", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.empty(), + Optional.of("Has Test"), + 1.0, + Optional.of("hasTest1"), + Optional.empty(), + Collections.emptyMap(), + Collections.emptyList(), + false); + + SearchableAnnotation annotation2 = + new SearchableAnnotation( + "test2", + SearchableAnnotation.FieldType.KEYWORD, + true, + true, + false, + false, + Optional.of("Test Filter"), + Optional.empty(), + 1.0, + Optional.empty(), + Optional.empty(), + Collections.emptyMap(), + Collections.emptyList(), + false); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = + new AggregationQueryBuilder(config, ImmutableList.of(annotation1, annotation2)); + + // Aggregate over fields and structured properties + List aggs = + builder.getAggregations( + ImmutableList.of( + "test1", + "test2", + "hasTest1", + "structuredProperties.ab.fgh.ten", + "structuredProperties.hello")); + Assert.assertEquals(aggs.size(), 5); + Set facets = + aggs.stream() + .map(aggB -> ((TermsAggregationBuilder) aggB).field()) + .collect(Collectors.toSet()); + Assert.assertEquals( + facets, + ImmutableSet.of( + "test1.keyword", + "test2.keyword", + "hasTest1", + "structuredProperties.ab_fgh_ten", + "structuredProperties.hello")); + } + @Test public void testMissingAggregation() { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java index 105ee2652dc30..47d18fe0d299c 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/CustomizedQueryHandlerTest.java @@ -1,8 +1,10 @@ package com.linkedin.metadata.search.query.request; +import static com.linkedin.metadata.Constants.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.metadata.config.search.CustomConfiguration; @@ -30,6 +32,14 @@ public class CustomizedQueryHandlerTest { static { try { + int maxSize = + Integer.parseInt( + System.getenv() + .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); + TEST_MAPPER + .getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder().maxStringLength(maxSize).build()); CustomConfiguration customConfiguration = new CustomConfiguration(); customConfiguration.setEnabled(true); customConfiguration.setFile("search_config_test.yml"); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 8cb28d3658ee7..38d630bc302f4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -140,7 +140,8 @@ public void testQueryBuilderFulltext() { "urn.delimited", 7.0f, "textArrayField.delimited", 0.4f, "nestedArrayStringField.delimited", 0.4f, - "wordGramField.delimited", 0.4f)); + "wordGramField.delimited", 0.4f, + "customProperties.delimited", 0.4f)); BoolQueryBuilder boolPrefixQuery = (BoolQueryBuilder) shouldQueries.get(1); assertTrue(boolPrefixQuery.should().size() > 0); @@ -165,7 +166,7 @@ public void testQueryBuilderFulltext() { }) .collect(Collectors.toList()); - assertEquals(prefixFieldWeights.size(), 28); + assertEquals(prefixFieldWeights.size(), 29); List.of( Pair.of("urn", 100.0f), @@ -200,7 +201,7 @@ public void testQueryBuilderStructured() { assertEquals(keywordQuery.queryString(), "testQuery"); assertNull(keywordQuery.analyzer()); Map keywordFields = keywordQuery.fields(); - assertEquals(keywordFields.size(), 21); + assertEquals(keywordFields.size(), 22); assertEquals(keywordFields.get("keyPart1").floatValue(), 10.0f); assertFalse(keywordFields.containsKey("keyPart3")); assertEquals(keywordFields.get("textFieldOverride").floatValue(), 1.0f); @@ -360,7 +361,7 @@ public void testGetStandardFieldsEntitySpec() { public void testGetStandardFields() { Set fieldConfigs = TEST_CUSTOM_BUILDER.getStandardFields(ImmutableList.of(TestEntitySpecBuilder.getSpec())); - assertEquals(fieldConfigs.size(), 21); + assertEquals(fieldConfigs.size(), 22); assertEquals( fieldConfigs.stream().map(SearchFieldConfig::fieldName).collect(Collectors.toSet()), Set.of( @@ -384,7 +385,8 @@ public void testGetStandardFields() { "wordGramField.wordGrams3", "textFieldOverride.delimited", "urn", - "wordGramField.wordGrams2")); + "wordGramField.wordGrams2", + "customProperties.delimited")); // customProperties.delimited Saas only assertEquals( fieldConfigs.stream() @@ -467,9 +469,9 @@ public void testGetStandardFields() { fieldConfigs = TEST_CUSTOM_BUILDER.getStandardFields( ImmutableList.of(TestEntitySpecBuilder.getSpec(), mockEntitySpec)); - // Same 21 from the original entity + newFieldNotInOriginal + 3 word gram fields from the + // Same 22 from the original entity + newFieldNotInOriginal + 3 word gram fields from the // textFieldOverride - assertEquals(fieldConfigs.size(), 26); + assertEquals(fieldConfigs.size(), 27); assertEquals( fieldConfigs.stream().map(SearchFieldConfig::fieldName).collect(Collectors.toSet()), Set.of( @@ -498,7 +500,8 @@ public void testGetStandardFields() { "fieldDoesntExistInOriginal.delimited", "textFieldOverride.wordGrams2", "textFieldOverride.wordGrams3", - "textFieldOverride.wordGrams4")); + "textFieldOverride.wordGrams4", + "customProperties.delimited")); // Field which only exists in first one: Should be the same assertEquals( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 03abd9ffe29d7..980b82194536e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -252,4 +252,75 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { + "}"; Assert.assertEquals(result.toString(), expected); } + + @Test + public void testGetQueryBuilderFromStructPropEqualsValue() { + + final Criterion singleValueCriterion = + new Criterion() + .setField("structuredProperties.ab.fgh.ten") + .setCondition(Condition.EQUAL) + .setValues(new StringArray(ImmutableList.of("value1"))); + + QueryBuilder result = ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false); + String expected = + "{\n" + + " \"terms\" : {\n" + + " \"structuredProperties.ab_fgh_ten\" : [\n" + + " \"value1\"\n" + + " ],\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"structuredProperties.ab_fgh_ten\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + } + + @Test + public void testGetQueryBuilderFromStructPropExists() { + final Criterion singleValueCriterion = + new Criterion().setField("structuredProperties.ab.fgh.ten").setCondition(Condition.EXISTS); + + QueryBuilder result = ESUtils.getQueryBuilderFromCriterion(singleValueCriterion, false); + String expected = + "{\n" + + " \"bool\" : {\n" + + " \"must\" : [\n" + + " {\n" + + " \"exists\" : {\n" + + " \"field\" : \"structuredProperties.ab_fgh_ten\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"adjust_pure_negative\" : true,\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"structuredProperties.ab_fgh_ten\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + + // No diff in the timeseries field case for this condition. + final Criterion timeseriesField = + new Criterion().setField("myTestField").setCondition(Condition.EXISTS); + + result = ESUtils.getQueryBuilderFromCriterion(timeseriesField, true); + expected = + "{\n" + + " \"bool\" : {\n" + + " \"must\" : [\n" + + " {\n" + + " \"exists\" : {\n" + + " \"field\" : \"myTestField\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"adjust_pure_negative\" : true,\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"myTestField\"\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + } } diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index 58ea020e42565..a22a774065852 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -10,9 +10,9 @@ import com.linkedin.datahub.graphql.generated.AutoCompleteResults; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.FilterOperator; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.search.LineageSearchResult; diff --git a/metadata-io/src/test/resources/forms/form_assignment_test_definition_complex.json b/metadata-io/src/test/resources/forms/form_assignment_test_definition_complex.json new file mode 100644 index 0000000000000..e68cbbd9aeff0 --- /dev/null +++ b/metadata-io/src/test/resources/forms/form_assignment_test_definition_complex.json @@ -0,0 +1,145 @@ +{ + "on": { + "types": ["dataset", "container", "dataJob", "dataFlow", "chart", "dashboard"], + "conditions": { + "or": [ + { + "or": [ + { + "property": "forms.incompleteForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + }, + { + "property": "forms.completedForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + } + ] + }, + { + "or": [ + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:hive"] + }, + { + "property": "container.container", + "operator": "equals", + "values": ["urn:li:container:test"] + }, + { + "property": "entityType", + "operator": "equals", + "values": ["dataset"] + }, + { + "property": "domains.domains", + "operator": "equals", + "values": ["urn:li:domain:test"] + } + ] + }, + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:snowflake"] + }, + { + "property": "container.container", + "operator": "equals", + "values": ["urn:li:container:test-2"] + }, + { + "property": "entityType", + "operator": "equals", + "values": ["dashboard"] + }, + { + "property": "domains.domains", + "operator": "equals", + "values": ["urn:li:domain:test-2"] + } + ] + } + ] + } + ] + } + }, + "rules": { + "or": [ + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:hive"] + }, + { + "property": "container.container", + "operator": "equals", + "values": ["urn:li:container:test"] + }, + { + "property": "entityType", + "operator": "equals", + "values": ["dataset"] + }, + { + "property": "domains.domains", + "operator": "equals", + "values": ["urn:li:domain:test"] + } + ] + }, + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:snowflake"] + }, + { + "property": "container.container", + "operator": "equals", + "values": ["urn:li:container:test-2"] + }, + { + "property": "entityType", + "operator": "equals", + "values": ["dashboard"] + }, + { + "property": "domains.domains", + "operator": "equals", + "values": ["urn:li:domain:test-2"] + } + ] + } + ] + }, + "actions": { + "passing": [ + { + "type": "ASSIGN_FORM", + "params": { + "formUrn": "urn:li:form:test" + } + } + ], + "failing": [ + { + "type": "UNASSIGN_FORM", + "params": { + "formUrn": "urn:li:form:test" + } + } + ] + } +} \ No newline at end of file diff --git a/metadata-io/src/test/resources/forms/form_assignment_test_definition_simple.json b/metadata-io/src/test/resources/forms/form_assignment_test_definition_simple.json new file mode 100644 index 0000000000000..a09fbc801414c --- /dev/null +++ b/metadata-io/src/test/resources/forms/form_assignment_test_definition_simple.json @@ -0,0 +1,67 @@ +{ + "on": { + "types": ["dataset", "container", "dataJob", "dataFlow", "chart", "dashboard"], + "conditions": { + "or": [ + { + "or": [ + { + "property": "forms.incompleteForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + }, + { + "property": "forms.completedForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + } + ] + }, + { + "or": [ + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:hive"] + } + ] + } + ] + } + ] + } + }, + "rules": { + "or": [ + { + "and": [ + { + "property": "dataPlatformInstance.platform", + "operator": "equals", + "values": ["urn:li:dataPlatform:hive"] + } + ] + } + ] + }, + "actions": { + "passing": [ + { + "type": "ASSIGN_FORM", + "params": { + "formUrn": "urn:li:form:test" + } + } + ], + "failing": [ + { + "type": "UNASSIGN_FORM", + "params": { + "formUrn": "urn:li:form:test" + } + } + ] + } +} \ No newline at end of file diff --git a/metadata-io/src/test/resources/forms/form_prompt_test_definition.json b/metadata-io/src/test/resources/forms/form_prompt_test_definition.json new file mode 100644 index 0000000000000..d797db7e25180 --- /dev/null +++ b/metadata-io/src/test/resources/forms/form_prompt_test_definition.json @@ -0,0 +1,39 @@ +{ + "on": { + "types": ["dataset", "container", "dataJob", "dataFlow", "chart", "dashboard"], + "conditions": { + "or": [ + { + "property": "forms.incompleteForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + }, + { + "property": "forms.completedForms.urn", + "operator": "equals", + "values": ["urn:li:form:test"] + } + ] + } + }, + "rules": { + "and": [ + { + "property": "structuredProperties.urn:li:structuredProperty:test.id", + "operator": "exists" + } + ] + }, + "actions": { + "passing": [], + "failing": [ + { + "type": "SET_FORM_PROMPT_INCOMPLETE", + "params": { + "formUrn": "urn:li:form:test", + "formPromptId": "test-id" + } + } + ] + } +} \ No newline at end of file diff --git a/metadata-jobs/common/build.gradle b/metadata-jobs/common/build.gradle index bdc3b7a44a98a..b0a3a6827b729 100644 --- a/metadata-jobs/common/build.gradle +++ b/metadata-jobs/common/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' } dependencies { diff --git a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java index e695788e09726..ae208c053d69f 100644 --- a/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java +++ b/metadata-jobs/mae-consumer-job/src/main/java/com/linkedin/metadata/kafka/MaeConsumerApplication.java @@ -14,14 +14,14 @@ exclude = {ElasticsearchRestClientAutoConfiguration.class, CassandraAutoConfiguration.class}) @ComponentScan( basePackages = { - // "com.linkedin.gms.factory.config", - // "com.linkedin.gms.factory.common", "com.linkedin.gms.factory.kafka", "com.linkedin.metadata.boot.kafka", "com.linkedin.metadata.kafka", "com.linkedin.metadata.dao.producer", "com.linkedin.gms.factory.config", "com.linkedin.gms.factory.entity.update.indices", + "com.linkedin.gms.factory.entityclient", + "com.linkedin.gms.factory.form", "com.linkedin.gms.factory.timeline.eventgenerator", "io.datahubproject.metadata.jobs.common.health.kafka" }, diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/application.properties b/metadata-jobs/mae-consumer-job/src/main/resources/application.properties index 7df61c93ab66d..f8b979e6fbac0 100644 --- a/metadata-jobs/mae-consumer-job/src/main/resources/application.properties +++ b/metadata-jobs/mae-consumer-job/src/main/resources/application.properties @@ -3,4 +3,4 @@ management.endpoints.web.exposure.include=metrics, health, info spring.mvc.servlet.path=/ management.health.elasticsearch.enabled=false management.health.neo4j.enabled=false -entityClient.preferredImpl=restli +entityClient.impl=restli diff --git a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java index 7135e4e44d459..b409a41600bd7 100644 --- a/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mae-consumer-job/src/test/java/com/linkedin/metadata/kafka/MaeConsumerApplicationTestConfiguration.java @@ -1,6 +1,5 @@ package com.linkedin.metadata.kafka; -import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; import com.linkedin.metadata.entity.EntityServiceImpl; @@ -22,8 +21,6 @@ public class MaeConsumerApplicationTestConfiguration { @MockBean private EntityServiceImpl _entityServiceImpl; - @MockBean private SystemRestliEntityClient restliEntityClient; - @MockBean private Database ebeanServer; @MockBean private EntityRegistry entityRegistry; diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java index f2eeef6e2c8e6..278c52030b5fc 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeLogProcessor.java @@ -9,6 +9,7 @@ import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; import com.linkedin.metadata.kafka.hook.UpdateIndicesHook; import com.linkedin.metadata.kafka.hook.event.EntityChangeEventGeneratorHook; +import com.linkedin.metadata.kafka.hook.form.FormAssignmentHook; import com.linkedin.metadata.kafka.hook.ingestion.IngestionSchedulerHook; import com.linkedin.metadata.kafka.hook.siblings.SiblingAssociationHook; import com.linkedin.metadata.utils.metrics.MetricUtils; @@ -36,7 +37,8 @@ IngestionSchedulerHook.class, EntityChangeEventGeneratorHook.class, KafkaEventConsumerFactory.class, - SiblingAssociationHook.class + SiblingAssociationHook.class, + FormAssignmentHook.class }) @EnableKafka public class MetadataChangeLogProcessor { @@ -95,6 +97,7 @@ public void consume(final ConsumerRecord consumerRecord) // Here - plug in additional "custom processor hooks" for (MetadataChangeLogHook hook : this.hooks) { if (!hook.isEnabled()) { + log.debug(String.format("Skipping disabled hook %s", hook.getClass())); continue; } try (Timer.Context ignored = @@ -102,7 +105,7 @@ public void consume(final ConsumerRecord consumerRecord) .time()) { hook.invoke(event); } catch (Exception e) { - // Just skip this hook and continue. - Note that this represents "at most once" + // Just skip this hook and continue. - Note that this represents "at most once"// // processing. MetricUtils.counter(this.getClass(), hook.getClass().getSimpleName() + "_failure").inc(); log.error( diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityHydratorConfig.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityHydratorConfig.java index 036968f9f6759..d8a959c0be624 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityHydratorConfig.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/config/EntityHydratorConfig.java @@ -1,23 +1,17 @@ package com.linkedin.metadata.kafka.config; import com.google.common.collect.ImmutableSet; -import com.linkedin.entity.client.SystemRestliEntityClient; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.kafka.hydrator.EntityHydrator; import com.linkedin.metadata.models.registry.EntityRegistry; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; @Configuration -@Import({RestliEntityClientFactory.class}) public class EntityHydratorConfig { - @Autowired - @Qualifier("systemRestliEntityClient") - private SystemRestliEntityClient _entityClient; + @Autowired private SystemEntityClient entityClient; @Autowired private EntityRegistry _entityRegistry; @@ -34,6 +28,6 @@ public class EntityHydratorConfig { @Bean public EntityHydrator getEntityHydrator() { - return new EntityHydrator(_entityRegistry, _entityClient); + return new EntityHydrator(_entityRegistry, entityClient); } } diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java index f3b5a09708cee..375d1580dab51 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHook.java @@ -6,8 +6,7 @@ import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.SetMode; -import com.linkedin.entity.client.SystemRestliEntityClient; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.metadata.Constants; import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; @@ -43,7 +42,7 @@ */ @Slf4j @Component -@Import({EntityRegistryFactory.class, RestliEntityClientFactory.class}) +@Import({EntityRegistryFactory.class}) public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { /** The list of aspects that are supported for generating semantic change events. */ @@ -78,7 +77,7 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { ImmutableSet.of("CREATE", "UPSERT", "DELETE"); private final EntityChangeEventGeneratorRegistry _entityChangeEventGeneratorRegistry; - private final SystemRestliEntityClient _entityClient; + private final SystemEntityClient _entityClient; private final EntityRegistry _entityRegistry; private final Boolean _isEnabled; @@ -86,7 +85,7 @@ public class EntityChangeEventGeneratorHook implements MetadataChangeLogHook { public EntityChangeEventGeneratorHook( @Nonnull @Qualifier("entityChangeEventGeneratorRegistry") final EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry, - @Nonnull final SystemRestliEntityClient entityClient, + @Nonnull final SystemEntityClient entityClient, @Nonnull final EntityRegistry entityRegistry, @Nonnull @Value("${entityChangeEvents.enabled:true}") Boolean isEnabled) { _entityChangeEventGeneratorRegistry = diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java new file mode 100644 index 0000000000000..91e8e186b07f7 --- /dev/null +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/form/FormAssignmentHook.java @@ -0,0 +1,130 @@ +package com.linkedin.metadata.kafka.hook.form; + +import static com.linkedin.metadata.Constants.*; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; +import com.linkedin.gms.factory.form.FormServiceFactory; +import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; +import com.linkedin.metadata.service.FormService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeLog; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.inject.Singleton; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Import; +import org.springframework.stereotype.Component; + +/** + * This hook is used for assigning / un-assigning forms for specific entities. + * + *

Specifically, this hook performs the following operations: + * + *

1. When a new dynamic form assignment is created, an automation (metadata test) with the form + * urn embedded is automatically generated, which is responsible for assigning the form to any + * entities in the target set. It also will attempt a removal of the form for any failing entities. + * + *

2. When a new form is created, or an existing one updated, automations (metadata tests) will + * be generated for each prompt in the metadata test which verifies that the entities with that test + * associated with it are complying with the prompt. When they are NOT, the test will mark the + * prompts as incomplete. + * + *

3. When a form is hard deleted, any automations used for assigning the form, or validating + * prompts, are automatically deleted. + * + *

Note that currently, Datasets, Dashboards, Charts, Data Jobs, Data Flows, Containers, are the + * only asset types supported for this hook. + * + *

TODO: In the future, let's decide whether we want to support automations to auto-mark form + * prompts as "completed" when they do in fact have the correct metadata. (Without user needing to + * explicitly fill out a form prompt response) + * + *

TODO: Write a unit test for this class. + */ +@Slf4j +@Component +@Singleton +@Import({FormServiceFactory.class, SystemAuthenticationFactory.class}) +public class FormAssignmentHook implements MetadataChangeLogHook { + + private static final Set SUPPORTED_UPDATE_TYPES = + ImmutableSet.of(ChangeType.UPSERT, ChangeType.CREATE, ChangeType.RESTATE); + + private final FormService _formService; + private final boolean _isEnabled; + + @Autowired + public FormAssignmentHook( + @Nonnull final FormService formService, + @Nonnull @Value("${forms.hook.enabled:true}") Boolean isEnabled) { + _formService = Objects.requireNonNull(formService, "formService is required"); + _isEnabled = isEnabled; + } + + @Override + public void init() {} + + @Override + public boolean isEnabled() { + return _isEnabled; + } + + @Override + public void invoke(@Nonnull final MetadataChangeLog event) { + if (_isEnabled && isEligibleForProcessing(event)) { + if (isFormDynamicFilterUpdated(event)) { + handleFormFilterUpdated(event); + } + } + } + + /** Handle an form filter update by adding updating the targeting automation for it. */ + private void handleFormFilterUpdated(@Nonnull final MetadataChangeLog event) { + // 1. Get the new form assignment + DynamicFormAssignment formFilters = + GenericRecordUtils.deserializeAspect( + event.getAspect().getValue(), + event.getAspect().getContentType(), + DynamicFormAssignment.class); + + // 2. Register a automation to assign it. + _formService.upsertFormAssignmentRunner(event.getEntityUrn(), formFilters); + } + + /** + * Returns true if the event should be processed, which is only true if the change is on the + * incident status aspect + */ + private boolean isEligibleForProcessing(@Nonnull final MetadataChangeLog event) { + return isFormPromptSetUpdated(event) + || isFormDynamicFilterUpdated(event) + || isFormDeleted(event); + } + + /** Returns true if an form is being hard-deleted. */ + private boolean isFormDeleted(@Nonnull final MetadataChangeLog event) { + return FORM_ENTITY_NAME.equals(event.getEntityType()) + && ChangeType.DELETE.equals(event.getChangeType()) + && FORM_KEY_ASPECT_NAME.equals(event.getAspectName()); + } + + /** Returns true if the event represents an update the prompt set of a form. */ + private boolean isFormPromptSetUpdated(@Nonnull final MetadataChangeLog event) { + return FORM_ENTITY_NAME.equals(event.getEntityType()) + && SUPPORTED_UPDATE_TYPES.contains(event.getChangeType()) + && FORM_INFO_ASPECT_NAME.equals(event.getAspectName()); + } + + /** Returns true if the event represents an update to the dynamic filter for a form. */ + private boolean isFormDynamicFilterUpdated(@Nonnull final MetadataChangeLog event) { + return FORM_ENTITY_NAME.equals(event.getEntityType()) + && SUPPORTED_UPDATE_TYPES.contains(event.getChangeType()) + && DYNAMIC_FORM_ASSIGNMENT_ASPECT_NAME.equals(event.getAspectName()); + } +} diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java index 67198d13772a3..7a1aaa7f6a056 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHook.java @@ -14,9 +14,9 @@ import com.linkedin.dataset.UpstreamArray; import com.linkedin.dataset.UpstreamLineage; import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.search.EntitySearchServiceFactory; import com.linkedin.metadata.Constants; @@ -72,14 +72,14 @@ public class SiblingAssociationHook implements MetadataChangeLogHook { public static final String SOURCE_SUBTYPE_V2 = "Source"; private final EntityRegistry _entityRegistry; - private final SystemRestliEntityClient _entityClient; + private final SystemEntityClient _entityClient; private final EntitySearchService _searchService; private final boolean _isEnabled; @Autowired public SiblingAssociationHook( @Nonnull final EntityRegistry entityRegistry, - @Nonnull final SystemRestliEntityClient entityClient, + @Nonnull final SystemEntityClient entityClient, @Nonnull final EntitySearchService searchService, @Nonnull @Value("${siblings.enabled:true}") Boolean isEnabled) { _entityRegistry = entityRegistry; diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityHydrator.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityHydrator.java index 7a8fdd11fac43..6ad7cdbcad3e6 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityHydrator.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/hydrator/EntityHydrator.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.urn.Urn; import com.linkedin.entity.EntityResponse; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.r2.RemoteInvocationException; @@ -24,7 +24,7 @@ public class EntityHydrator { private final EntityRegistry _entityRegistry; - private final SystemRestliEntityClient _entityClient; + private final SystemEntityClient entityClient; private final ChartHydrator _chartHydrator = new ChartHydrator(); private final CorpUserHydrator _corpUserHydrator = new CorpUserHydrator(); private final DashboardHydrator _dashboardHydrator = new DashboardHydrator(); @@ -55,7 +55,7 @@ public Optional getHydratedEntity(String entityTypeName, String urn) .collect(Collectors.toSet())) .orElse(Set.of()); entityResponse = - _entityClient.batchGetV2(Collections.singleton(urnObj), aspectNames).get(urnObj); + entityClient.batchGetV2(Collections.singleton(urnObj), aspectNames).get(urnObj); } catch (RemoteInvocationException | URISyntaxException e) { log.error("Error while calling GMS to hydrate entity for urn {}", urn); return Optional.empty(); diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java index a227668e22e9b..89ad6105be9cb 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/UpdateIndicesHookTest.java @@ -28,6 +28,7 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener; +import com.linkedin.metadata.client.EntityClientAspectRetriever; import com.linkedin.metadata.config.SystemUpdateConfiguration; import com.linkedin.metadata.config.search.ElasticSearchConfiguration; import com.linkedin.metadata.graph.Edge; @@ -121,9 +122,10 @@ public void setupTest() { _mockEntitySearchService, _mockTimeseriesAspectService, _mockSystemMetadataService, - ENTITY_REGISTRY, _searchDocumentTransformer, _mockEntityIndexBuilders); + _updateIndicesService.initializeAspectRetriever( + EntityClientAspectRetriever.builder().entityRegistry(ENTITY_REGISTRY).build()); _updateIndicesHook = new UpdateIndicesHook(_updateIndicesService, true); } @@ -198,9 +200,10 @@ public void testInputFieldsEdgesAreAdded() throws Exception { _mockEntitySearchService, _mockTimeseriesAspectService, _mockSystemMetadataService, - mockEntityRegistry, _searchDocumentTransformer, _mockEntityIndexBuilders); + _updateIndicesService.initializeAspectRetriever( + EntityClientAspectRetriever.builder().entityRegistry(mockEntityRegistry).build()); _updateIndicesHook = new UpdateIndicesHook(_updateIndicesService, true); _updateIndicesHook.invoke(event); diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java index 8400e19ce49a3..021186404b2cb 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/event/EntityChangeEventGeneratorHookTest.java @@ -41,7 +41,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.key.DatasetKey; @@ -93,14 +93,14 @@ public class EntityChangeEventGeneratorHookTest { private static final String TEST_DATA_JOB_URN = "urn:li:dataJob:job"; private Urn actorUrn; - private SystemRestliEntityClient _mockClient; + private SystemEntityClient _mockClient; private EntityService _mockEntityService; private EntityChangeEventGeneratorHook _entityChangeEventHook; @BeforeMethod public void setupTest() throws URISyntaxException { actorUrn = Urn.createFromString(TEST_ACTOR_URN); - _mockClient = Mockito.mock(SystemRestliEntityClient.class); + _mockClient = Mockito.mock(SystemEntityClient.class); _mockEntityService = Mockito.mock(EntityService.class); EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry = createEntityChangeEventGeneratorRegistry(); @@ -776,12 +776,12 @@ private EntityRegistry createMockEntityRegistry() { } private void verifyProducePlatformEvent( - SystemRestliEntityClient mockClient, PlatformEvent platformEvent) throws Exception { + SystemEntityClient mockClient, PlatformEvent platformEvent) throws Exception { verifyProducePlatformEvent(mockClient, platformEvent, true); } private void verifyProducePlatformEvent( - SystemRestliEntityClient mockClient, PlatformEvent platformEvent, boolean noMoreInteractions) + SystemEntityClient mockClient, PlatformEvent platformEvent, boolean noMoreInteractions) throws Exception { // Verify event has been emitted. verify(mockClient, Mockito.times(1)) diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java index d4c6d122a6689..3823668adeace 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/siblings/SiblingAssociationHookTest.java @@ -21,7 +21,7 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.key.DatasetKey; import com.linkedin.metadata.models.registry.ConfigEntityRegistry; @@ -41,7 +41,7 @@ public class SiblingAssociationHookTest { private SiblingAssociationHook _siblingAssociationHook; - SystemRestliEntityClient _mockEntityClient; + SystemEntityClient _mockEntityClient; EntitySearchService _mockSearchService; @BeforeMethod @@ -51,7 +51,7 @@ public void setupTest() { SiblingAssociationHookTest.class .getClassLoader() .getResourceAsStream("test-entity-registry-siblings.yml")); - _mockEntityClient = Mockito.mock(SystemRestliEntityClient.class); + _mockEntityClient = Mockito.mock(SystemEntityClient.class); _mockSearchService = Mockito.mock(EntitySearchService.class); _siblingAssociationHook = new SiblingAssociationHook(registry, _mockEntityClient, _mockSearchService, true); diff --git a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java index 44b2ce54e19c8..fc47679bebd39 100644 --- a/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java +++ b/metadata-jobs/mae-consumer/src/test/java/com/linkedin/metadata/kafka/hook/spring/MCLSpringTestConfiguration.java @@ -1,8 +1,11 @@ package com.linkedin.metadata.kafka.hook.spring; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.datahub.authentication.Authentication; import com.datahub.metadata.ingestion.IngestionScheduler; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.kafka.schemaregistry.SchemaRegistryConfig; import com.linkedin.metadata.boot.kafka.DataHubUpgradeKafkaListener; import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; @@ -14,7 +17,9 @@ import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import org.apache.avro.generic.GenericRecord; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; @@ -40,12 +45,18 @@ public class MCLSpringTestConfiguration { @MockBean public IngestionScheduler ingestionScheduler; - @MockBean(name = "systemRestliEntityClient") - public SystemRestliEntityClient entityClient; + @Bean + public SystemEntityClient systemEntityClient( + @Qualifier("systemAuthentication") Authentication systemAuthentication) { + SystemEntityClient systemEntityClient = mock(SystemEntityClient.class); + when(systemEntityClient.getSystemAuthentication()).thenReturn(systemAuthentication); + return systemEntityClient; + } @MockBean public ElasticSearchService searchService; - @MockBean public Authentication systemAuthentication; + @MockBean(name = "systemAuthentication") + public Authentication systemAuthentication; @MockBean(name = "dataHubUpgradeKafkaListener") public DataHubUpgradeKafkaListener dataHubUpgradeKafkaListener; diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java index 181a723e1cd25..1210bf37059b4 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/kafka/MceConsumerApplication.java @@ -1,6 +1,5 @@ package com.linkedin.metadata.kafka; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.telemetry.ScheduledAnalyticsFactory; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import org.springframework.boot.SpringApplication; @@ -22,6 +21,7 @@ "com.linkedin.gms.factory.config", "com.linkedin.gms.factory.entity", "com.linkedin.gms.factory.entityregistry", + "com.linkedin.gms.factory.entityclient", "com.linkedin.gms.factory.kafka", "com.linkedin.gms.factory.search", "com.linkedin.gms.factory.secret", @@ -30,12 +30,14 @@ "com.linkedin.metadata.restli", "com.linkedin.metadata.kafka", "com.linkedin.metadata.dao.producer", + "com.linkedin.gms.factory.form", + "com.linkedin.metadata.dao.producer", "io.datahubproject.metadata.jobs.common.health.kafka" }, excludeFilters = { @ComponentScan.Filter( type = FilterType.ASSIGNABLE_TYPE, - classes = {ScheduledAnalyticsFactory.class, RestliEntityClientFactory.class}) + classes = {ScheduledAnalyticsFactory.class}) }) @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class MceConsumerApplication { diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java index a4747c72c20fa..b41e6bc75af19 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java @@ -1,12 +1,8 @@ package com.linkedin.metadata.restli; import com.datahub.auth.authentication.filter.AuthenticationFilter; -import com.linkedin.entity.client.RestliEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; -import com.linkedin.parseq.retry.backoff.ExponentialBackoff; -import com.linkedin.restli.client.Client; import com.linkedin.restli.server.RestliHandlerServlet; -import java.net.URI; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.servlet.FilterRegistrationBean; @@ -14,7 +10,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; @Configuration @Import({SystemAuthenticationFactory.class}) @@ -29,14 +24,6 @@ public class RestliServletConfig { @Value("${entityClient.numRetries:3}") private int numRetries; - @Bean("restliEntityClient") - @Primary - public RestliEntityClient restliEntityClient() { - String selfUri = String.format("http://localhost:%s/gms/", configuredPort); - final Client restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(selfUri), null); - return new RestliEntityClient(restClient, new ExponentialBackoff(retryInterval), numRetries); - } - @Bean("restliServletRegistration") public ServletRegistrationBean restliServletRegistration( RestliHandlerServlet servlet) { diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java index 6d19db97fb39f..bce8664689e2c 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTest.java @@ -22,7 +22,7 @@ public class MceConsumerApplicationTest extends AbstractTestNGSpringContextTests @Autowired private TestRestTemplate restTemplate; - @Autowired private EntityService _mockEntityService; + @Autowired private EntityService _mockEntityService; @Autowired private KafkaHealthIndicator kafkaHealthIndicator; diff --git a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java index 1a44265c7a92a..93a6ae8fb4797 100644 --- a/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java +++ b/metadata-jobs/mce-consumer-job/src/test/java/com/linkedin/metadata/kafka/MceConsumerApplicationTestConfiguration.java @@ -1,7 +1,10 @@ package com.linkedin.metadata.kafka; -import com.linkedin.entity.client.RestliEntityClient; +import com.datahub.authentication.Authentication; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.entity.client.SystemRestliEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; +import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.dao.producer.KafkaHealthChecker; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.SiblingGraphService; @@ -15,6 +18,7 @@ import io.ebean.Database; import java.net.URI; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.web.client.TestRestTemplate; @@ -30,14 +34,21 @@ public class MceConsumerApplicationTestConfiguration { @MockBean public KafkaHealthChecker kafkaHealthChecker; - @MockBean public EntityService _entityService; + @MockBean public EntityService _entityService; - @Bean("restliEntityClient") + @Bean @Primary - public RestliEntityClient restliEntityClient() { + public SystemEntityClient systemEntityClient( + @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider, + @Qualifier("systemAuthentication") final Authentication systemAuthentication) { String selfUri = restTemplate.getRootUri(); final Client restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(selfUri), null); - return new RestliEntityClient(restClient, new ExponentialBackoff(1), 1); + return new SystemRestliEntityClient( + restClient, + new ExponentialBackoff(1), + 1, + systemAuthentication, + configurationProvider.getCache().getClient().getEntityClient()); } @MockBean public Database ebeanServer; diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index 5fa65c06de714..49604924acb68 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -53,4 +53,4 @@ processResources.dependsOn avroSchemaSources clean { project.delete("src/main/resources/avro") -} \ No newline at end of file +} diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index e22a8ba813704..352fa93f56a04 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -5,8 +5,8 @@ import com.codahale.metrics.Timer; import com.datahub.authentication.Authentication; import com.linkedin.entity.Entity; -import com.linkedin.entity.client.SystemRestliEntityClient; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.metadata.EventUtils; @@ -48,7 +48,7 @@ public class MetadataChangeEventsProcessor { @NonNull private final Authentication systemAuthentication; - private final SystemRestliEntityClient entityClient; + private final SystemEntityClient entityClient; private final Producer kafkaProducer; private final Histogram kafkaLagStats = diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java index 26d5f66f4929a..a4f5a287bc8fd 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java @@ -3,8 +3,8 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import com.linkedin.entity.client.SystemRestliEntityClient; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; import com.linkedin.gms.factory.kafka.KafkaEventConsumerFactory; import com.linkedin.metadata.EventUtils; @@ -42,7 +42,7 @@ @RequiredArgsConstructor public class MetadataChangeProposalsProcessor { - private final SystemRestliEntityClient entityClient; + private final SystemEntityClient entityClient; private final Producer kafkaProducer; private final Histogram kafkaLagStats = diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java index b61858aef22cd..955d5c67c09a7 100644 --- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java @@ -46,7 +46,7 @@ public PlatformEventProcessor() { public void consume(final ConsumerRecord consumerRecord) { try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) { - log.info("Consuming a Platform Event"); + log.debug("Consuming a Platform Event"); kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp()); final GenericRecord record = consumerRecord.value(); diff --git a/metadata-models-custom/README.md b/metadata-models-custom/README.md index 94399a67806a6..10801c3d8ed23 100644 --- a/metadata-models-custom/README.md +++ b/metadata-models-custom/README.md @@ -396,6 +396,26 @@ public class CustomDataQualityRulesMCLSideEffect extends MCLSideEffect { return timeseriesOptional.stream(); } + + private Optional buildEvent(MetadataChangeLog originMCP) { + if (originMCP.getAspect() != null) { + DataQualityRuleEvent event = new DataQualityRuleEvent(); + if (event.getActor() != null) { + event.setActor(event.getActor()); + } + event.setEventTimestamp(originMCP.getSystemMetadata().getLastObserved()); + event.setTimestampMillis(originMCP.getSystemMetadata().getLastObserved()); + if (originMCP.getPreviousAspectValue() == null) { + event.setEventType("RuleCreated"); + } else { + event.setEventType("RuleUpdated"); + } + event.setAffectedDataset(originMCP.getEntityUrn()); + + return Optional.of(event); + } + return Optional.empty(); + } } ``` diff --git a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCLSideEffect.java b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCLSideEffect.java index a8735bae1521a..ba72a97908846 100644 --- a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCLSideEffect.java +++ b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCLSideEffect.java @@ -4,7 +4,6 @@ import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.entity.ebean.batch.MCLBatchItemImpl; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeLog; import com.mycompany.dq.DataQualityRuleEvent; @@ -20,9 +19,7 @@ public CustomDataQualityRulesMCLSideEffect(AspectPluginConfig config) { @Override protected Stream applyMCLSideEffect( - @Nonnull MCLBatchItem input, - @Nonnull EntityRegistry entityRegistry, - @Nonnull AspectRetriever aspectRetriever) { + @Nonnull MCLBatchItem input, @Nonnull AspectRetriever aspectRetriever) { // Generate Timeseries event aspect based on non-Timeseries aspect MetadataChangeLog originMCP = input.getMetadataChangeLog(); @@ -42,9 +39,7 @@ protected Stream applyMCLSideEffect( }) .map( eventMCP -> - MCLBatchItemImpl.builder() - .metadataChangeLog(eventMCP) - .build(entityRegistry, aspectRetriever)); + MCLBatchItemImpl.builder().metadataChangeLog(eventMCP).build(aspectRetriever)); return timeseriesOptional.stream(); } diff --git a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java index 2c989725f4f9d..d2041c443503e 100644 --- a/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java +++ b/metadata-models-custom/src/main/java/com/linkedin/metadata/aspect/plugins/hooks/CustomDataQualityRulesMCPSideEffect.java @@ -6,7 +6,6 @@ import com.linkedin.metadata.aspect.plugins.config.AspectPluginConfig; import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; -import com.linkedin.metadata.models.registry.EntityRegistry; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -18,7 +17,7 @@ public CustomDataQualityRulesMCPSideEffect(AspectPluginConfig aspectPluginConfig @Override protected Stream applyMCPSideEffect( - UpsertItem input, EntityRegistry entityRegistry, @Nonnull AspectRetriever aspectRetriever) { + UpsertItem input, @Nonnull AspectRetriever aspectRetriever) { // Mirror aspects to another URN in SQL & Search Urn mirror = UrnUtils.getUrn(input.getUrn().toString().replace(",PROD)", ",DEV)")); return Stream.of( @@ -28,6 +27,6 @@ protected Stream applyMCPSideEffect( .aspect(input.getAspect()) .auditStamp(input.getAuditStamp()) .systemMetadata(input.getSystemMetadata()) - .build(entityRegistry, aspectRetriever)); + .build(aspectRetriever)); } } diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 04c90fa444f0c..179e1eac177ac 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -18,10 +18,14 @@ dependencies { api project(path: ':li-utils', configuration: "dataTemplate") dataModel project(':li-utils') + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 + compileOnly externalDependency.annotationApi + compileOnly externalDependency.javaxValidation + compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok - compileOnly externalDependency.swaggerAnnotations - compileOnly externalDependency.springBootStarterValidation + api externalDependency.swaggerAnnotations compileOnly externalDependency.jacksonCore compileOnly externalDependency.jacksonDataBind @@ -43,11 +47,10 @@ mainAvroSchemaJar.dependsOn generateAvroSchema pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO] -tasks.register('generateJsonSchema', GenerateJsonSchemaTask) { +task generateJsonSchema(type: GenerateJsonSchemaTask, dependsOn: 'generateAvroSchema') { it.setInputDirectory("$projectDir/src/mainGeneratedAvroSchema") it.setOutputDirectory("$projectDir/src/generatedJsonSchema") it.setEntityRegistryYaml("${project(':metadata-models').projectDir}/src/main/resources/entity-registry.yml") - dependsOn generateAvroSchema } // https://github.com/int128/gradle-swagger-generator-plugin#task-type-generateswaggercode diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/CustomProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/CustomProperties.pdl index 8390a05846c83..cc70bb5c60fc6 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/CustomProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/CustomProperties.pdl @@ -9,6 +9,7 @@ record CustomProperties { */ @Searchable = { "/*": { + "fieldType": "TEXT", "queryByDefault": true } } diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/FieldFormPromptAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/FieldFormPromptAssociation.pdl new file mode 100644 index 0000000000000..d05f2308d82a5 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/FieldFormPromptAssociation.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.common + +/** + * Information about the status of a particular prompt for a specific schema field + * on an entity. + */ +record FieldFormPromptAssociation { + /** + * The field path on a schema field. + */ + fieldPath: string + + /** + * The last time this prompt was touched for the field on the entity (set, unset) + */ + lastModified: AuditStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/FormAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/FormAssociation.pdl new file mode 100644 index 0000000000000..558672478c19b --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/FormAssociation.pdl @@ -0,0 +1,21 @@ +namespace com.linkedin.common + +/** + * Properties of an applied form. + */ +record FormAssociation { + /** + * Urn of the applied form + */ + urn: Urn + + /** + * A list of prompts that are not yet complete for this form. + */ + incompletePrompts: array[FormPromptAssociation] = [] + + /** + * A list of prompts that have been completed for this form. + */ + completedPrompts: array[FormPromptAssociation] = [] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptAssociation.pdl new file mode 100644 index 0000000000000..ee0f1041e23c4 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptAssociation.pdl @@ -0,0 +1,23 @@ +namespace com.linkedin.common + +/** + * Information about the status of a particular prompt. + * Note that this is where we can add additional information about individual responses: + * actor, timestamp, and the response itself. + */ +record FormPromptAssociation { + /** + * The id for the prompt. This must be GLOBALLY UNIQUE. + */ + id: string + + /** + * The last time this prompt was touched for the entity (set, unset) + */ + lastModified: AuditStamp + + /** + * Optional information about the field-level prompt associations. + */ + fieldAssociations: optional FormPromptFieldAssociations +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptFieldAssociations.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptFieldAssociations.pdl new file mode 100644 index 0000000000000..419aa8aa3921d --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/FormPromptFieldAssociations.pdl @@ -0,0 +1,16 @@ +namespace com.linkedin.common + +/** + * Information about the field-level prompt associations on a top-level prompt association. + */ +record FormPromptFieldAssociations { + /** + * A list of field-level prompt associations that are not yet complete for this form. + */ + completedFieldPrompts: optional array[FieldFormPromptAssociation] + + /** + * A list of field-level prompt associations that are complete for this form. + */ + incompleteFieldPrompts: optional array[FieldFormPromptAssociation] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/FormVerificationAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/FormVerificationAssociation.pdl new file mode 100644 index 0000000000000..066e72f2f2a20 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/FormVerificationAssociation.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.common + +/** + * An association between a verification and an entity that has been granted + * via completion of one or more forms of type 'VERIFICATION'. + */ +record FormVerificationAssociation { + /** + * The urn of the form that granted this verification. + */ + form: Urn + + /** + * An audit stamp capturing who and when verification was applied for this form. + */ + lastModified: optional AuditStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl new file mode 100644 index 0000000000000..0a97c7d5099ed --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/Forms.pdl @@ -0,0 +1,66 @@ +namespace com.linkedin.common + +/** + * Forms that are assigned to this entity to be filled out + */ +@Aspect = { + "name": "forms" +} +record Forms { + /** + * All incomplete forms assigned to the entity. + */ + @Searchable = { + "/*/urn": { + "fieldType": "URN", + "fieldName": "incompleteForms" + }, + "/*/completedPrompts/*/id" : { + "fieldType": "KEYWORD", + "fieldName": "incompleteFormsCompletedPromptIds", + }, + "/*/incompletePrompts/*/id" : { + "fieldType": "KEYWORD", + "fieldName": "incompleteFormsIncompletePromptIds", + }, + "/*/completedPrompts/*/lastModified/time" : { + "fieldType": "DATETIME", + "fieldName": "incompleteFormsCompletedPromptResponseTimes", + } + } + incompleteForms: array[FormAssociation] + + /** + * All complete forms assigned to the entity. + */ + @Searchable = { + "/*/urn": { + "fieldType": "URN", + "fieldName": "completedForms" + }, + "/*/completedPrompts/*/id" : { + "fieldType": "KEYWORD", + "fieldName": "completedFormsCompletedPromptIds", + }, + "/*/incompletePrompts/*/id" : { + "fieldType": "KEYWORD", + "fieldName": "completedFormsIncompletePromptIds", + }, + "/*/completedPrompts/*/lastModified/time" : { + "fieldType": "DATETIME", + "fieldName": "completedFormsCompletedPromptResponseTimes", + } + } + completedForms: array[FormAssociation] + + /** + * Verifications that have been applied to the entity via completed forms. + */ + @Searchable = { + "/*/form": { + "fieldType": "URN", + "fieldName": "verifiedForms" + } + } + verifications: array[FormVerificationAssociation] = [] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl index 9f0f0ff6f24a2..80dc07981816a 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl @@ -20,8 +20,14 @@ record GlossaryTermAssociation { } urn: GlossaryTermUrn + /** + * The user URN which will be credited for adding associating this term to the entity + */ + actor: optional Urn + /** * Additional context about the association */ context: optional string + } diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/PropertyValue.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/PropertyValue.pdl new file mode 100644 index 0000000000000..c8f1e4d5009dc --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/common/PropertyValue.pdl @@ -0,0 +1,13 @@ +namespace com.linkedin.common + +record PropertyValue { + value: union [ + string, + double + ] + + /** + * Optional description of the property value + */ + description: optional string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/datahub/DataHubSearchConfig.pdl b/metadata-models/src/main/pegasus/com/linkedin/datahub/DataHubSearchConfig.pdl new file mode 100644 index 0000000000000..2d09d828d10bd --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/datahub/DataHubSearchConfig.pdl @@ -0,0 +1,87 @@ +namespace com.linkedin.datahub + +/** +* Configuration for how any given field should be indexed and matched in the DataHub search index. +**/ +record DataHubSearchConfig { + + /** + * Name of the field in the search index. Defaults to the field name otherwise + **/ + fieldName: optional string + + /** + * Type of the field. Defines how the field is indexed and matched + **/ + fieldType: optional enum SearchFieldType { + KEYWORD, + TEXT, + TEXT_PARTIAL, + BROWSE_PATH, + URN, + URN_PARTIAL, + BOOLEAN, + COUNT, + DATETIME, + OBJECT, + BROWSE_PATH_V2, + WORD_GRAM + } + + /** + * Whether we should match the field for the default search query + **/ + queryByDefault: boolean = false + + /** + * Whether we should use the field for default autocomplete + **/ + enableAutocomplete: boolean = false + + /** + * Whether or not to add field to filters. + **/ + addToFilters: boolean = false + + /** + * Whether or not to add the "has values" to filters. + * check if this is conditional on addToFilters being true + **/ + addHasValuesToFilters: boolean = true + + /** + * Display name of the filter + **/ + filterNameOverride: optional string + + /** + * Display name of the has values filter + **/ + hasValuesFilterNameOverride: optional string + + /** + * Boost multiplier to the match score. Matches on fields with higher boost score ranks higher + **/ + boostScore: double = 1.0 + + /** + * If set, add a index field of the given name that checks whether the field exists + **/ + hasValuesFieldName: optional string + + /** + * If set, add a index field of the given name that checks the number of elements + **/ + numValuesFieldName: optional string + + /** + * (Optional) Weights to apply to score for a given value + **/ + weightsPerFieldValue: optional map[string, double] + + /** + * (Optional) Aliases for this given field that can be used for sorting etc. + **/ + fieldNameAliases: optional array[string] + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeInfo.pdl new file mode 100644 index 0000000000000..4e3ea9d01e92d --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeInfo.pdl @@ -0,0 +1,21 @@ +namespace com.linkedin.datatype + +@Aspect = { + "name": "dataTypeInfo" +} +record DataTypeInfo { + /** + * The qualified name for the data type. Usually a unique namespace + name, e.g. datahub.string + */ + qualifiedName: string + + /** + * An optional display name for the data type. + */ + displayName: optional string + + /** + * An optional description for the data type. + */ + description: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeKey.pdl new file mode 100644 index 0000000000000..e0ea2b6974381 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/datatype/DataTypeKey.pdl @@ -0,0 +1,11 @@ +namespace com.linkedin.datatype + +@Aspect = { + "name": "dataTypeKey" +} +record DataTypeKey { + /** + * A unique id for a data type. Usually this will be a unique namespace + data type name. + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeInfo.pdl new file mode 100644 index 0000000000000..3a741a4d8f0b8 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeInfo.pdl @@ -0,0 +1,22 @@ +namespace com.linkedin.entitytype + +@Aspect = { + "name": "entityTypeInfo" +} +record EntityTypeInfo { + /** + * The fully qualified name for the entity type, which usually consists of a namespace + * plus an identifier or name, e.g. datahub.dataset + */ + qualifiedName: string + + /** + * The display name for the Entity Type. + */ + displayName: optional string + + /** + * A description for the Entity Type: what is it for? + */ + description: optional string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl new file mode 100644 index 0000000000000..d857c7ff611e3 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl @@ -0,0 +1,11 @@ +namespace com.linkedin.entitytype + +@Aspect = { + "name": "entityTypeKey" +} +record EntityTypeKey { + /** + * A unique id for an entity type. Usually this will be a unique namespace + entity name. + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/form/DynamicFormAssignment.pdl b/metadata-models/src/main/pegasus/com/linkedin/form/DynamicFormAssignment.pdl new file mode 100644 index 0000000000000..93ecf017efb3a --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/form/DynamicFormAssignment.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.form + +import com.linkedin.metadata.query.filter.Filter + +/** + * Information about how a form is assigned to entities dynamically. Provide a filter to + * match a set of entities instead of explicitly applying a form to specific entities. + */ +@Aspect = { + "name": "dynamicFormAssignment" +} +record DynamicFormAssignment { + /** + * The filter applied when assigning this form to entities. Entities that match this filter + * will have this form applied to them. Right now this filter only supports filtering by + * platform, entity type, container, and domain through the UI. + */ + filter: Filter +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/form/FormActorAssignment.pdl b/metadata-models/src/main/pegasus/com/linkedin/form/FormActorAssignment.pdl new file mode 100644 index 0000000000000..e58eb4c7c56a8 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/form/FormActorAssignment.pdl @@ -0,0 +1,21 @@ +namespace com.linkedin.form + +import com.linkedin.common.Urn + +record FormActorAssignment { + /** + * Whether the form should be assigned to the owners of assets that it is applied to. + * This is the default. + */ + owners: boolean = true + + /** + * Optional: Specific set of groups that are targeted by this form assignment. + */ + groups: optional array[Urn] + + /** + * Optional: Specific set of users that are targeted by this form assignment. + */ + users: optional array[Urn] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/form/FormInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/form/FormInfo.pdl new file mode 100644 index 0000000000000..b17bd1537a17c --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/form/FormInfo.pdl @@ -0,0 +1,51 @@ +namespace com.linkedin.form + +import com.linkedin.common.Urn + +/** + * Information about a form to help with filling out metadata on entities. + */ +@Aspect = { + "name": "formInfo" +} +record FormInfo { + /** + * Display name of the form + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL" + } + name: string + + /** + * Description of the form + */ + description: optional string + + /** + * The type of this form + */ + @Searchable = { + "fieldType": "KEYWORD" + } + type: enum FormType { + /** + * A form simply used for collecting metadata fields for an entity. + */ + COMPLETION + /** + * This form is used for "verifying" that entities comply with a policy via presence of a specific set of metadata fields. + */ + VERIFICATION + } = "COMPLETION" + + /** + * List of prompts to present to the user to encourage filling out metadata + */ + prompts: array[FormPrompt] = [] + + /** + * Who the form is assigned to, e.g. who should see the form when visiting the entity page or governance center + */ + actors: FormActorAssignment = { "owners": true } +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl b/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl new file mode 100644 index 0000000000000..73f06552d46ab --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/form/FormPrompt.pdl @@ -0,0 +1,53 @@ +namespace com.linkedin.form + +import com.linkedin.common.Urn + +/** + * A prompt to present to the user to encourage filling out metadata + */ +record FormPrompt { + /** + * The unique id for this prompt. This must be GLOBALLY unique. + */ + id: string + + /** + * The title of this prompt + */ + title: string + + /** + * The description of this prompt + */ + description: optional string + + /** + * The type of prompt + */ + type: enum FormPromptType { + /** + * This prompt is meant to apply a structured property to an entity + */ + STRUCTURED_PROPERTY + /** + * This prompt is meant to apply a structured property to a schema fields entity + */ + FIELDS_STRUCTURED_PROPERTY + } + + /** + * An optional set of information specific to structured properties prompts. + * This should be filled out if the prompt is type STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY. + */ + structuredPropertyParams: optional record StructuredPropertyParams { + /** + * The structured property that is required on this entity + */ + urn: Urn + } + + /** + * Whether the prompt is required to be completed, in order for the form to be marked as complete. + */ + required: boolean = true +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl index c3388d4f462d4..b4a6f4b47b221 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryNodeInfo.pdl @@ -1,5 +1,6 @@ namespace com.linkedin.glossary +import com.linkedin.common.CustomProperties import com.linkedin.common.GlossaryNodeUrn /** @@ -8,7 +9,7 @@ import com.linkedin.common.GlossaryNodeUrn @Aspect = { "name": "glossaryNodeInfo" } -record GlossaryNodeInfo { +record GlossaryNodeInfo includes CustomProperties { /** * Definition of business node diff --git a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl index e987a71be7131..1de826f1b2aa6 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/glossary/GlossaryTermInfo.pdl @@ -3,6 +3,7 @@ namespace com.linkedin.glossary import com.linkedin.common.Url import com.linkedin.common.GlossaryNodeUrn import com.linkedin.common.CustomProperties +import com.linkedin.schema.PrimitiveValueDataType /** * Properties associated with a GlossaryTerm @@ -76,4 +77,5 @@ record GlossaryTermInfo includes CustomProperties { */ @deprecated rawSchema: optional string + } diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/FormKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/FormKey.pdl new file mode 100644 index 0000000000000..124d65d0e7452 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/FormKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a Form + */ +@Aspect = { + "name": "formKey", +} +record FormKey { + /** + * Unique id for the form. + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/PrimitivePropertyValue.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/PrimitivePropertyValue.pdl new file mode 100644 index 0000000000000..93dbb14c7f969 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/PrimitivePropertyValue.pdl @@ -0,0 +1,9 @@ +namespace com.linkedin.structured + +/** +* Represents a stored primitive property value +**/ +typeref PrimitivePropertyValue = union [ + string, + double + ] \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/PropertyValue.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/PropertyValue.pdl new file mode 100644 index 0000000000000..012ce5416364f --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/PropertyValue.pdl @@ -0,0 +1,10 @@ +namespace com.linkedin.structured + +record PropertyValue { + value: PrimitivePropertyValue + + /** + * Optional description of the property value + */ + description: optional string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredProperties.pdl new file mode 100644 index 0000000000000..f79e8fd86e825 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredProperties.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.structured + +/** + * Properties about an entity governed by StructuredPropertyDefinition + */ +@Aspect = { + "name": "structuredProperties" +} +record StructuredProperties { + /** + * Custom property bag. + */ + properties: array[StructuredPropertyValueAssignment] +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl new file mode 100644 index 0000000000000..1b263b679531a --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl @@ -0,0 +1,74 @@ +namespace com.linkedin.structured + +import com.linkedin.common.Urn +import com.linkedin.datahub.DataHubSearchConfig + +@Aspect = { + "name": "propertyDefinition" +} +record StructuredPropertyDefinition { + /** + * The fully qualified name of the property. e.g. io.acryl.datahub.myProperty + */ + @Searchable = {} + qualifiedName: string + + /** + * The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id. + */ + @Searchable = {} + displayName: optional string + + /** + * The value type of the property. Must be a dataType. + * e.g. To indicate that the property is of type DATE, use urn:li:dataType:datahub.date + */ + valueType: Urn + + /** + * A map that allows for type specialization of the valueType. + * e.g. a valueType of urn:li:dataType:datahub.urn + * can be specialized to be a USER or GROUP URN by adding a typeQualifier like + * { "allowedTypes": ["urn:li:entityType:datahub.corpuser", "urn:li:entityType:datahub.corpGroup"] } + */ + typeQualifier: optional map[string, array[string]] + + /** + * A list of allowed values that the property is allowed to take. + * If this is not specified, then the property can take any value of given type. + */ + allowedValues: optional array[PropertyValue] + + /** + * The cardinality of the property. If not specified, then the property is assumed to be single valued.. + */ + cardinality: optional enum PropertyCardinality { + SINGLE + MULTIPLE + } = "SINGLE" + + @Relationship = { + "/*": { + "name": "StructuredPropertyOf", + "entityTypes": [ "entityType" ] + } + } + @Searchable = { + "/*": { + "fieldName": "entityTypes" + } + } + entityTypes: array[Urn] + + /** + * The description of the property. This is the description that will be shown in the UI. + */ + description: optional string + + /** + * Search configuration for this property. If not specified, then the property is indexed using the default mapping. + * from the logical type. + */ + searchConfiguration: optional DataHubSearchConfig +} + diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyKey.pdl new file mode 100644 index 0000000000000..16fec7b2a5ab6 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyKey.pdl @@ -0,0 +1,11 @@ +namespace com.linkedin.structured + +@Aspect = { + "name": "structuredPropertyKey" +} +record StructuredPropertyKey { + /** + * The id for a structured proeprty. + */ + id: string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyValueAssignment.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyValueAssignment.pdl new file mode 100644 index 0000000000000..d8b8a93a3edb6 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyValueAssignment.pdl @@ -0,0 +1,29 @@ +namespace com.linkedin.structured +import com.linkedin.common.Urn +import com.linkedin.common.AuditStamp + +record StructuredPropertyValueAssignment { + + /** + * The property that is being assigned a value. + */ + propertyUrn: Urn + + /** + * The value assigned to the property. + */ + values: array[PrimitivePropertyValue] + + /** + * Audit stamp containing who created this relationship edge and when + */ + created: optional AuditStamp + + /** + * Audit stamp containing who last modified this relationship edge and when + */ + lastModified: optional AuditStamp + +} + + diff --git a/metadata-models/src/main/resources/JavaSpring/model.mustache b/metadata-models/src/main/resources/JavaSpring/model.mustache index 72da42612777c..a048f249a6b3d 100644 --- a/metadata-models/src/main/resources/JavaSpring/model.mustache +++ b/metadata-models/src/main/resources/JavaSpring/model.mustache @@ -9,9 +9,9 @@ import java.io.Serializable; {{/serializableModel}} {{#useBeanValidation}} import org.springframework.validation.annotation.Validated; -import jakarta.validation.Valid; +import javax.validation.Valid; import com.fasterxml.jackson.annotation.JsonInclude; -import jakarta.validation.constraints.*; +import javax.validation.constraints.*; {{/useBeanValidation}} {{#jackson}} {{#withXml}} @@ -20,7 +20,7 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; {{/withXml}} {{/jackson}} {{#withXml}} -import jakarta.xml.bind.annotation.*; +import javax.xml.bind.annotation.*; {{/withXml}} {{/x-is-composed-model}} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 9d8c4bfdab0da..65382c747a16a 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -42,6 +42,8 @@ entities: - dataPlatformInstance - browsePathsV2 - access + - structuredProperties + - forms - name: dataHubPolicy doc: DataHub Policies represent access policies granted to users or groups on metadata operations like edit, view etc. category: internal @@ -67,6 +69,7 @@ entities: - institutionalMemory - dataPlatformInstance - browsePathsV2 + - structuredProperties - subTypes - name: dataFlow category: core @@ -85,6 +88,7 @@ entities: - institutionalMemory - dataPlatformInstance - browsePathsV2 + - structuredProperties - name: dataProcess keyAspect: dataProcessKey aspects: @@ -409,7 +413,8 @@ entities: - name: schemaField category: core keyAspect: schemaFieldKey - aspects: [] + aspects: + - structuredProperties - name: globalSettings doc: Global settings for an the platform category: internal @@ -468,5 +473,51 @@ entities: - dataContractProperties - dataContractStatus - status - + - name: entityType + doc: A type of entity in the DataHub Metadata Model. + category: core + keyAspect: entityTypeKey + aspects: + - entityTypeInfo + - institutionalMemory + - status + - name: dataType + doc: A type of data element stored within DataHub. + category: core + keyAspect: dataTypeKey + aspects: + - dataTypeInfo + - institutionalMemory + - status + - name: structuredProperty + doc: Structured Property represents a property meant for extending the core model of a logical entity + category: core + keyAspect: structuredPropertyKey + aspects: + - propertyDefinition + - institutionalMemory + - status + - name: form + category: core + keyAspect: formKey + aspects: + - formInfo + - dynamicFormAssignment + - ownership events: +plugins: + aspectPayloadValidators: + - className: 'com.linkedin.metadata.aspect.validation.PropertyDefinitionValidator' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: structuredProperty + aspectName: propertyDefinition + - className: 'com.linkedin.metadata.aspect.validation.StructuredPropertiesValidator' + enabled: true + supportedOperations: + - UPSERT + supportedEntityAspectNames: + - entityName: '*' + aspectName: structuredProperties \ No newline at end of file diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java index 8ce7675edf580..c4b01fea8c09d 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/group/GroupService.java @@ -28,6 +28,9 @@ import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -175,6 +178,17 @@ public void migrateGroupMembershipToNativeGroupMembership( userUrnList.forEach(userUrn -> addUserToNativeGroup(userUrn, groupUrn, authentication)); } + public List getGroupsForUser( + @Nonnull final Urn userUrn, @Nonnull final Authentication authentication) throws Exception { + final NativeGroupMembership nativeGroupMembership = + getExistingNativeGroupMembership(userUrn, authentication); + final GroupMembership groupMembership = getExistingGroupMembership(userUrn, authentication); + final List allGroups = new ArrayList<>(); + allGroups.addAll(nativeGroupMembership.getNativeGroups()); + allGroups.addAll(groupMembership.getGroups()); + return allGroups; + } + NativeGroupMembership getExistingNativeGroupMembership( @Nonnull final Urn userUrn, final Authentication authentication) throws Exception { final EntityResponse entityResponse = @@ -186,7 +200,7 @@ NativeGroupMembership getExistingNativeGroupMembership( authentication) .get(userUrn); - NativeGroupMembership nativeGroupMembership; + final NativeGroupMembership nativeGroupMembership; if (entityResponse == null || !entityResponse.getAspects().containsKey(NATIVE_GROUP_MEMBERSHIP_ASPECT_NAME)) { // If the user doesn't have the NativeGroupMembership aspect, create one. @@ -204,6 +218,32 @@ NativeGroupMembership getExistingNativeGroupMembership( return nativeGroupMembership; } + GroupMembership getExistingGroupMembership( + @Nonnull final Urn userUrn, @Nonnull final Authentication authentication) + throws RemoteInvocationException, URISyntaxException { + final EntityResponse entityResponse = + _entityClient + .batchGetV2( + CORP_USER_ENTITY_NAME, + Collections.singleton(userUrn), + Collections.singleton(GROUP_MEMBERSHIP_ASPECT_NAME), + authentication) + .get(userUrn); + + final GroupMembership groupMembership; + if (entityResponse == null + || !entityResponse.getAspects().containsKey(GROUP_MEMBERSHIP_ASPECT_NAME)) { + // If the user doesn't have the GroupMembership aspect, create one. + groupMembership = new GroupMembership(); + groupMembership.setGroups(new UrnArray()); + } else { + groupMembership = + new GroupMembership( + entityResponse.getAspects().get(GROUP_MEMBERSHIP_ASPECT_NAME).getValue().data()); + } + return groupMembership; + } + String createGroupInfo( @Nonnull final CorpGroupKey corpGroupKey, @Nonnull final String groupName, diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java index 40555107f4c79..e072a59ae77ff 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authentication/token/StatefulTokenService.java @@ -13,7 +13,6 @@ import com.linkedin.metadata.entity.AspectUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.key.DataHubAccessTokenKey; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -41,7 +40,7 @@ @Slf4j public class StatefulTokenService extends StatelessTokenService { - private final EntityService _entityService; + private final EntityService _entityService; private final LoadingCache _revokedTokenCache; private final String salt; @@ -49,7 +48,7 @@ public StatefulTokenService( @Nonnull final String signingKey, @Nonnull final String signingAlgorithm, @Nullable final String iss, - @Nonnull final EntityService entityService, + @Nonnull final EntityService entityService, @Nonnull final String salt) { super(signingKey, signingAlgorithm, iss); this._entityService = entityService; @@ -154,11 +153,7 @@ public String generateAccessToken( _entityService.ingestProposal( AspectsBatchImpl.builder() - .mcps( - proposalStream.collect(Collectors.toList()), - auditStamp, - _entityService.getEntityRegistry(), - _entityService.getSystemEntityClient()) + .mcps(proposalStream.collect(Collectors.toList()), auditStamp, _entityService) .build(), false); diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java index c2d9c42693311..653bbecbfa8ad 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/DefaultEntitySpecResolver.java @@ -8,6 +8,7 @@ import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider; import com.datahub.authorization.fieldresolverprovider.GroupMembershipFieldResolverProvider; import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider; +import com.datahub.authorization.fieldresolverprovider.TagFieldResolverProvider; import com.google.common.collect.ImmutableList; import com.linkedin.entity.client.EntityClient; import com.linkedin.util.Pair; @@ -26,7 +27,8 @@ public DefaultEntitySpecResolver(Authentication systemAuthentication, EntityClie new DomainFieldResolverProvider(entityClient, systemAuthentication), new OwnerFieldResolverProvider(entityClient, systemAuthentication), new DataPlatformInstanceFieldResolverProvider(entityClient, systemAuthentication), - new GroupMembershipFieldResolverProvider(entityClient, systemAuthentication)); + new GroupMembershipFieldResolverProvider(entityClient, systemAuthentication), + new TagFieldResolverProvider(entityClient, systemAuthentication)); } @Override diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProvider.java new file mode 100644 index 0000000000000..2cfd803249734 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProvider.java @@ -0,0 +1,65 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authentication.Authentication; +import com.datahub.authorization.EntityFieldType; +import com.datahub.authorization.EntitySpec; +import com.datahub.authorization.FieldResolver; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** Provides field resolver for owners given entitySpec */ +@Slf4j +@RequiredArgsConstructor +public class TagFieldResolverProvider implements EntityFieldResolverProvider { + + private final EntityClient _entityClient; + private final Authentication _systemAuthentication; + + @Override + public List getFieldTypes() { + return Collections.singletonList(EntityFieldType.TAG); + } + + @Override + public FieldResolver getFieldResolver(EntitySpec entitySpec) { + return FieldResolver.getResolverFromFunction(entitySpec, this::getTags); + } + + private FieldResolver.FieldValue getTags(EntitySpec entitySpec) { + Urn entityUrn = UrnUtils.getUrn(entitySpec.getEntity()); + EnvelopedAspect globalTagsAspect; + try { + EntityResponse response = + _entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + Collections.singleton(Constants.GLOBAL_TAGS_ASPECT_NAME), + _systemAuthentication); + if (response == null + || !response.getAspects().containsKey(Constants.GLOBAL_TAGS_ASPECT_NAME)) { + return FieldResolver.emptyFieldValue(); + } + globalTagsAspect = response.getAspects().get(Constants.GLOBAL_TAGS_ASPECT_NAME); + } catch (Exception e) { + log.error("Error while retrieving tags aspect for urn {}", entityUrn, e); + return FieldResolver.emptyFieldValue(); + } + GlobalTags globalTags = new GlobalTags(globalTagsAspect.getValue().data()); + return FieldResolver.FieldValue.builder() + .values( + globalTags.getTags().stream() + .map(tag -> tag.getTag().toString()) + .collect(Collectors.toSet())) + .build(); + } +} diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java index c7f06eeba6e85..8ecfb5a2c27bb 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java @@ -44,6 +44,7 @@ public class PolicyEngineTest { private static final String AUTHORIZED_GROUP = "urn:li:corpGroup:authorizedGroup"; private static final String RESOURCE_URN = "urn:li:dataset:test"; private static final String DOMAIN_URN = "urn:li:domain:domain1"; + private static final String TAG_URN = "urn:li:tag:allowed"; private static final String OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__technical_owner"; private static final String OTHER_OWNERSHIP_TYPE_URN = "urn:li:ownershipType:__system__data_steward"; @@ -69,7 +70,8 @@ public void setupTest() throws Exception { AUTHORIZED_PRINCIPAL, Collections.emptySet(), Collections.emptySet(), - Collections.singleton(AUTHORIZED_GROUP)); + Collections.singleton(AUTHORIZED_GROUP), + Collections.emptySet()); unauthorizedUserUrn = Urn.createFromString(UNAUTHORIZED_PRINCIPAL); resolvedUnauthorizedUserSpec = buildEntityResolvers(CORP_USER_ENTITY_NAME, UNAUTHORIZED_PRINCIPAL); @@ -595,6 +597,7 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersMatch() throws Except RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. PolicyEngine.PolicyEvaluationResult result1 = @@ -653,6 +656,7 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersTypeMatch() throws Ex RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); PolicyEngine.PolicyEvaluationResult result1 = @@ -712,6 +716,7 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersTypeNoMatch() throws RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); PolicyEngine.PolicyEvaluationResult result1 = @@ -767,6 +772,7 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Excep RESOURCE_URN, ImmutableSet.of(AUTHORIZED_GROUP), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. PolicyEngine.PolicyEvaluationResult result1 = @@ -1037,6 +1043,7 @@ public void testEvaluatePolicyResourceFilterSpecificResourceMatchDomain() throws RESOURCE_URN, Collections.emptySet(), Collections.singleton(DOMAIN_URN), + Collections.emptySet(), Collections.emptySet()); PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( @@ -1082,6 +1089,7 @@ public void testEvaluatePolicyResourceFilterSpecificResourceNoMatchDomain() thro RESOURCE_URN, Collections.emptySet(), Collections.singleton("urn:li:domain:domain2"), + Collections.emptySet(), Collections.emptySet()); // Domain doesn't match PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( @@ -1095,6 +1103,52 @@ public void testEvaluatePolicyResourceFilterSpecificResourceNoMatchDomain() thro verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); } + @Test + public void testEvaluatePolicyResourceFilterSpecificResourceMatchTag() throws Exception { + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("VIEW_ENTITY_PAGE")); + dataHubPolicyInfo.setDisplayName("Tag-based policy"); + dataHubPolicyInfo.setDescription("Allow viewing entity pages based on tags"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(true); + actorFilter.setAllGroups(true); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setFilter( + FilterUtils.newFilter( + ImmutableMap.of( + EntityFieldType.TYPE, + Collections.singletonList("dataset"), + EntityFieldType.TAG, + Collections.singletonList(TAG_URN)))); + dataHubPolicyInfo.setResources(resourceFilter); + + ResolvedEntitySpec resourceSpec = + buildEntityResolvers( + "dataset", + RESOURCE_URN, + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet(), + Collections.singleton(TAG_URN)); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy( + dataHubPolicyInfo, + resolvedAuthorizedUserSpec, + "VIEW_ENTITY_PAGE", + Optional.of(resourceSpec)); + assertTrue(result.isGranted()); + + // Verify no network calls + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); + } + @Test public void testGetGrantedPrivileges() throws Exception { // Policy 1, match dataset type and domain @@ -1180,6 +1234,7 @@ public void testGetGrantedPrivileges() throws Exception { RESOURCE_URN, Collections.emptySet(), Collections.singleton(DOMAIN_URN), + Collections.emptySet(), Collections.emptySet()); // Everything matches assertEquals( _policyEngine.getGrantedPrivileges( @@ -1192,6 +1247,7 @@ public void testGetGrantedPrivileges() throws Exception { RESOURCE_URN, Collections.emptySet(), Collections.singleton("urn:li:domain:domain2"), + Collections.emptySet(), Collections.emptySet()); // Domain doesn't match assertEquals( _policyEngine.getGrantedPrivileges( @@ -1204,6 +1260,7 @@ public void testGetGrantedPrivileges() throws Exception { "urn:li:dataset:random", Collections.emptySet(), Collections.singleton(DOMAIN_URN), + Collections.emptySet(), Collections.emptySet()); // Resource doesn't match assertEquals( _policyEngine.getGrantedPrivileges( @@ -1228,6 +1285,7 @@ public void testGetGrantedPrivileges() throws Exception { RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL), Collections.singleton(DOMAIN_URN), + Collections.emptySet(), Collections.emptySet()); // Is owner assertEquals( _policyEngine.getGrantedPrivileges( @@ -1240,6 +1298,7 @@ public void testGetGrantedPrivileges() throws Exception { RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL), Collections.singleton(DOMAIN_URN), + Collections.emptySet(), Collections.emptySet()); // Resource type doesn't match assertEquals( _policyEngine.getGrantedPrivileges( @@ -1289,6 +1348,7 @@ public void testGetMatchingActorsResourceMatch() throws Exception { RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL, AUTHORIZED_GROUP), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec)); @@ -1406,6 +1466,7 @@ public void testGetMatchingActorsByRoleResourceMatch() throws Exception { RESOURCE_URN, ImmutableSet.of(), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); PolicyEngine.PolicyActors actors = @@ -1506,6 +1567,7 @@ public static ResolvedEntitySpec buildEntityResolvers(String entityType, String entityUrn, Collections.emptySet(), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet()); } @@ -1514,7 +1576,8 @@ public static ResolvedEntitySpec buildEntityResolvers( String entityUrn, Set owners, Set domains, - Set groups) { + Set groups, + Set tags) { return new ResolvedEntitySpec( new EntitySpec(entityType, entityUrn), ImmutableMap.of( @@ -1527,6 +1590,8 @@ public static ResolvedEntitySpec buildEntityResolvers( EntityFieldType.DOMAIN, FieldResolver.getResolverFromValues(domains), EntityFieldType.GROUP_MEMBERSHIP, - FieldResolver.getResolverFromValues(groups))); + FieldResolver.getResolverFromValues(groups), + EntityFieldType.TAG, + FieldResolver.getResolverFromValues(tags))); } } diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProviderTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProviderTest.java new file mode 100644 index 0000000000000..de5ef09cd4251 --- /dev/null +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/fieldresolverprovider/TagFieldResolverProviderTest.java @@ -0,0 +1,154 @@ +package com.datahub.authorization.fieldresolverprovider; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.datahub.authentication.Authentication; +import com.datahub.authorization.EntityFieldType; +import com.datahub.authorization.EntitySpec; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.TagUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.Set; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class TagFieldResolverProviderTest { + + private static final String TAG_URN = "urn:li:tag:test"; + private static final String RESOURCE_URN = + "urn:li:dataset:(urn:li:dataPlatform:s3,test-platform-instance.testDataset,PROD)"; + private static final EntitySpec RESOURCE_SPEC = new EntitySpec(DATASET_ENTITY_NAME, RESOURCE_URN); + + @Mock private EntityClient entityClientMock; + @Mock private Authentication systemAuthenticationMock; + + private TagFieldResolverProvider tagFieldResolverProvider; + + @BeforeMethod + public void setup() { + MockitoAnnotations.initMocks(this); + tagFieldResolverProvider = + new TagFieldResolverProvider(entityClientMock, systemAuthenticationMock); + } + + @Test + public void shouldReturnTagType() { + assertEquals(EntityFieldType.TAG, tagFieldResolverProvider.getFieldTypes().get(0)); + } + + @Test + public void shouldReturnEmptyFieldValueWhenResponseIsNull() + throws RemoteInvocationException, URISyntaxException { + when(entityClientMock.getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock))) + .thenReturn(null); + + var result = tagFieldResolverProvider.getFieldResolver(RESOURCE_SPEC); + + assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty()); + verify(entityClientMock, times(1)) + .getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock)); + } + + @Test + public void shouldReturnEmptyFieldValueWhenResourceHasNoTag() + throws RemoteInvocationException, URISyntaxException { + var entityResponseMock = mock(EntityResponse.class); + when(entityResponseMock.getAspects()).thenReturn(new EnvelopedAspectMap()); + when(entityClientMock.getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock))) + .thenReturn(entityResponseMock); + + var result = tagFieldResolverProvider.getFieldResolver(RESOURCE_SPEC); + + assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty()); + verify(entityClientMock, times(1)) + .getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock)); + } + + @Test + public void shouldReturnEmptyFieldValueWhenThereIsAnException() + throws RemoteInvocationException, URISyntaxException { + when(entityClientMock.getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock))) + .thenThrow(new RemoteInvocationException()); + + var result = tagFieldResolverProvider.getFieldResolver(RESOURCE_SPEC); + + assertTrue(result.getFieldValuesFuture().join().getValues().isEmpty()); + verify(entityClientMock, times(1)) + .getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock)); + } + + @Test + public void shouldReturnFieldValueWithTagOfTheResource() + throws RemoteInvocationException, URISyntaxException { + + var tagAssociation = new TagAssociation(); + tagAssociation.setTag(new TagUrn("test")); + var tags = new TagAssociationArray(tagAssociation); + var globalTags = new GlobalTags().setTags(tags); + var entityResponseMock = mock(EntityResponse.class); + var envelopedAspectMap = new EnvelopedAspectMap(); + envelopedAspectMap.put( + GLOBAL_TAGS_ASPECT_NAME, new EnvelopedAspect().setValue(new Aspect(globalTags.data()))); + when(entityResponseMock.getAspects()).thenReturn(envelopedAspectMap); + when(entityClientMock.getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock))) + .thenReturn(entityResponseMock); + + var result = tagFieldResolverProvider.getFieldResolver(RESOURCE_SPEC); + + assertEquals(Set.of(TAG_URN), result.getFieldValuesFuture().join().getValues()); + verify(entityClientMock, times(1)) + .getV2( + eq(DATASET_ENTITY_NAME), + any(Urn.class), + eq(Collections.singleton(GLOBAL_TAGS_ASPECT_NAME)), + eq(systemAuthenticationMock)); + } +} diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle index 80cf6541261c2..f912e2ac01f0b 100644 --- a/metadata-service/configuration/build.gradle +++ b/metadata-service/configuration/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' } apply from: "../../gradle/versioning/versioning.gradle" @@ -7,8 +7,9 @@ dependencies { implementation externalDependency.jacksonDataBind implementation externalDependency.slf4jApi - implementation externalDependency.springCore - implementation externalDependency.springBeans + + // Newer Spring libraries require JDK17 classes, allow for JDK11 + compileOnly externalDependency.springBootAutoconfigureJdk11 compileOnly externalDependency.lombok diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java index bc749a373c5b0..eb5243c0e5e4a 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/VisualConfiguration.java @@ -8,9 +8,18 @@ public class VisualConfiguration { /** Asset related configurations */ public AssetsConfiguration assets; + /** Custom app title to show in the browse tab */ + public String appTitle; + /** Queries tab related configurations */ public QueriesTabConfig queriesTab; + /** + * Boolean flag disabling viewing the Business Glossary page for users without the 'Manage + * Glossaries' privilege + */ + public boolean hideGlossary; + /** Queries tab related configurations */ public EntityProfileConfig entityProfile; diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index cfc84491ab0ae..2b202d513c9bf 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -1,3 +1,6 @@ +# The base URL where DataHub is accessible to users. +baseUrl: ${DATAHUB_BASE_URL:http://localhost:9002} + # App Layer authentication: # Enable if you want all requests to the Metadata Service to be authenticated. Disabled by default. @@ -113,7 +116,9 @@ visualConfig: queriesTabResultSize: ${REACT_APP_QUERIES_TAB_RESULT_SIZE:5} assets: logoUrl: ${REACT_APP_LOGO_URL:/assets/platforms/datahublogo.png} - faviconUrl: ${REACT_APP_FAVICON_URL:/assets/favicon.ico} + faviconUrl: ${REACT_APP_FAVICON_URL:/assets/icons/favicon.ico} + appTitle: ${REACT_APP_TITLE:} + hideGlossary: ${REACT_APP_HIDE_GLOSSARY:false} entityProfile: # we only support default tab for domains right now. In order to implement for other entities, update React code domainDefaultTab: ${DOMAIN_DEFAULT_TAB:} # set to DOCUMENTATION_TAB to show documentation tab first @@ -305,6 +310,11 @@ systemUpdate: backOffFactor: ${BOOTSTRAP_SYSTEM_UPDATE_BACK_OFF_FACTOR:2} # Multiplicative factor for back off, default values will result in waiting 5min 15s waitForSystemUpdate: ${BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE:true} +structuredProperties: + enabled: ${ENABLE_STRUCTURED_PROPERTIES_HOOK:true} # applies structured properties mappings + writeEnabled: ${ENABLE_STRUCTURED_PROPERTIES_WRITE:true} # write structured property values + systemUpdateEnabled: ${ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE:false} # applies structured property mappings in system update job + healthCheck: cacheDurationSeconds: ${HEALTH_CHECK_CACHE_DURATION_SECONDS:5} @@ -324,6 +334,7 @@ featureFlags: uiEnabled: ${PRE_PROCESS_HOOKS_UI_ENABLED:true} # Circumvents Kafka for processing index updates for UI changes sourced from GraphQL to avoid processing delays showAcrylInfo: ${SHOW_ACRYL_INFO:false} # Show different CTAs within DataHub around moving to Managed DataHub. Set to true for the demo site. nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again + schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} @@ -375,5 +386,12 @@ cache: status: 20 corpUserCredentials: 20 corpUserSettings: 20 + structuredProperty: + propertyDefinition: 86400 # 1 day + structuredPropertyKey: 86400 # 1 day springdoc.api-docs.groups.enabled: true + +forms: + hook: + enabled: {$FORMS_HOOK_ENABLED:true} \ No newline at end of file diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java index ec398388ae77b..7b823e552da97 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizerChainFactory.java @@ -1,6 +1,5 @@ package com.linkedin.gms.factory.auth; -import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizerChain; import com.datahub.authorization.AuthorizerContext; import com.datahub.authorization.DataHubAuthorizer; @@ -18,8 +17,8 @@ import com.datahub.plugins.loader.IsolatedClassLoader; import com.datahub.plugins.loader.PluginPermissionManagerImpl; import com.google.common.collect.ImmutableMap; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import jakarta.annotation.Nonnull; import java.nio.file.Path; @@ -47,39 +46,29 @@ public class AuthorizerChainFactory { @Qualifier("configurationProvider") private ConfigurationProvider configurationProvider; - @Autowired - @Qualifier("dataHubAuthorizer") - private DataHubAuthorizer dataHubAuthorizer; - - @Autowired - @Qualifier("systemAuthentication") - private Authentication systemAuthentication; - - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient entityClient; - @Bean(name = "authorizerChain") @Scope("singleton") @Nonnull - protected AuthorizerChain getInstance() { - final EntitySpecResolver resolver = initResolver(); + protected AuthorizerChain getInstance( + final DataHubAuthorizer dataHubAuthorizer, final SystemEntityClient systemEntityClient) { + final EntitySpecResolver resolver = initResolver(systemEntityClient); // Extract + initialize customer authorizers from application configs. final List authorizers = new ArrayList<>(initCustomAuthorizers(resolver)); if (configurationProvider.getAuthorization().getDefaultAuthorizer().isEnabled()) { AuthorizerContext ctx = new AuthorizerContext(Collections.emptyMap(), resolver); - this.dataHubAuthorizer.init(Collections.emptyMap(), ctx); + dataHubAuthorizer.init(Collections.emptyMap(), ctx); log.info("Default DataHubAuthorizer is enabled. Appending it to the authorization chain."); - authorizers.add(this.dataHubAuthorizer); + authorizers.add(dataHubAuthorizer); } return new AuthorizerChain(authorizers, dataHubAuthorizer); } - private EntitySpecResolver initResolver() { - return new DefaultEntitySpecResolver(systemAuthentication, entityClient); + private EntitySpecResolver initResolver(SystemEntityClient systemEntityClient) { + return new DefaultEntitySpecResolver( + systemEntityClient.getSystemAuthentication(), systemEntityClient); } private List initCustomAuthorizers(EntitySpecResolver resolver) { @@ -121,7 +110,7 @@ private void registerAuthorizer( // Get security mode set by user SecurityMode securityMode = SecurityMode.valueOf( - this.configurationProvider.getDatahub().getPlugin().getPluginSecurityMode()); + configurationProvider.getDatahub().getPlugin().getPluginSecurityMode()); // Create permission manager with security mode PluginPermissionManager permissionManager = new PluginPermissionManagerImpl(securityMode); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java index 3b23243f76742..0935e8ad0e7d4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubAuthorizerFactory.java @@ -1,33 +1,19 @@ package com.linkedin.gms.factory.auth; -import com.datahub.authentication.Authentication; import com.datahub.authorization.DataHubAuthorizer; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) -@Import({RestliEntityClientFactory.class}) public class DataHubAuthorizerFactory { - @Autowired - @Qualifier("systemAuthentication") - private Authentication systemAuthentication; - - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient entityClient; - @Value("${authorization.defaultAuthorizer.cacheRefreshIntervalSecs}") private Integer policyCacheRefreshIntervalSeconds; @@ -40,7 +26,7 @@ public class DataHubAuthorizerFactory { @Bean(name = "dataHubAuthorizer") @Scope("singleton") @Nonnull - protected DataHubAuthorizer getInstance() { + protected DataHubAuthorizer dataHubAuthorizer(final SystemEntityClient systemEntityClient) { final DataHubAuthorizer.AuthorizationMode mode = policiesEnabled @@ -48,8 +34,8 @@ protected DataHubAuthorizer getInstance() { : DataHubAuthorizer.AuthorizationMode.ALLOW_ALL; return new DataHubAuthorizer( - systemAuthentication, - entityClient, + systemEntityClient.getSystemAuthentication(), + systemEntityClient, 10, policyCacheRefreshIntervalSeconds, mode, diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java index 83544e4165ae3..beb467d614930 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/DataHubTokenServiceFactory.java @@ -28,16 +28,16 @@ public class DataHubTokenServiceFactory { @Value("${authentication.tokenService.issuer:datahub-metadata-service}") private String issuer; - /** + @Inject + @Named("entityService") + private EntityService _entityService; + */ + /** + @Inject + @Named("entityService") + private EntityService _entityService; + */ @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Bean(name = "dataHubTokenService") @Scope("singleton") @Nonnull protected StatefulTokenService getInstance() { return new StatefulTokenService( - this.signingKey, this.signingAlgorithm, this.issuer, this._entityService, this.saltingKey); + signingKey, signingAlgorithm, issuer, _entityService, saltingKey); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java index 7c6c4384d7343..47af58a8d8626 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/GroupServiceFactory.java @@ -1,7 +1,7 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.group.GroupService; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.spring.YamlPropertySourceFactory; @@ -18,11 +18,7 @@ public class GroupServiceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; - - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; + private EntityService _entityService; @Autowired @Qualifier("graphClient") @@ -31,7 +27,8 @@ public class GroupServiceFactory { @Bean(name = "groupService") @Scope("singleton") @Nonnull - protected GroupService getInstance() throws Exception { - return new GroupService(this._javaEntityClient, this._entityService, this._graphClient); + protected GroupService getInstance(@Qualifier("entityClient") final EntityClient entityClient) + throws Exception { + return new GroupService(entityClient, _entityService, _graphClient); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java index c44eada46794d..7a2b14fdb0f28 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/InviteTokenServiceFactory.java @@ -1,7 +1,7 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.invite.InviteTokenService; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; @@ -15,9 +15,6 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class InviteTokenServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; @Autowired @Qualifier("dataHubSecretService") @@ -26,7 +23,8 @@ public class InviteTokenServiceFactory { @Bean(name = "inviteTokenService") @Scope("singleton") @Nonnull - protected InviteTokenService getInstance() throws Exception { - return new InviteTokenService(this._javaEntityClient, this._secretService); + protected InviteTokenService getInstance( + @Qualifier("entityClient") final EntityClient entityClient) throws Exception { + return new InviteTokenService(entityClient, _secretService); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java index 844f3a094b6b7..0ed8f1a4b7af4 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/NativeUserServiceFactory.java @@ -1,8 +1,8 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.user.NativeUserService; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; @@ -19,11 +19,7 @@ public class NativeUserServiceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; - - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; + private EntityService _entityService; @Autowired @Qualifier("dataHubSecretService") @@ -34,11 +30,8 @@ public class NativeUserServiceFactory { @Bean(name = "nativeUserService") @Scope("singleton") @Nonnull - protected NativeUserService getInstance() throws Exception { + protected NativeUserService getInstance(final SystemEntityClient entityClient) throws Exception { return new NativeUserService( - _entityService, - _javaEntityClient, - _secretService, - _configurationProvider.getAuthentication()); + _entityService, entityClient, _secretService, _configurationProvider.getAuthentication()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java index a6ae703576a3e..317d8583ef1c3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/PostServiceFactory.java @@ -1,10 +1,9 @@ package com.linkedin.gms.factory.auth; import com.datahub.authentication.post.PostService; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,14 +13,12 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class PostServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; @Bean(name = "postService") @Scope("singleton") @Nonnull - protected PostService getInstance() throws Exception { - return new PostService(this._javaEntityClient); + protected PostService getInstance(@Qualifier("entityClient") final EntityClient entityClient) + throws Exception { + return new PostService(entityClient); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java index 7696d5201493a..9321e2544a493 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/RoleServiceFactory.java @@ -1,10 +1,9 @@ package com.linkedin.gms.factory.auth; import com.datahub.authorization.role.RoleService; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -15,14 +14,11 @@ @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class RoleServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; - @Bean(name = "roleService") @Scope("singleton") @Nonnull - protected RoleService getInstance() throws Exception { - return new RoleService(this._javaEntityClient); + protected RoleService getInstance(@Qualifier("entityClient") final EntityClient entityClient) + throws Exception { + return new RoleService(entityClient); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java index 52d13b05a654d..efe688ceee3ff 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/SystemAuthenticationFactory.java @@ -34,8 +34,8 @@ public class SystemAuthenticationFactory { @Nonnull protected Authentication getInstance() { // TODO: Change to service - final Actor systemActor = new Actor(ActorType.USER, this.systemClientId); + final Actor systemActor = new Actor(ActorType.USER, systemClientId); return new Authentication( - systemActor, String.format("Basic %s:%s", this.systemClientId, this.systemSecret)); + systemActor, String.format("Basic %s:%s", systemClientId, systemSecret)); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java index 5663162186b83..465d28542f371 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/SiblingGraphServiceFactory.java @@ -18,7 +18,7 @@ public class SiblingGraphServiceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Autowired @Qualifier("graphService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index 5c7c2370ab337..e969793fac1ef 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -60,9 +60,15 @@ public class ConfigurationProvider { /** System Update configurations */ private SystemUpdateConfiguration systemUpdate; + /** The base URL where DataHub is hosted. */ + private String baseUrl; + /** Configuration for caching */ private CacheConfiguration cache; /** Configuration for the health check server */ private HealthCheckConfiguration healthCheck; + + /** Structured properties related configurations */ + private StructuredPropertiesConfiguration structuredProperties; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/StructuredPropertiesConfiguration.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/StructuredPropertiesConfiguration.java new file mode 100644 index 0000000000000..6d4d4ea30c863 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/StructuredPropertiesConfiguration.java @@ -0,0 +1,10 @@ +package com.linkedin.gms.factory.config; + +import lombok.Data; + +@Data +public class StructuredPropertiesConfiguration { + private boolean enabled; + private boolean writeEnabled; + private boolean systemUpdateEnabled; +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java index 739211855cacd..39d42b6fb7568 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/dataproduct/DataProductServiceFactory.java @@ -1,6 +1,6 @@ package com.linkedin.gms.factory.dataproduct; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.service.DataProductService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; @@ -15,9 +15,6 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class DataProductServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; @Autowired @Qualifier("graphClient") @@ -26,7 +23,8 @@ public class DataProductServiceFactory { @Bean(name = "dataProductService") @Scope("singleton") @Nonnull - protected DataProductService getInstance() throws Exception { - return new DataProductService(_javaEntityClient, _graphClient); + protected DataProductService getInstance( + @Qualifier("entityClient") final EntityClient entityClient) throws Exception { + return new DataProductService(entityClient, _graphClient); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/CassandraSessionFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/CassandraSessionFactory.java index 326537ee07cbd..788dc3777e539 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/CassandraSessionFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/CassandraSessionFactory.java @@ -9,6 +9,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.net.ssl.SSLContext; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -16,6 +17,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; +@Slf4j @Configuration public class CassandraSessionFactory { @@ -50,7 +52,7 @@ protected CqlSession createSession() { try { csb = csb.withSslContext(SSLContext.getDefault()); } catch (Exception e) { - e.printStackTrace(); + log.error("Error creating cassandra ssl session", e); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java index 8644327747281..6bc2d3c7be63f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/DeleteEntityServiceFactory.java @@ -8,7 +8,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; import org.springframework.context.annotation.Import; @Configuration @@ -16,14 +15,13 @@ public class DeleteEntityServiceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Autowired @Qualifier("graphService") private GraphService _graphService; @Bean(name = "deleteEntityService") - @DependsOn({"entityService"}) @Nonnull protected DeleteEntityService createDeleteEntityService() { return new DeleteEntityService(_entityService, _graphService); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java index 88a3f5749343b..5fd64b02d08a8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/EntityServiceFactory.java @@ -48,16 +48,14 @@ protected EntityService createInstance( final KafkaEventProducer eventProducer = new KafkaEventProducer(producer, convention, kafkaHealthChecker); FeatureFlags featureFlags = configurationProvider.getFeatureFlags(); - EntityService entityService = - new EntityServiceImpl( - aspectDao, - eventProducer, - entityRegistry, - featureFlags.isAlwaysEmitChangeLog(), - updateIndicesService, - featureFlags.getPreProcessHooks(), - _ebeanMaxTransactionRetry); - return entityService; + return new EntityServiceImpl( + aspectDao, + eventProducer, + entityRegistry, + featureFlags.isAlwaysEmitChangeLog(), + updateIndicesService, + featureFlags.getPreProcessHooks(), + _ebeanMaxTransactionRetry); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java deleted file mode 100644 index c550fc161b606..0000000000000 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/JavaEntityClientFactory.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.linkedin.gms.factory.entity; - -import com.datahub.authentication.Authentication; -import com.linkedin.entity.client.RestliEntityClient; -import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory; -import com.linkedin.metadata.client.JavaEntityClient; -import com.linkedin.metadata.client.SystemJavaEntityClient; -import com.linkedin.metadata.entity.DeleteEntityService; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; -import com.linkedin.metadata.event.EventProducer; -import com.linkedin.metadata.search.EntitySearchService; -import com.linkedin.metadata.search.LineageSearchService; -import com.linkedin.metadata.search.SearchService; -import com.linkedin.metadata.search.client.CachingEntitySearchService; -import com.linkedin.metadata.timeseries.TimeseriesAspectService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; - -@Configuration -@ConditionalOnExpression("'${entityClient.preferredImpl:java}'.equals('java')") -@Import({DataHubKafkaProducerFactory.class}) -public class JavaEntityClientFactory { - - @Autowired - @Qualifier("entityService") - private EntityService _entityService; - - @Autowired - @Qualifier("deleteEntityService") - private DeleteEntityService _deleteEntityService; - - @Autowired - @Qualifier("searchService") - private SearchService _searchService; - - @Autowired - @Qualifier("entitySearchService") - private EntitySearchService _entitySearchService; - - @Autowired - @Qualifier("cachingEntitySearchService") - private CachingEntitySearchService _cachingEntitySearchService; - - @Autowired - @Qualifier("timeseriesAspectService") - private TimeseriesAspectService _timeseriesAspectService; - - @Autowired - @Qualifier("relationshipSearchService") - private LineageSearchService _lineageSearchService; - - @Autowired - @Qualifier("kafkaEventProducer") - private EventProducer _eventProducer; - - @Bean("javaEntityClient") - public JavaEntityClient getJavaEntityClient( - @Qualifier("restliEntityClient") final RestliEntityClient restliEntityClient) { - return new JavaEntityClient( - _entityService, - _deleteEntityService, - _entitySearchService, - _cachingEntitySearchService, - _searchService, - _lineageSearchService, - _timeseriesAspectService, - _eventProducer, - restliEntityClient); - } - - @Bean("systemJavaEntityClient") - public SystemJavaEntityClient systemJavaEntityClient( - @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider, - @Qualifier("systemAuthentication") final Authentication systemAuthentication, - @Qualifier("systemRestliEntityClient") final RestliEntityClient restliEntityClient) { - SystemJavaEntityClient systemJavaEntityClient = - new SystemJavaEntityClient( - _entityService, - _deleteEntityService, - _entitySearchService, - _cachingEntitySearchService, - _searchService, - _lineageSearchService, - _timeseriesAspectService, - _eventProducer, - restliEntityClient, - systemAuthentication, - configurationProvider.getCache().getClient().getEntityClient()); - - _entityService.setSystemEntityClient(systemJavaEntityClient); - - return systemJavaEntityClient; - } -} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java index dae5f903d7d80..31ad933b9579d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RetentionServiceFactory.java @@ -33,9 +33,9 @@ public class RetentionServiceFactory { @DependsOn({"cassandraSession", "entityService"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra") @Nonnull - protected RetentionService createCassandraInstance(CqlSession session) { - RetentionService retentionService = - new CassandraRetentionService(_entityService, session, _batchSize); + protected RetentionService createCassandraInstance(CqlSession session) { + RetentionService retentionService = + new CassandraRetentionService<>(_entityService, session, _batchSize); _entityService.setRetentionService(retentionService); return retentionService; } @@ -44,9 +44,9 @@ protected RetentionService createCassandraInstance(CqlSession session) { @DependsOn({"ebeanServer", "entityService"}) @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true) @Nonnull - protected RetentionService createEbeanInstance(Database server) { - RetentionService retentionService = - new EbeanRetentionService(_entityService, server, _batchSize); + protected RetentionService createEbeanInstance(Database server) { + RetentionService retentionService = + new EbeanRetentionService<>(_entityService, server, _batchSize); _entityService.setRetentionService(retentionService); return retentionService; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RollbackServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RollbackServiceFactory.java new file mode 100644 index 0000000000000..e1055835616ea --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RollbackServiceFactory.java @@ -0,0 +1,27 @@ +package com.linkedin.gms.factory.entity; + +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.service.RollbackService; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RollbackServiceFactory { + + @Value("${authorization.restApiAuthorization:false}") + boolean restApiAuthorizationEnabled; + + @Bean + @Nonnull + protected RollbackService rollbackService( + final EntityService entityService, + final SystemMetadataService systemMetadataService, + final TimeseriesAspectService timeseriesAspectService) { + return new RollbackService( + entityService, systemMetadataService, timeseriesAspectService, restApiAuthorizationEnabled); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java index d8c1422f988c2..34c1887d67c56 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/update/indices/UpdateIndicesServiceFactory.java @@ -1,7 +1,8 @@ package com.linkedin.gms.factory.entity.update.indices; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.search.EntityIndexBuildersFactory; +import com.linkedin.metadata.client.EntityClientAspectRetriever; import com.linkedin.metadata.graph.GraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.EntitySearchService; @@ -22,7 +23,7 @@ public class UpdateIndicesServiceFactory { @Autowired private ApplicationContext context; - @Value("${entityClient.preferredImpl:java}") + @Value("${entityClient.impl:java}") private String entityClientImpl; @Bean @@ -34,18 +35,27 @@ public UpdateIndicesService updateIndicesService( EntityRegistry entityRegistry, SearchDocumentTransformer searchDocumentTransformer, EntityIndexBuilders entityIndexBuilders) { + UpdateIndicesService updateIndicesService = new UpdateIndicesService( graphService, entitySearchService, timeseriesAspectService, systemMetadataService, - entityRegistry, searchDocumentTransformer, entityIndexBuilders); if ("restli".equals(entityClientImpl)) { - updateIndicesService.setSystemEntityClient(context.getBean(SystemRestliEntityClient.class)); + /* + When restli mode the EntityService is not available. Wire in an AspectRetriever here instead + based on the entity client + */ + SystemEntityClient systemEntityClient = context.getBean(SystemEntityClient.class); + updateIndicesService.initializeAspectRetriever( + EntityClientAspectRetriever.builder() + .entityRegistry(entityRegistry) + .entityClient(systemEntityClient) + .build()); } return updateIndicesService; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java new file mode 100644 index 0000000000000..c6fe0d6e95f48 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/EntityClientConfigFactory.java @@ -0,0 +1,20 @@ +package com.linkedin.gms.factory.entityclient; + +import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class EntityClientConfigFactory { + + @Bean + public EntityClientCacheConfig entityClientCacheConfig( + @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider) { + return configurationProvider.getCache().getClient().getEntityClient(); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java new file mode 100644 index 0000000000000..530136e32662f --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java @@ -0,0 +1,85 @@ +package com.linkedin.gms.factory.entityclient; + +import com.datahub.authentication.Authentication; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.metadata.client.SystemJavaEntityClient; +import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; +import com.linkedin.metadata.entity.DeleteEntityService; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.event.EventProducer; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.LineageSearchService; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.search.client.CachingEntitySearchService; +import com.linkedin.metadata.service.RollbackService; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import javax.inject.Singleton; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +/** The *Java* Entity Client should be preferred if executing within the GMS service. */ +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +@ConditionalOnProperty(name = "entityClient.impl", havingValue = "java", matchIfMissing = true) +public class JavaEntityClientFactory { + + @Bean("entityClient") + @Singleton + public EntityClient entityClient( + final @Qualifier("entityService") EntityService _entityService, + final @Qualifier("deleteEntityService") DeleteEntityService _deleteEntityService, + final @Qualifier("searchService") SearchService _searchService, + final @Qualifier("entitySearchService") EntitySearchService _entitySearchService, + final @Qualifier("cachingEntitySearchService") CachingEntitySearchService + _cachingEntitySearchService, + final @Qualifier("timeseriesAspectService") TimeseriesAspectService _timeseriesAspectService, + final @Qualifier("relationshipSearchService") LineageSearchService _lineageSearchService, + final @Qualifier("kafkaEventProducer") EventProducer _eventProducer, + final RollbackService rollbackService) { + return new JavaEntityClient( + _entityService, + _deleteEntityService, + _entitySearchService, + _cachingEntitySearchService, + _searchService, + _lineageSearchService, + _timeseriesAspectService, + rollbackService, + _eventProducer); + } + + @Bean("systemEntityClient") + @Singleton + public SystemEntityClient systemEntityClient( + final @Qualifier("entityService") EntityService _entityService, + final @Qualifier("deleteEntityService") DeleteEntityService _deleteEntityService, + final @Qualifier("searchService") SearchService _searchService, + final @Qualifier("entitySearchService") EntitySearchService _entitySearchService, + final @Qualifier("cachingEntitySearchService") CachingEntitySearchService + _cachingEntitySearchService, + final @Qualifier("timeseriesAspectService") TimeseriesAspectService _timeseriesAspectService, + final @Qualifier("relationshipSearchService") LineageSearchService _lineageSearchService, + final @Qualifier("kafkaEventProducer") EventProducer _eventProducer, + final RollbackService rollbackService, + final EntityClientCacheConfig entityClientCacheConfig, + @Qualifier("systemAuthentication") final Authentication systemAuthentication) { + return new SystemJavaEntityClient( + _entityService, + _deleteEntityService, + _entitySearchService, + _cachingEntitySearchService, + _searchService, + _lineageSearchService, + _timeseriesAspectService, + rollbackService, + _eventProducer, + systemAuthentication, + entityClientCacheConfig); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RestliEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java similarity index 53% rename from metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RestliEntityClientFactory.java rename to metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java index 1dee8c4aa4d27..88989b1833e78 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entity/RestliEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/RestliEntityClientFactory.java @@ -1,47 +1,40 @@ -package com.linkedin.gms.factory.entity; +package com.linkedin.gms.factory.entityclient; import com.datahub.authentication.Authentication; +import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.entity.client.SystemRestliEntityClient; -import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; import com.linkedin.metadata.restli.DefaultRestliClientFactory; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import com.linkedin.parseq.retry.backoff.ExponentialBackoff; import com.linkedin.restli.client.Client; import java.net.URI; +import javax.inject.Singleton; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +/** The Java Entity Client should be preferred if executing within the GMS service. */ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +@ConditionalOnProperty(name = "entityClient.impl", havingValue = "restli") public class RestliEntityClientFactory { - @Value("${datahub.gms.host}") - private String gmsHost; - - @Value("${datahub.gms.port}") - private int gmsPort; - - @Value("${datahub.gms.useSSL}") - private boolean gmsUseSSL; - - @Value("${datahub.gms.uri}") - private String gmsUri; - - @Value("${datahub.gms.sslContext.protocol}") - private String gmsSslProtocol; - - @Value("${entityClient.retryInterval:2}") - private int retryInterval; - - @Value("${entityClient.numRetries:3}") - private int numRetries; - - @Bean("restliEntityClient") - public RestliEntityClient getRestliEntityClient() { + @Bean("entityClient") + @Singleton + public EntityClient entityClient( + @Value("${datahub.gms.host}") String gmsHost, + @Value("${datahub.gms.port}") int gmsPort, + @Value("${datahub.gms.useSSL}") boolean gmsUseSSL, + @Value("${datahub.gms.uri}") String gmsUri, + @Value("${datahub.gms.sslContext.protocol}") String gmsSslProtocol, + @Value("${entityClient.retryInterval:2}") int retryInterval, + @Value("${entityClient.numRetries:3}") int numRetries) { final Client restClient; if (gmsUri != null) { restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(gmsUri), gmsSslProtocol); @@ -52,10 +45,19 @@ public RestliEntityClient getRestliEntityClient() { return new RestliEntityClient(restClient, new ExponentialBackoff(retryInterval), numRetries); } - @Bean("systemRestliEntityClient") - public SystemRestliEntityClient systemRestliEntityClient( - @Qualifier("configurationProvider") final ConfigurationProvider configurationProvider, + @Bean("systemEntityClient") + @Singleton + public SystemEntityClient systemEntityClient( + @Value("${datahub.gms.host}") String gmsHost, + @Value("${datahub.gms.port}") int gmsPort, + @Value("${datahub.gms.useSSL}") boolean gmsUseSSL, + @Value("${datahub.gms.uri}") String gmsUri, + @Value("${datahub.gms.sslContext.protocol}") String gmsSslProtocol, + @Value("${entityClient.retryInterval:2}") int retryInterval, + @Value("${entityClient.numRetries:3}") int numRetries, + final EntityClientCacheConfig entityClientCacheConfig, @Qualifier("systemAuthentication") final Authentication systemAuthentication) { + final Client restClient; if (gmsUri != null) { restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(gmsUri), gmsSslProtocol); @@ -68,6 +70,6 @@ public SystemRestliEntityClient systemRestliEntityClient( new ExponentialBackoff(retryInterval), numRetries, systemAuthentication, - configurationProvider.getCache().getClient().getEntityClient()); + entityClientCacheConfig); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java new file mode 100644 index 0000000000000..73be819028f57 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/form/FormServiceFactory.java @@ -0,0 +1,21 @@ +package com.linkedin.gms.factory.form; + +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.metadata.service.FormService; +import com.linkedin.metadata.spring.YamlPropertySourceFactory; +import javax.annotation.Nonnull; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.Scope; + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class FormServiceFactory { + @Bean(name = "formService") + @Scope("singleton") + @Nonnull + protected FormService getInstance(final SystemEntityClient entityClient) throws Exception { + return new FormService(entityClient, entityClient.getSystemAuthentication()); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 723715a13b1c1..60697e57a9afb 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -10,17 +10,16 @@ import com.linkedin.datahub.graphql.GmsGraphQLEngineArgs; import com.linkedin.datahub.graphql.GraphQLEngine; import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory; import com.linkedin.gms.factory.common.GitVersionFactory; import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; import com.linkedin.gms.factory.common.SiblingGraphServiceFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.recommendation.RecommendationServiceFactory; -import com.linkedin.metadata.client.JavaEntityClient; -import com.linkedin.metadata.client.SystemJavaEntityClient; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.graph.GraphService; @@ -29,6 +28,7 @@ import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; import com.linkedin.metadata.service.DataProductService; +import com.linkedin.metadata.service.FormService; import com.linkedin.metadata.service.LineageService; import com.linkedin.metadata.service.OwnershipTypeService; import com.linkedin.metadata.service.QueryService; @@ -52,7 +52,6 @@ @Import({ RestHighLevelClientFactory.class, IndexConventionFactory.class, - RestliEntityClientFactory.class, RecommendationServiceFactory.class, EntityRegistryFactory.class, DataHubTokenServiceFactory.class, @@ -68,14 +67,6 @@ public class GraphQLEngineFactory { @Qualifier(IndexConventionFactory.INDEX_CONVENTION_BEAN) private IndexConvention indexConvention; - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _entityClient; - - @Autowired - @Qualifier("systemJavaEntityClient") - private SystemJavaEntityClient _systemEntityClient; - @Autowired @Qualifier("graphClient") private GraphClient _graphClient; @@ -86,7 +77,7 @@ public class GraphQLEngineFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Autowired @Qualifier("graphService") @@ -172,15 +163,21 @@ public class GraphQLEngineFactory { @Qualifier("dataProductService") private DataProductService _dataProductService; + @Autowired + @Qualifier("formService") + private FormService _formService; + @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; @Bean(name = "graphQLEngine") @Nonnull - protected GraphQLEngine getInstance() { + protected GraphQLEngine getInstance( + @Qualifier("entityClient") final EntityClient entityClient, + @Qualifier("systemEntityClient") final SystemEntityClient systemEntityClient) { GmsGraphQLEngineArgs args = new GmsGraphQLEngineArgs(); - args.setEntityClient(_entityClient); - args.setSystemEntityClient(_systemEntityClient); + args.setEntityClient(entityClient); + args.setSystemEntityClient(systemEntityClient); args.setGraphClient(_graphClient); args.setUsageClient(_usageClient); if (isAnalyticsEnabled) { @@ -215,6 +212,7 @@ protected GraphQLEngine getInstance() { args.setLineageService(_lineageService); args.setQueryService(_queryService); args.setFeatureFlags(_configProvider.getFeatureFlags()); + args.setFormService(_formService); args.setDataProductService(_dataProductService); return new GmsGraphQLEngine(args).builder().build(); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java index 78b9c5d52efdd..0ba953d66730c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ingestion/IngestionSchedulerFactory.java @@ -1,11 +1,9 @@ package com.linkedin.gms.factory.ingestion; -import com.datahub.authentication.Authentication; import com.datahub.metadata.ingestion.IngestionScheduler; -import com.linkedin.entity.client.RestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; import com.linkedin.gms.factory.config.ConfigurationProvider; -import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; import org.springframework.beans.factory.annotation.Autowired; @@ -16,18 +14,10 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.Scope; -@Import({SystemAuthenticationFactory.class, RestliEntityClientFactory.class}) +@Import({SystemAuthenticationFactory.class}) @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class IngestionSchedulerFactory { - @Autowired - @Qualifier("systemAuthentication") - private Authentication _systemAuthentication; - - @Autowired - @Qualifier("restliEntityClient") - private RestliEntityClient _entityClient; - @Autowired @Qualifier("configurationProvider") private ConfigurationProvider _configProvider; @@ -43,10 +33,10 @@ public class IngestionSchedulerFactory { @Bean(name = "ingestionScheduler") @Scope("singleton") @Nonnull - protected IngestionScheduler getInstance() { + protected IngestionScheduler getInstance(final SystemEntityClient entityClient) { return new IngestionScheduler( - _systemAuthentication, - _entityClient, + entityClient.getSystemAuthentication(), + entityClient, _configProvider.getIngestion(), _delayIntervalSeconds, _refreshIntervalSeconds); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java index d82a789c9c086..0d00218d1990e 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java @@ -127,7 +127,6 @@ use DefaultErrorHandler (does back-off retry and then logs) rather than stopping DeserializationException.class, new CommonContainerStoppingErrorHandler()); factory.setCommonErrorHandler(delegatingErrorHandler); } - log.info( String.format( "Event-based KafkaListenerContainerFactory built successfully. Consumer concurrency = %s", diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java index a88e1d971973b..c06ebae27f3af 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/schemaregistry/AwsGlueSchemaRegistryFactory.java @@ -35,7 +35,7 @@ public class AwsGlueSchemaRegistryFactory { @Bean("schemaRegistryConfig") @Nonnull - protected SchemaRegistryConfig getInstance(ConfigurationProvider configurationProvider) { + protected SchemaRegistryConfig getInstance(final ConfigurationProvider configurationProvider) { Map props = new HashMap<>(); // FIXME: Properties for this factory should come from ConfigurationProvider object, // specifically under the diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java index 1589b33862bfe..d81df694c420d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/lineage/LineageServiceFactory.java @@ -1,10 +1,9 @@ package com.linkedin.gms.factory.lineage; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.service.LineageService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,14 +13,12 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class LineageServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; @Bean(name = "lineageService") @Scope("singleton") @Nonnull - protected LineageService getInstance() throws Exception { - return new LineageService(this._javaEntityClient); + protected LineageService getInstance(@Qualifier("entityClient") final EntityClient entityClient) + throws Exception { + return new LineageService(entityClient); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java index ff48a922adf22..5403ca80fa5a8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/ownership/OwnershipTypeServiceFactory.java @@ -1,12 +1,9 @@ package com.linkedin.gms.factory.ownership; -import com.datahub.authentication.Authentication; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.OwnershipTypeService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; @@ -15,18 +12,12 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class OwnershipTypeServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; - - @Autowired - @Qualifier("systemAuthentication") - private Authentication _authentication; @Bean(name = "ownerShipTypeService") @Scope("singleton") @Nonnull - protected OwnershipTypeService getInstance() throws Exception { - return new OwnershipTypeService(_javaEntityClient, _authentication); + protected OwnershipTypeService getInstance(final SystemEntityClient entityClient) + throws Exception { + return new OwnershipTypeService(entityClient, entityClient.getSystemAuthentication()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java index cf81cbf70d5eb..64af400708e6c 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/query/QueryServiceFactory.java @@ -1,12 +1,9 @@ package com.linkedin.gms.factory.query; -import com.datahub.authentication.Authentication; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.QueryService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; @@ -15,18 +12,11 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class QueryServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; - - @Autowired - @Qualifier("systemAuthentication") - private Authentication _authentication; @Bean(name = "queryService") @Scope("singleton") @Nonnull - protected QueryService getInstance() throws Exception { - return new QueryService(_javaEntityClient, _authentication); + protected QueryService getInstance(final SystemEntityClient entityClient) throws Exception { + return new QueryService(entityClient, entityClient.getSystemAuthentication()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java index f3be4db147399..9b8707b746b29 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/MostPopularCandidateSourceFactory.java @@ -31,7 +31,7 @@ public class MostPopularCandidateSourceFactory { @Autowired @Qualifier("entityService") - private EntityService entityService; + private EntityService entityService; @Bean(name = "mostPopularCandidateSource") @Nonnull diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java index ac227faf06c4c..cfdb705dc3f6d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyEditedCandidateSourceFactory.java @@ -31,7 +31,7 @@ public class RecentlyEditedCandidateSourceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Bean(name = "recentlyEditedCandidateSource") @Nonnull diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java index 6f17846efc1cd..742ed685fd6e1 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/RecentlyViewedCandidateSourceFactory.java @@ -31,7 +31,7 @@ public class RecentlyViewedCandidateSourceFactory { @Autowired @Qualifier("entityService") - private EntityService entityService; + private EntityService entityService; @Bean(name = "recentlyViewedCandidateSource") @Nonnull diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/TopPlatformsCandidateSourceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/TopPlatformsCandidateSourceFactory.java index ad241e7717545..8b1ef069423ee 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/TopPlatformsCandidateSourceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/recommendation/candidatesource/TopPlatformsCandidateSourceFactory.java @@ -18,7 +18,7 @@ public class TopPlatformsCandidateSourceFactory { @Autowired @Qualifier("entityService") - private EntityService entityService; + private EntityService entityService; @Autowired @Qualifier("entitySearchService") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java index 2b6d495e4fe33..7b5f4e18d4d53 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/ElasticSearchServiceFactory.java @@ -1,5 +1,8 @@ package com.linkedin.gms.factory.search; +import static com.linkedin.metadata.Constants.*; + +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.linkedin.gms.factory.config.ConfigurationProvider; @@ -32,6 +35,16 @@ public class ElasticSearchServiceFactory { private static final ObjectMapper YAML_MAPPER = new YAMLMapper(); + static { + int maxSize = + Integer.parseInt( + System.getenv() + .getOrDefault(INGESTION_MAX_SERIALIZED_STRING_LENGTH, MAX_JACKSON_STRING_SIZE)); + YAML_MAPPER + .getFactory() + .setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(maxSize).build()); + } + @Autowired @Qualifier("baseElasticSearchComponents") private BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java index 17103240c938b..0d7d2e9c1855f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/LineageSearchServiceFactory.java @@ -19,6 +19,8 @@ @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class LineageSearchServiceFactory { + public static final String LINEAGE_SEARCH_SERVICE_CACHE_NAME = "relationshipSearchService"; + @Bean(name = "relationshipSearchService") @Primary @Nonnull @@ -31,7 +33,7 @@ protected LineageSearchService getInstance( return new LineageSearchService( searchService, graphService, - cacheEnabled ? cacheManager.getCache("relationshipSearchService") : null, + cacheEnabled ? cacheManager.getCache(LINEAGE_SEARCH_SERVICE_CACHE_NAME) : null, cacheEnabled, configurationProvider.getCache().getSearch().getLineage()); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java index 32ad2175c9052..1fddb51065a1d 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java @@ -1,12 +1,9 @@ package com.linkedin.gms.factory.search.views; -import com.datahub.authentication.Authentication; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.ViewService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; @@ -15,18 +12,11 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class ViewServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; - - @Autowired - @Qualifier("systemAuthentication") - private Authentication _authentication; @Bean(name = "viewService") @Scope("singleton") @Nonnull - protected ViewService getInstance() throws Exception { - return new ViewService(_javaEntityClient, _authentication); + protected ViewService getInstance(final SystemEntityClient entityClient) throws Exception { + return new ViewService(entityClient, entityClient.getSystemAuthentication()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java index f0d09a815628d..a3f533a22f7ee 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java @@ -1,12 +1,9 @@ package com.linkedin.gms.factory.settings; -import com.datahub.authentication.Authentication; -import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.service.SettingsService; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; @@ -15,18 +12,10 @@ @Configuration @PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) public class SettingsServiceFactory { - @Autowired - @Qualifier("javaEntityClient") - private JavaEntityClient _javaEntityClient; - - @Autowired - @Qualifier("systemAuthentication") - private Authentication _authentication; - @Bean(name = "settingsService") @Scope("singleton") @Nonnull - protected SettingsService getInstance() throws Exception { - return new SettingsService(_javaEntityClient, _authentication); + protected SettingsService getInstance(final SystemEntityClient entityClient) throws Exception { + return new SettingsService(entityClient, entityClient.getSystemAuthentication()); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java index b735e490f583e..393bbdf155485 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/DailyReport.java @@ -25,7 +25,7 @@ public class DailyReport { private final IndexConvention _indexConvention; private final RestHighLevelClient _elasticClient; private final ConfigurationProvider _configurationProvider; - private final EntityService _entityService; + private final EntityService _entityService; private final GitVersion _gitVersion; private static final String MIXPANEL_TOKEN = "5ee83d940754d63cacbf7d34daa6f44a"; @@ -36,7 +36,7 @@ public DailyReport( IndexConvention indexConvention, RestHighLevelClient elasticClient, ConfigurationProvider configurationProvider, - EntityService entityService, + EntityService entityService, GitVersion gitVersion) { this._indexConvention = indexConvention; this._elasticClient = elasticClient; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java index 4986e705fd7b4..7d3638d44769b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/ScheduledAnalyticsFactory.java @@ -24,7 +24,7 @@ public DailyReport dailyReport( @Qualifier("elasticSearchRestHighLevelClient") RestHighLevelClient elasticClient, @Qualifier(IndexConventionFactory.INDEX_CONVENTION_BEAN) IndexConvention indexConvention, ConfigurationProvider configurationProvider, - EntityService entityService, + EntityService entityService, GitVersion gitVersion) { return new DailyReport( indexConvention, elasticClient, configurationProvider, entityService, gitVersion); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java index 748acb4a9499e..2e8317df6b14b 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java @@ -17,7 +17,7 @@ public final class TelemetryUtils { private static String _clientId; - public static String getClientId(EntityService entityService) { + public static String getClientId(EntityService entityService) { if (_clientId == null) { createClientIdIfNotPresent(entityService); RecordTemplate clientIdTemplate = @@ -28,7 +28,7 @@ public static String getClientId(EntityService entityService) { return _clientId; } - private static void createClientIdIfNotPresent(EntityService entityService) { + private static void createClientIdIfNotPresent(EntityService entityService) { String uuid = UUID.randomUUID().toString(); TelemetryClientId clientId = new TelemetryClientId().setClientId(uuid); final AuditStamp clientIdStamp = new AuditStamp(); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java index 4e858fb5cdefd..cb0ef29b50a89 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TrackingServiceFactory.java @@ -32,7 +32,7 @@ public class TrackingServiceFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Autowired @Qualifier("gitVersion") diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java index 53a98977413e4..50d4125257fb2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/timeline/eventgenerator/EntityChangeEventGeneratorRegistryFactory.java @@ -2,8 +2,7 @@ import static com.linkedin.metadata.Constants.*; -import com.datahub.authentication.Authentication; -import com.linkedin.entity.client.SystemRestliEntityClient; +import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.timeline.eventgenerator.AssertionRunEventChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DataProcessInstanceRunEventChangeEventGenerator; import com.linkedin.metadata.timeline.eventgenerator.DatasetPropertiesChangeEventGenerator; @@ -32,12 +31,10 @@ public class EntityChangeEventGeneratorRegistryFactory { @Autowired ApplicationContext applicationContext; @Bean(name = "entityChangeEventGeneratorRegistry") - @DependsOn({"restliEntityClient", "systemAuthentication"}) + @DependsOn({"systemEntityClient"}) @Nonnull protected EntityChangeEventGeneratorRegistry entityChangeEventGeneratorRegistry() { - final SystemRestliEntityClient entityClient = - applicationContext.getBean(SystemRestliEntityClient.class); - final Authentication systemAuthentication = applicationContext.getBean(Authentication.class); + final SystemEntityClient entityClient = applicationContext.getBean(SystemEntityClient.class); final EntityChangeEventGeneratorRegistry registry = new EntityChangeEventGeneratorRegistry(); registry.register(SCHEMA_METADATA_ASPECT_NAME, new SchemaMetadataChangeEventGenerator()); registry.register( diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java index dc82fc4907edc..7ff91affdf765 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/BootstrapStep.java @@ -40,7 +40,7 @@ static Urn getUpgradeUrn(String upgradeId) { new DataHubUpgradeKey().setId(upgradeId), Constants.DATA_HUB_UPGRADE_ENTITY_NAME); } - static void setUpgradeResult(Urn urn, EntityService entityService) throws URISyntaxException { + static void setUpgradeResult(Urn urn, EntityService entityService) throws URISyntaxException { final AuditStamp auditStamp = new AuditStamp() .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java index ff5d3f215d86b..ed8a53aa594c8 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/UpgradeStep.java @@ -7,7 +7,6 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.key.DataHubUpgradeKey; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -21,12 +20,12 @@ @Slf4j public abstract class UpgradeStep implements BootstrapStep { - protected final EntityService _entityService; + protected final EntityService _entityService; private final String _version; private final String _upgradeId; private final Urn _upgradeUrn; - public UpgradeStep(EntityService entityService, String version, String upgradeId) { + public UpgradeStep(EntityService entityService, String version, String upgradeId) { this._entityService = entityService; this._version = version; this._upgradeId = upgradeId; diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java index 70fa91ae61861..b808c3da5d8d0 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java @@ -13,7 +13,9 @@ import com.linkedin.metadata.boot.steps.IndexDataPlatformsStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformInstancesStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformsStep; +import com.linkedin.metadata.boot.steps.IngestDataTypesStep; import com.linkedin.metadata.boot.steps.IngestDefaultGlobalSettingsStep; +import com.linkedin.metadata.boot.steps.IngestEntityTypesStep; import com.linkedin.metadata.boot.steps.IngestOwnershipTypesStep; import com.linkedin.metadata.boot.steps.IngestPoliciesStep; import com.linkedin.metadata.boot.steps.IngestRetentionPoliciesStep; @@ -54,7 +56,7 @@ public class BootstrapManagerFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Autowired @Qualifier("entityRegistry") @@ -131,6 +133,8 @@ protected BootstrapManager createInstance() { new WaitForSystemUpdateStep(_dataHubUpgradeKafkaListener, _configurationProvider); final IngestOwnershipTypesStep ingestOwnershipTypesStep = new IngestOwnershipTypesStep(_entityService, _ownershipTypesResource); + final IngestDataTypesStep ingestDataTypesStep = new IngestDataTypesStep(_entityService); + final IngestEntityTypesStep ingestEntityTypesStep = new IngestEntityTypesStep(_entityService); final List finalSteps = new ArrayList<>( @@ -148,7 +152,9 @@ protected BootstrapManager createInstance() { removeClientIdAspectStep, restoreDbtSiblingsIndices, indexDataPlatformsStep, - restoreColumnLineageIndices)); + restoreColumnLineageIndices, + ingestDataTypesStep, + ingestEntityTypesStep)); if (_upgradeDefaultBrowsePathsEnabled) { finalSteps.add(new UpgradeDefaultBrowsePathsStep(_entityService)); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java index 2436938c6c026..f13037c1e21c7 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/IngestRetentionPoliciesStepFactory.java @@ -26,7 +26,7 @@ public class IngestRetentionPoliciesStepFactory { @Autowired @Qualifier("entityService") - private EntityService _entityService; + private EntityService _entityService; @Value("${entityService.retention.enabled}") private Boolean _enableRetention; diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java index 770c0d2840fe8..80e139dcd5c65 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2Step.java @@ -47,7 +47,7 @@ public class BackfillBrowsePathsV2Step extends UpgradeStep { private final SearchService _searchService; - public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { + public BackfillBrowsePathsV2Step(EntityService entityService, SearchService searchService) { super(entityService, VERSION, UPGRADE_ID); _searchService = searchService; } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IndexDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IndexDataPlatformsStep.java index c46cfdd61158d..591082235ff30 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IndexDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IndexDataPlatformsStep.java @@ -34,7 +34,7 @@ public class IndexDataPlatformsStep extends UpgradeStep { private final EntityRegistry _entityRegistry; public IndexDataPlatformsStep( - EntityService entityService, + EntityService entityService, EntitySearchService entitySearchService, EntityRegistry entityRegistry) { super(entityService, VERSION, UPGRADE_ID); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java index e2f0b70526af5..716ae292338ed 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStep.java @@ -28,7 +28,7 @@ public class IngestDataPlatformInstancesStep implements BootstrapStep { private static final int BATCH_SIZE = 1000; - private final EntityService _entityService; + private final EntityService _entityService; private final AspectMigrationsDao _migrationsDao; @Override @@ -81,8 +81,7 @@ public void execute() throws Exception { .aspectName(DATA_PLATFORM_INSTANCE_ASPECT_NAME) .aspect(dataPlatformInstance.get()) .auditStamp(aspectAuditStamp) - .build( - _entityService.getEntityRegistry(), _entityService.getSystemEntityClient())); + .build(_entityService)); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java index 37eac6d5ec470..89ed493e162cc 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java @@ -31,7 +31,7 @@ public class IngestDataPlatformsStep implements BootstrapStep { private static final String PLATFORM_ASPECT_NAME = "dataPlatformInfo"; - private final EntityService _entityService; + private final EntityService _entityService; @Override public String name() { @@ -91,9 +91,7 @@ public void execute() throws IOException, URISyntaxException { new AuditStamp() .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) .setTime(System.currentTimeMillis())) - .build( - _entityService.getEntityRegistry(), - _entityService.getSystemEntityClient()); + .build(_entityService); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataTypesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataTypesStep.java new file mode 100644 index 0000000000000..6f3a415b521e4 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataTypesStep.java @@ -0,0 +1,103 @@ +package com.linkedin.metadata.boot.steps; + +import static com.linkedin.metadata.Constants.*; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.datatype.DataTypeInfo; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ClassPathResource; + +/** This bootstrap step is responsible for ingesting default data types. */ +@Slf4j +public class IngestDataTypesStep implements BootstrapStep { + + private static final String DEFAULT_FILE_PATH = "./boot/data_types.json"; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private final EntityService _entityService; + private final String _resourcePath; + + public IngestDataTypesStep(@Nonnull final EntityService entityService) { + this(entityService, DEFAULT_FILE_PATH); + } + + public IngestDataTypesStep( + @Nonnull final EntityService entityService, @Nonnull final String filePath) { + _entityService = Objects.requireNonNull(entityService, "entityService must not be null"); + _resourcePath = filePath; + } + + @Override + public String name() { + return "IngestDataTypesStep"; + } + + @Override + public void execute() throws Exception { + log.info("Ingesting default data types..."); + + // 1. Read from the file into JSON. + final JsonNode dataTypesObj = + JSON_MAPPER.readTree(new ClassPathResource(_resourcePath).getFile()); + + if (!dataTypesObj.isArray()) { + throw new RuntimeException( + String.format( + "Found malformed data types file, expected an Array but found %s", + dataTypesObj.getNodeType())); + } + + log.info("Ingesting {} data types types", dataTypesObj.size()); + int numIngested = 0; + for (final JsonNode roleObj : dataTypesObj) { + final Urn urn = Urn.createFromString(roleObj.get("urn").asText()); + final DataTypeInfo info = + RecordUtils.toRecordTemplate(DataTypeInfo.class, roleObj.get("info").toString()); + log.info(String.format("Ingesting default data type with urn %s", urn)); + ingestDataType(urn, info); + numIngested++; + } + log.info("Ingested {} new data types", numIngested); + } + + private void ingestDataType(final Urn dataTypeUrn, final DataTypeInfo info) throws Exception { + // Write key + final MetadataChangeProposal keyAspectProposal = new MetadataChangeProposal(); + final AspectSpec keyAspectSpec = _entityService.getKeyAspectSpec(dataTypeUrn.getEntityType()); + GenericAspect keyAspect = + GenericRecordUtils.serializeAspect( + EntityKeyUtils.convertUrnToEntityKey(dataTypeUrn, keyAspectSpec)); + keyAspectProposal.setAspect(keyAspect); + keyAspectProposal.setAspectName(keyAspectSpec.getName()); + keyAspectProposal.setEntityType(DATA_TYPE_ENTITY_NAME); + keyAspectProposal.setChangeType(ChangeType.UPSERT); + keyAspectProposal.setEntityUrn(dataTypeUrn); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(dataTypeUrn); + proposal.setEntityType(DATA_TYPE_ENTITY_NAME); + proposal.setAspectName(DATA_TYPE_INFO_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(info)); + proposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal( + proposal, + new AuditStamp() + .setActor(Urn.createFromString(SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()), + false); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java index 194e1ddd73c2c..1420ec116be8f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java @@ -41,15 +41,15 @@ public class IngestDefaultGlobalSettingsStep implements BootstrapStep { private static final String DEFAULT_SETTINGS_RESOURCE_PATH = "./boot/global_settings.json"; - private final EntityService _entityService; + private final EntityService _entityService; private final String _resourcePath; - public IngestDefaultGlobalSettingsStep(@Nonnull final EntityService entityService) { + public IngestDefaultGlobalSettingsStep(@Nonnull final EntityService entityService) { this(entityService, DEFAULT_SETTINGS_RESOURCE_PATH); } public IngestDefaultGlobalSettingsStep( - @Nonnull final EntityService entityService, @Nonnull final String resourcePath) { + @Nonnull final EntityService entityService, @Nonnull final String resourcePath) { _entityService = Objects.requireNonNull(entityService); _resourcePath = Objects.requireNonNull(resourcePath); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStep.java new file mode 100644 index 0000000000000..b2213eda71cae --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStep.java @@ -0,0 +1,88 @@ +package com.linkedin.metadata.boot.steps; + +import static com.linkedin.metadata.Constants.*; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entitytype.EntityTypeInfo; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.GenericAspect; +import com.linkedin.mxe.MetadataChangeProposal; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** This bootstrap step is responsible for ingesting default data types. */ +@Slf4j +public class IngestEntityTypesStep implements BootstrapStep { + + private static final String DATAHUB_NAMESPACE = "datahub"; + private final EntityService _entityService; + + public IngestEntityTypesStep(@Nonnull final EntityService entityService) { + _entityService = Objects.requireNonNull(entityService, "entityService must not be null"); + } + + @Override + public String name() { + return "IngestEntityTypesStep"; + } + + @Override + public void execute() throws Exception { + log.info("Ingesting entity types from base entity registry..."); + + log.info( + "Ingesting {} entity types", _entityService.getEntityRegistry().getEntitySpecs().size()); + int numIngested = 0; + for (final EntitySpec spec : _entityService.getEntityRegistry().getEntitySpecs().values()) { + final Urn entityTypeUrn = + UrnUtils.getUrn( + String.format("urn:li:entityType:%s.%s", DATAHUB_NAMESPACE, spec.getName())); + final EntityTypeInfo info = + new EntityTypeInfo() + .setDisplayName(spec.getName()) // TODO: Support display name in the entity registry. + .setQualifiedName(entityTypeUrn.getId()); + log.info(String.format("Ingesting entity type with urn %s", entityTypeUrn)); + ingestEntityType(entityTypeUrn, info); + numIngested++; + } + log.info("Ingested {} new entity types", numIngested); + } + + private void ingestEntityType(final Urn entityTypeUrn, final EntityTypeInfo info) + throws Exception { + // Write key + final MetadataChangeProposal keyAspectProposal = new MetadataChangeProposal(); + final AspectSpec keyAspectSpec = _entityService.getKeyAspectSpec(entityTypeUrn.getEntityType()); + GenericAspect keyAspect = + GenericRecordUtils.serializeAspect( + EntityKeyUtils.convertUrnToEntityKey(entityTypeUrn, keyAspectSpec)); + keyAspectProposal.setAspect(keyAspect); + keyAspectProposal.setAspectName(keyAspectSpec.getName()); + keyAspectProposal.setEntityType(ENTITY_TYPE_ENTITY_NAME); + keyAspectProposal.setChangeType(ChangeType.UPSERT); + keyAspectProposal.setEntityUrn(entityTypeUrn); + + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(entityTypeUrn); + proposal.setEntityType(ENTITY_TYPE_ENTITY_NAME); + proposal.setAspectName(ENTITY_TYPE_INFO_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(info)); + proposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal( + proposal, + new AuditStamp() + .setActor(Urn.createFromString(SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()), + false); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java index fc1c82fc6d631..02d965b44fc88 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestOwnershipTypesStep.java @@ -34,7 +34,7 @@ public class IngestOwnershipTypesStep implements BootstrapStep { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); - private final EntityService _entityService; + private final EntityService _entityService; private final Resource _ownershipTypesResource; @Override @@ -100,11 +100,7 @@ private void ingestOwnershipType( _entityService.ingestProposal( AspectsBatchImpl.builder() - .mcps( - List.of(keyAspectProposal, proposal), - auditStamp, - _entityService.getEntityRegistry(), - _entityService.getSystemEntityClient()) + .mcps(List.of(keyAspectProposal, proposal), auditStamp, _entityService) .build(), false); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java index 9b9feb8e14638..f925c96e333fd 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java @@ -46,7 +46,7 @@ public class IngestPoliciesStep implements BootstrapStep { private static final String POLICY_INFO_ASPECT_NAME = "dataHubPolicyInfo"; private final EntityRegistry _entityRegistry; - private final EntityService _entityService; + private final EntityService _entityService; private final EntitySearchService _entitySearchService; private final SearchDocumentTransformer _searchDocumentTransformer; @@ -210,8 +210,7 @@ private void ingestPolicy(final Urn urn, final DataHubPolicyInfo info) throws UR new AuditStamp() .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) .setTime(System.currentTimeMillis()), - _entityRegistry, - _entityService.getSystemEntityClient()) + _entityService) .build(), false); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRolesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRolesStep.java index 9ce4d9ce644a8..28b556e78de12 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRolesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRolesStep.java @@ -31,7 +31,7 @@ @RequiredArgsConstructor public class IngestRolesStep implements BootstrapStep { private static final int SLEEP_SECONDS = 60; - private final EntityService _entityService; + private final EntityService _entityService; private final EntityRegistry _entityRegistry; @Override @@ -130,8 +130,7 @@ private void ingestRole( new AuditStamp() .setActor(Urn.createFromString(SYSTEM_ACTOR)) .setTime(System.currentTimeMillis()), - _entityRegistry, - _entityService.getSystemEntityClient()) + _entityService) .build(), false); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRootUserStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRootUserStep.java index 9e00b960482c5..1f8127d8be108 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRootUserStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestRootUserStep.java @@ -29,7 +29,7 @@ public class IngestRootUserStep implements BootstrapStep { private static final String USER_INFO_ASPECT_NAME = "corpUserInfo"; - private final EntityService _entityService; + private final EntityService _entityService; @Override public String name() { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndices.java index 919ba93c9213e..2e60df54452cc 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndices.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndices.java @@ -10,7 +10,6 @@ import com.linkedin.metadata.boot.UpgradeStep; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.ExtraInfo; @@ -31,8 +30,7 @@ public class RestoreColumnLineageIndices extends UpgradeStep { private final EntityRegistry _entityRegistry; public RestoreColumnLineageIndices( - @Nonnull final EntityService entityService, - @Nonnull final EntityRegistry entityRegistry) { + @Nonnull final EntityService entityService, @Nonnull final EntityRegistry entityRegistry) { super(entityService, VERSION, UPGRADE_ID); _entityRegistry = Objects.requireNonNull(entityRegistry, "entityRegistry must not be null"); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java index e2d367a034491..789a4cbd11878 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreDbtSiblingsIndices.java @@ -13,7 +13,6 @@ import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.BootstrapStep; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; import com.linkedin.metadata.key.DataHubUpgradeKey; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; @@ -47,7 +46,7 @@ public class RestoreDbtSiblingsIndices implements BootstrapStep { private static final Integer BATCH_SIZE = 1000; private static final Integer SLEEP_SECONDS = 120; - private final EntityService _entityService; + private final EntityService _entityService; private final EntityRegistry _entityRegistry; @Override diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java index 319bbd084e05c..5c2b2c28e6dcf 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java @@ -38,7 +38,7 @@ public class RestoreGlossaryIndices extends UpgradeStep { private final EntityRegistry _entityRegistry; public RestoreGlossaryIndices( - EntityService entityService, + EntityService entityService, EntitySearchService entitySearchService, EntityRegistry entityRegistry) { super(entityService, VERSION, UPGRADE_ID); diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java index e2d59b505a568..3eedbb48aaeca 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStep.java @@ -39,7 +39,7 @@ public class UpgradeDefaultBrowsePathsStep extends UpgradeStep { private static final String UPGRADE_ID = "upgrade-default-browse-paths-step"; private static final Integer BATCH_SIZE = 5000; - public UpgradeDefaultBrowsePathsStep(EntityService entityService) { + public UpgradeDefaultBrowsePathsStep(EntityService entityService) { super(entityService, VERSION, UPGRADE_ID); } diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java new file mode 100644 index 0000000000000..87f1546bd9557 --- /dev/null +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/ElasticSearchIndexBuilderFactoryDefaultsTest.java @@ -0,0 +1,27 @@ +package com.linkedin.gms.factory.search; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; +import java.util.Map; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.Test; + +@TestPropertySource(locations = "classpath:/application.yml") +@SpringBootTest(classes = {ElasticSearchIndexBuilderFactory.class}) +@EnableConfigurationProperties(ConfigurationProvider.class) +public class ElasticSearchIndexBuilderFactoryDefaultsTest extends AbstractTestNGSpringContextTests { + @Autowired ESIndexBuilder test; + + @Test + void testInjection() { + assertNotNull(test); + assertEquals(Map.of(), test.getIndexSettingOverrides()); + } +} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java index 8268eeff48c5e..0657141562089 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/BackfillBrowsePathsV2StepTest.java @@ -76,7 +76,7 @@ public class BackfillBrowsePathsV2StepTest { @Test public void testExecuteNoExistingBrowsePaths() throws Exception { - final EntityService mockService = initMockService(); + final EntityService mockService = initMockService(); final SearchService mockSearchService = initMockSearchService(); final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); @@ -110,7 +110,7 @@ public void testExecuteNoExistingBrowsePaths() throws Exception { @Test public void testDoesNotRunWhenAlreadyExecuted() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final SearchService mockSearchService = initMockSearchService(); final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); @@ -140,8 +140,8 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { Mockito.anyBoolean()); } - private EntityService initMockService() throws URISyntaxException { - final EntityService mockService = Mockito.mock(EntityService.class); + private EntityService initMockService() throws URISyntaxException { + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry registry = new UpgradeDefaultBrowsePathsStepTest.TestEntityRegistry(); Mockito.when(mockService.getEntityRegistry()).thenReturn(registry); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java index 41672a07a2389..1ac0f2f4f914a 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataPlatformInstancesStepTest.java @@ -39,7 +39,7 @@ public class IngestDataPlatformInstancesStepTest { @Test public void testExecuteDoesNothingWhenDataPlatformInstanceAspectsAlreadyExists() throws Exception { - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); mockDBWithDataPlatformInstanceAspects(migrationsDao); @@ -55,7 +55,7 @@ public void testExecuteDoesNothingWhenDataPlatformInstanceAspectsAlreadyExists() @Test public void testExecuteCopesWithEmptyDB() throws Exception { - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); mockEmptyDB(migrationsDao); @@ -73,7 +73,7 @@ public void testExecuteCopesWithEmptyDB() throws Exception { @Test public void testExecuteChecksKeySpecForAllUrns() throws Exception { final EntityRegistry entityRegistry = getTestEntityRegistry(); - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); final int countOfCorpUserEntities = 2; final int countOfChartEntities = 4; @@ -96,7 +96,7 @@ public void testExecuteChecksKeySpecForAllUrns() throws Exception { @Test public void testExecuteWhenSomeEntitiesShouldReceiveDataPlatformInstance() throws Exception { final EntityRegistry entityRegistry = getTestEntityRegistry(); - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); final AspectMigrationsDao migrationsDao = mock(AspectMigrationsDao.class); final int countOfCorpUserEntities = 5; final int countOfChartEntities = 7; @@ -161,7 +161,7 @@ private void mockEmptyDB(AspectMigrationsDao migrationsDao) { private void mockDBWithWorkToDo( EntityRegistry entityRegistry, - EntityService entityService, + EntityService entityService, AspectMigrationsDao migrationsDao, int countOfCorpUserEntities, int countOfChartEntities) { @@ -194,7 +194,7 @@ private List insertMockEntities( String entity, String urnTemplate, EntityRegistry entityRegistry, - EntityService entityService) { + EntityService entityService) { EntitySpec entitySpec = entityRegistry.getEntitySpec(entity); AspectSpec keySpec = entitySpec.getKeyAspectSpec(); List urns = new ArrayList<>(); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java new file mode 100644 index 0000000000000..2bbd06c8a61a4 --- /dev/null +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDataTypesStepTest.java @@ -0,0 +1,81 @@ +package com.linkedin.metadata.boot.steps; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datatype.DataTypeInfo; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import org.jetbrains.annotations.NotNull; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class IngestDataTypesStepTest { + + private static final Urn TEST_DATA_TYPE_URN = UrnUtils.getUrn("urn:li:dataType:datahub.test"); + + @Test + public void testExecuteValidDataTypesNoExistingDataTypes() throws Exception { + EntityRegistry testEntityRegistry = getTestEntityRegistry(); + final EntityService entityService = mock(EntityService.class); + when(entityService.getEntityRegistry()).thenReturn(testEntityRegistry); + when(entityService.getKeyAspectSpec(anyString())) + .thenAnswer( + args -> testEntityRegistry.getEntitySpec(args.getArgument(0)).getKeyAspectSpec()); + + final IngestDataTypesStep step = + new IngestDataTypesStep(entityService, "./boot/test_data_types_valid.json"); + + step.execute(); + + DataTypeInfo expectedResult = new DataTypeInfo(); + expectedResult.setDescription("Test Description"); + expectedResult.setDisplayName("Test Name"); + expectedResult.setQualifiedName("datahub.test"); + + Mockito.verify(entityService, times(1)) + .ingestProposal( + Mockito.eq(buildUpdateDataTypeProposal(expectedResult)), + Mockito.any(AuditStamp.class), + Mockito.eq(false)); + } + + @Test + public void testExecuteInvalidJson() throws Exception { + final EntityService entityService = mock(EntityService.class); + + final IngestDataTypesStep step = + new IngestDataTypesStep(entityService, "./boot/test_data_types_invalid.json"); + + Assert.assertThrows(RuntimeException.class, step::execute); + + // Verify no interactions + verifyNoInteractions(entityService); + } + + private static MetadataChangeProposal buildUpdateDataTypeProposal(final DataTypeInfo info) { + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(TEST_DATA_TYPE_URN); + mcp.setEntityType(DATA_TYPE_ENTITY_NAME); + mcp.setAspectName(DATA_TYPE_INFO_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(info)); + return mcp; + } + + @NotNull + private ConfigEntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + IngestDataPlatformInstancesStepTest.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yaml")); + } +} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java index b28a6e9f5cc5b..783c82934599c 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java @@ -25,7 +25,7 @@ public class IngestDefaultGlobalSettingsStepTest { @Test public void testExecuteValidSettingsNoExistingSettings() throws Exception { - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); configureEntityServiceMock(entityService, null); final IngestDefaultGlobalSettingsStep step = @@ -49,7 +49,7 @@ public void testExecuteValidSettingsNoExistingSettings() throws Exception { public void testExecuteValidSettingsExistingSettings() throws Exception { // Verify that the user provided settings overrides are NOT overwritten. - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); final GlobalSettingsInfo existingSettings = new GlobalSettingsInfo() .setViews( @@ -77,7 +77,7 @@ public void testExecuteValidSettingsExistingSettings() throws Exception { @Test public void testExecuteInvalidJsonSettings() throws Exception { - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); configureEntityServiceMock(entityService, null); final IngestDefaultGlobalSettingsStep step = @@ -92,7 +92,7 @@ public void testExecuteInvalidJsonSettings() throws Exception { @Test public void testExecuteInvalidModelSettings() throws Exception { - final EntityService entityService = mock(EntityService.class); + final EntityService entityService = mock(EntityService.class); configureEntityServiceMock(entityService, null); final IngestDefaultGlobalSettingsStep step = @@ -106,7 +106,7 @@ public void testExecuteInvalidModelSettings() throws Exception { } private static void configureEntityServiceMock( - final EntityService mockService, final GlobalSettingsInfo settingsInfo) { + final EntityService mockService, final GlobalSettingsInfo settingsInfo) { Mockito.when( mockService.getAspect( Mockito.eq(GLOBAL_SETTINGS_URN), diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java new file mode 100644 index 0000000000000..0b87283fbe2f7 --- /dev/null +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestEntityTypesStepTest.java @@ -0,0 +1,91 @@ +package com.linkedin.metadata.boot.steps; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.entitytype.EntityTypeInfo; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.registry.ConfigEntityRegistry; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import org.jetbrains.annotations.NotNull; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class IngestEntityTypesStepTest { + + @Test + public void testExecuteTestEntityRegistry() throws Exception { + EntityRegistry testEntityRegistry = getTestEntityRegistry(); + final EntityService entityService = mock(EntityService.class); + when(entityService.getEntityRegistry()).thenReturn(testEntityRegistry); + when(entityService.getKeyAspectSpec(anyString())) + .thenAnswer( + args -> testEntityRegistry.getEntitySpec(args.getArgument(0)).getKeyAspectSpec()); + + final IngestEntityTypesStep step = new IngestEntityTypesStep(entityService); + + step.execute(); + + Urn userUrn = + Urn.createFromString(String.format("urn:li:entityType:datahub.%s", CORP_USER_ENTITY_NAME)); + EntityTypeInfo userInfo = new EntityTypeInfo(); + userInfo.setDisplayName("corpuser"); + userInfo.setQualifiedName("datahub.corpuser"); + + Urn chartUrn = + Urn.createFromString(String.format("urn:li:entityType:datahub.%s", CHART_ENTITY_NAME)); + EntityTypeInfo chartInfo = new EntityTypeInfo(); + chartInfo.setDisplayName("chart"); + chartInfo.setQualifiedName("datahub.chart"); + + Urn dataPlatformUrn = + Urn.createFromString( + String.format("urn:li:entityType:datahub.%s", DATA_PLATFORM_ENTITY_NAME)); + EntityTypeInfo dataPlatformInfo = new EntityTypeInfo(); + dataPlatformInfo.setDisplayName("dataPlatform"); + dataPlatformInfo.setQualifiedName("datahub.dataPlatform"); + + // Verify all entities were ingested. + Mockito.verify(entityService, times(1)) + .ingestProposal( + Mockito.eq(buildUpdateEntityTypeProposal(userUrn, userInfo)), + Mockito.any(AuditStamp.class), + Mockito.eq(false)); + + Mockito.verify(entityService, times(1)) + .ingestProposal( + Mockito.eq(buildUpdateEntityTypeProposal(chartUrn, chartInfo)), + Mockito.any(AuditStamp.class), + Mockito.eq(false)); + + Mockito.verify(entityService, times(1)) + .ingestProposal( + Mockito.eq(buildUpdateEntityTypeProposal(dataPlatformUrn, dataPlatformInfo)), + Mockito.any(AuditStamp.class), + Mockito.eq(false)); + } + + private static MetadataChangeProposal buildUpdateEntityTypeProposal( + final Urn entityTypeUrn, final EntityTypeInfo info) { + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(entityTypeUrn); + mcp.setEntityType(ENTITY_TYPE_ENTITY_NAME); + mcp.setAspectName(ENTITY_TYPE_INFO_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(info)); + return mcp; + } + + @NotNull + private ConfigEntityRegistry getTestEntityRegistry() { + return new ConfigEntityRegistry( + IngestDataPlatformInstancesStepTest.class + .getClassLoader() + .getResourceAsStream("test-entity-registry.yaml")); + } +} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndicesTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndicesTest.java index 3b23368d8e99f..9e647da9ef2e9 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndicesTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreColumnLineageIndicesTest.java @@ -46,7 +46,7 @@ public class RestoreColumnLineageIndicesTest { @Test public void testExecuteFirstTime() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); mockGetUpgradeStep(false, VERSION_1, mockService); @@ -109,7 +109,7 @@ public void testExecuteFirstTime() throws Exception { @Test public void testExecuteWithNewVersion() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); mockGetUpgradeStep(true, VERSION_2, mockService); @@ -172,7 +172,7 @@ public void testExecuteWithNewVersion() throws Exception { @Test public void testDoesNotExecuteWithSameVersion() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); mockGetUpgradeStep(true, VERSION_1, mockService); @@ -233,7 +233,8 @@ public void testDoesNotExecuteWithSameVersion() throws Exception { Mockito.eq(ChangeType.RESTATE)); } - private void mockGetUpstreamLineage(@Nonnull Urn datasetUrn, @Nonnull EntityService mockService) { + private void mockGetUpstreamLineage( + @Nonnull Urn datasetUrn, @Nonnull EntityService mockService) { final List extraInfos = ImmutableList.of( new ExtraInfo() @@ -276,7 +277,7 @@ private void mockGetUpstreamLineage(@Nonnull Urn datasetUrn, @Nonnull EntityServ } private void mockGetInputFields( - @Nonnull Urn entityUrn, @Nonnull String entityName, @Nonnull EntityService mockService) { + @Nonnull Urn entityUrn, @Nonnull String entityName, @Nonnull EntityService mockService) { final List extraInfos = ImmutableList.of( new ExtraInfo() @@ -325,7 +326,7 @@ private AspectSpec mockAspectSpecs(@Nonnull EntityRegistry mockRegistry) { } private void mockGetUpgradeStep( - boolean shouldReturnResponse, @Nonnull String version, @Nonnull EntityService mockService) + boolean shouldReturnResponse, @Nonnull String version, @Nonnull EntityService mockService) throws Exception { final Urn upgradeEntityUrn = UrnUtils.getUrn(COLUMN_LINEAGE_UPGRADE_URN); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java index a4f0c5e0aaba0..4a4532763f02b 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndicesTest.java @@ -40,7 +40,7 @@ public class RestoreGlossaryIndicesTest { "urn:li:%s:%s", Constants.DATA_HUB_UPGRADE_ENTITY_NAME, "restore-glossary-indices-ui"); private void mockGetTermInfo( - Urn glossaryTermUrn, EntitySearchService mockSearchService, EntityService mockService) + Urn glossaryTermUrn, EntitySearchService mockSearchService, EntityService mockService) throws Exception { Map termInfoAspects = new HashMap<>(); termInfoAspects.put( @@ -79,7 +79,7 @@ private void mockGetTermInfo( } private void mockGetNodeInfo( - Urn glossaryNodeUrn, EntitySearchService mockSearchService, EntityService mockService) + Urn glossaryNodeUrn, EntitySearchService mockSearchService, EntityService mockService) throws Exception { Map nodeInfoAspects = new HashMap<>(); nodeInfoAspects.put( @@ -140,7 +140,7 @@ public void testExecuteFirstTime() throws Exception { Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); @@ -215,7 +215,7 @@ public void testExecutesWithNewVersion() throws Exception { Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); @@ -298,7 +298,7 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { Urn.createFromString("urn:li:glossaryTerm:11115397daf94708a8822b8106cfd451"); final Urn glossaryNodeUrn = Urn.createFromString("urn:li:glossaryNode:22225397daf94708a8822b8106cfd451"); - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntitySearchService mockSearchService = Mockito.mock(EntitySearchService.class); final EntityRegistry mockRegistry = Mockito.mock(EntityRegistry.class); diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java index 17159ba1baf53..024ad7b16a844 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/UpgradeDefaultBrowsePathsStepTest.java @@ -12,6 +12,7 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.ListResult; import com.linkedin.metadata.models.AspectSpec; @@ -19,7 +20,6 @@ import com.linkedin.metadata.models.EntitySpecBuilder; import com.linkedin.metadata.models.EventSpec; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import com.linkedin.metadata.query.ExtraInfo; import com.linkedin.metadata.query.ExtraInfoArray; import com.linkedin.metadata.query.ListResultMetadata; @@ -48,7 +48,7 @@ public class UpgradeDefaultBrowsePathsStepTest { @Test public void testExecuteNoExistingBrowsePaths() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry registry = new TestEntityRegistry(); Mockito.when(mockService.getEntityRegistry()).thenReturn(registry); @@ -104,7 +104,7 @@ public void testExecuteFirstTime() throws Exception { Urn testUrn2 = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset2,PROD)"); - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry registry = new TestEntityRegistry(); Mockito.when(mockService.getEntityRegistry()).thenReturn(registry); Mockito.when(mockService.buildDefaultBrowsePath(Mockito.eq(testUrn1))) @@ -193,7 +193,7 @@ public void testDoesNotRunWhenBrowsePathIsNotQualified() throws Exception { "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset4,PROD)"); // Do not // migrate - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final EntityRegistry registry = new TestEntityRegistry(); Mockito.when(mockService.getEntityRegistry()).thenReturn(registry); @@ -269,7 +269,7 @@ public void testDoesNotRunWhenBrowsePathIsNotQualified() throws Exception { @Test public void testDoesNotRunWhenAlreadyExecuted() throws Exception { - final EntityService mockService = Mockito.mock(EntityService.class); + final EntityService mockService = Mockito.mock(EntityService.class); final Urn upgradeEntityUrn = Urn.createFromString(UPGRADE_URN); com.linkedin.upgrade.DataHubUpgradeRequest upgradeRequest = @@ -297,7 +297,7 @@ public void testDoesNotRunWhenAlreadyExecuted() throws Exception { Mockito.anyBoolean()); } - private void initMockServiceOtherEntities(EntityService mockService) { + private void initMockServiceOtherEntities(EntityService mockService) { List skippedEntityTypes = ImmutableList.of( Constants.DASHBOARD_ENTITY_NAME, diff --git a/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java b/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java index 9931f044931b6..17bf7810f71e4 100644 --- a/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java +++ b/metadata-service/factories/src/test/java/io/datahubproject/telemetry/TelemetryUtilsTest.java @@ -12,7 +12,7 @@ public class TelemetryUtilsTest { - EntityService _entityService; + EntityService _entityService; @BeforeMethod public void init() { diff --git a/metadata-service/factories/src/test/resources/boot/test_data_types_invalid.json b/metadata-service/factories/src/test/resources/boot/test_data_types_invalid.json new file mode 100644 index 0000000000000..ed1d8a7b45abe --- /dev/null +++ b/metadata-service/factories/src/test/resources/boot/test_data_types_invalid.json @@ -0,0 +1,9 @@ +[ + { + "urn": "urn:li:dataType:datahub.test", + "badField": { + "qualifiedName":"datahub.test", + "description": "Test Description" + } + } +] \ No newline at end of file diff --git a/metadata-service/factories/src/test/resources/boot/test_data_types_valid.json b/metadata-service/factories/src/test/resources/boot/test_data_types_valid.json new file mode 100644 index 0000000000000..3694c92947aa1 --- /dev/null +++ b/metadata-service/factories/src/test/resources/boot/test_data_types_valid.json @@ -0,0 +1,10 @@ +[ + { + "urn": "urn:li:dataType:datahub.test", + "info": { + "qualifiedName":"datahub.test", + "displayName": "Test Name", + "description": "Test Description" + } + } +] \ No newline at end of file diff --git a/metadata-service/factories/src/test/resources/test-entity-registry.yaml b/metadata-service/factories/src/test/resources/test-entity-registry.yaml index fe32b413751e6..400b22446c186 100644 --- a/metadata-service/factories/src/test/resources/test-entity-registry.yaml +++ b/metadata-service/factories/src/test/resources/test-entity-registry.yaml @@ -13,4 +13,20 @@ entities: category: core keyAspect: dataPlatformKey aspects: - - dataPlatformInfo \ No newline at end of file + - dataPlatformInfo + - name: entityType + doc: A type of entity in the DataHub Metadata Model. + category: core + keyAspect: entityTypeKey + aspects: + - entityTypeInfo + - institutionalMemory + - status + - name: dataType + doc: A type of data element stored within DataHub. + category: core + keyAspect: dataTypeKey + aspects: + - dataTypeInfo + - institutionalMemory + - status \ No newline at end of file diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle index fb49727fa70d1..016ac6693f55b 100644 --- a/metadata-service/openapi-entity-servlet/build.gradle +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -75,7 +75,7 @@ task openApiGenerate(type: GenerateSwaggerCode, dependsOn: [mergeApiComponents, 'java11' : "true", 'modelPropertyNaming': "original", 'modelPackage' : "io.datahubproject.openapi.generated", - 'apiPackage' : "io.datahubproject.openapi.generated.controller", + 'apiPackage' : "io.datahubproject.openapi.v2.generated.controller", 'delegatePattern' : "false" ] } diff --git a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java similarity index 86% rename from metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java rename to metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java index d7c8268903508..39a7e4722988e 100644 --- a/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/delegates/EntityApiDelegateImpl.java +++ b/metadata-service/openapi-entity-servlet/src/main/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImpl.java @@ -1,4 +1,4 @@ -package io.datahubproject.openapi.delegates; +package io.datahubproject.openapi.v2.delegates; import static io.datahubproject.openapi.util.ReflectionCache.toLowerFirst; @@ -35,10 +35,16 @@ import io.datahubproject.openapi.generated.DeprecationAspectResponseV2; import io.datahubproject.openapi.generated.DomainsAspectRequestV2; import io.datahubproject.openapi.generated.DomainsAspectResponseV2; +import io.datahubproject.openapi.generated.DynamicFormAssignmentAspectRequestV2; +import io.datahubproject.openapi.generated.DynamicFormAssignmentAspectResponseV2; import io.datahubproject.openapi.generated.EditableChartPropertiesAspectRequestV2; import io.datahubproject.openapi.generated.EditableChartPropertiesAspectResponseV2; import io.datahubproject.openapi.generated.EditableDatasetPropertiesAspectRequestV2; import io.datahubproject.openapi.generated.EditableDatasetPropertiesAspectResponseV2; +import io.datahubproject.openapi.generated.FormInfoAspectRequestV2; +import io.datahubproject.openapi.generated.FormInfoAspectResponseV2; +import io.datahubproject.openapi.generated.FormsAspectRequestV2; +import io.datahubproject.openapi.generated.FormsAspectResponseV2; import io.datahubproject.openapi.generated.GlobalTagsAspectRequestV2; import io.datahubproject.openapi.generated.GlobalTagsAspectResponseV2; import io.datahubproject.openapi.generated.GlossaryTermsAspectRequestV2; @@ -66,7 +72,7 @@ public class EntityApiDelegateImpl { private final EntityRegistry _entityRegistry; - private final EntityService _entityService; + private final EntityService _entityService; private final SearchService _searchService; private final EntitiesController _v1Controller; private final AuthorizerChain _authorizationChain; @@ -79,7 +85,7 @@ public class EntityApiDelegateImpl { private final StackWalker walker = StackWalker.getInstance(); public EntityApiDelegateImpl( - EntityService entityService, + EntityService entityService, SearchService searchService, EntitiesController entitiesController, boolean restApiAuthorizationEnabled, @@ -732,4 +738,111 @@ public ResponseEntity deleteDataProductProperties(String urn) { walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); return deleteAspect(urn, methodNameToAspectName(methodName)); } + + public ResponseEntity createForms(FormsAspectRequestV2 body, String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect( + urn, + methodNameToAspectName(methodName), + body, + FormsAspectRequestV2.class, + FormsAspectResponseV2.class); + } + + public ResponseEntity deleteForms(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getForms( + String urn, @jakarta.validation.Valid Boolean systemMetadata) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect( + urn, + systemMetadata, + methodNameToAspectName(methodName), + _respClazz, + FormsAspectResponseV2.class); + } + + public ResponseEntity headForms(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity createDynamicFormAssignment( + DynamicFormAssignmentAspectRequestV2 body, String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect( + urn, + methodNameToAspectName(methodName), + body, + DynamicFormAssignmentAspectRequestV2.class, + DynamicFormAssignmentAspectResponseV2.class); + } + + public ResponseEntity createFormInfo( + FormInfoAspectRequestV2 body, String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return createAspect( + urn, + methodNameToAspectName(methodName), + body, + FormInfoAspectRequestV2.class, + FormInfoAspectResponseV2.class); + } + + public ResponseEntity deleteDynamicFormAssignment(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headDynamicFormAssignment(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity headFormInfo(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return headAspect(urn, methodNameToAspectName(methodName)); + } + + public ResponseEntity getFormInfo( + String urn, @jakarta.validation.Valid Boolean systemMetadata) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect( + urn, + systemMetadata, + methodNameToAspectName(methodName), + _respClazz, + FormInfoAspectResponseV2.class); + } + + public ResponseEntity getDynamicFormAssignment( + String urn, @jakarta.validation.Valid Boolean systemMetadata) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return getAspect( + urn, + systemMetadata, + methodNameToAspectName(methodName), + _respClazz, + DynamicFormAssignmentAspectResponseV2.class); + } + + public ResponseEntity deleteFormInfo(String urn) { + String methodName = + walker.walk(frames -> frames.findFirst().map(StackWalker.StackFrame::getMethodName)).get(); + return deleteAspect(urn, methodNameToAspectName(methodName)); + } } diff --git a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache index 4a29b95eabc5d..7ac087f220561 100644 --- a/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache +++ b/metadata-service/openapi-entity-servlet/src/main/resources/JavaSpring/apiController.mustache @@ -1,6 +1,6 @@ package {{package}}; -import io.datahubproject.openapi.delegates.EntityApiDelegateImpl; +import io.datahubproject.openapi.v2.delegates.EntityApiDelegateImpl; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.search.SearchService; import io.datahubproject.openapi.entities.EntitiesController; diff --git a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java similarity index 97% rename from metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java rename to metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java index 1f8f0a5023513..d4217c9fd1b66 100644 --- a/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/delegates/EntityApiDelegateImplTest.java +++ b/metadata-service/openapi-entity-servlet/src/test/java/io/datahubproject/openapi/v2/delegates/EntityApiDelegateImplTest.java @@ -1,4 +1,4 @@ -package io.datahubproject.openapi.delegates; +package io.datahubproject.openapi.v2.delegates; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.testng.Assert.*; @@ -32,8 +32,8 @@ import io.datahubproject.openapi.generated.Status; import io.datahubproject.openapi.generated.StatusAspectRequestV2; import io.datahubproject.openapi.generated.TagAssociation; -import io.datahubproject.openapi.generated.controller.ChartApiController; -import io.datahubproject.openapi.generated.controller.DatasetApiController; +import io.datahubproject.openapi.v2.generated.controller.ChartApiController; +import io.datahubproject.openapi.v2.generated.controller.DatasetApiController; import java.util.List; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; @@ -51,7 +51,7 @@ import org.testng.annotations.Test; @SpringBootTest(classes = {SpringWebConfig.class}) -@ComponentScan(basePackages = {"io.datahubproject.openapi.generated.controller"}) +@ComponentScan(basePackages = {"io.datahubproject.openapi.v2.generated.controller"}) @Import({OpenAPIEntityTestConfiguration.class}) @AutoConfigureMockMvc public class EntityApiDelegateImplTest extends AbstractTestNGSpringContextTests { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java index cc040d29657b2..f4689a9862825 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/GlobalControllerExceptionHandler.java @@ -1,14 +1,25 @@ package io.datahubproject.openapi; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.ConversionNotSupportedException; +import org.springframework.core.Ordered; import org.springframework.core.convert.ConversionFailedException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.servlet.mvc.support.DefaultHandlerExceptionResolver; +@Slf4j @ControllerAdvice -public class GlobalControllerExceptionHandler { - @ExceptionHandler(ConversionFailedException.class) +public class GlobalControllerExceptionHandler extends DefaultHandlerExceptionResolver { + + public GlobalControllerExceptionHandler() { + setOrder(Ordered.HIGHEST_PRECEDENCE); + setWarnLogCategory(getClass().getName()); + } + + @ExceptionHandler({ConversionFailedException.class, ConversionNotSupportedException.class}) public ResponseEntity handleConflict(RuntimeException ex) { return new ResponseEntity<>(ex.getMessage(), HttpStatus.BAD_REQUEST); } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java index a8721b23d1fa2..2336bea565e59 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/config/SpringWebConfig.java @@ -4,7 +4,9 @@ import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.info.Info; import io.swagger.v3.oas.annotations.servers.Server; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.springdoc.core.models.GroupedOpenApi; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -23,6 +25,20 @@ servers = {@Server(url = "/openapi/", description = "Default Server URL")}) @Configuration public class SpringWebConfig implements WebMvcConfigurer { + private static final Set OPERATIONS_PACKAGES = + Set.of("io.datahubproject.openapi.operations", "io.datahubproject.openapi.health"); + private static final Set V2_PACKAGES = Set.of("io.datahubproject.openapi.v2"); + private static final Set SCHEMA_REGISTRY_PACKAGES = + Set.of("io.datahubproject.openapi.schema.registry"); + + public static final Set NONDEFAULT_OPENAPI_PACKAGES; + + static { + NONDEFAULT_OPENAPI_PACKAGES = new HashSet<>(); + NONDEFAULT_OPENAPI_PACKAGES.addAll(OPERATIONS_PACKAGES); + NONDEFAULT_OPENAPI_PACKAGES.addAll(V2_PACKAGES); + NONDEFAULT_OPENAPI_PACKAGES.addAll(SCHEMA_REGISTRY_PACKAGES); + } @Override public void configureMessageConverters(List> messageConverters) { @@ -41,16 +57,23 @@ public void addFormatters(FormatterRegistry registry) { public GroupedOpenApi defaultOpenApiGroup() { return GroupedOpenApi.builder() .group("default") - .packagesToExclude( - "io.datahubproject.openapi.operations", "io.datahubproject.openapi.health") + .packagesToExclude(NONDEFAULT_OPENAPI_PACKAGES.toArray(String[]::new)) .build(); } @Bean public GroupedOpenApi operationsOpenApiGroup() { return GroupedOpenApi.builder() - .group("operations") - .packagesToScan("io.datahubproject.openapi.operations", "io.datahubproject.openapi.health") + .group("Operations") + .packagesToScan(OPERATIONS_PACKAGES.toArray(String[]::new)) + .build(); + } + + @Bean + public GroupedOpenApi openApiGroupV3() { + return GroupedOpenApi.builder() + .group("OpenAPI v2") + .packagesToScan(V2_PACKAGES.toArray(String[]::new)) .build(); } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java index c87820465dc88..a7e88966e4f87 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/MappingUtil.java @@ -465,11 +465,7 @@ public static Pair ingestProposal( AspectsBatch batch = AspectsBatchImpl.builder() - .mcps( - proposalStream.collect(Collectors.toList()), - auditStamp, - entityService.getEntityRegistry(), - entityService.getSystemEntityClient()) + .mcps(proposalStream.collect(Collectors.toList()), auditStamp, entityService) .build(); Set proposalResult = entityService.ingestProposal(batch, async); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java index 31577429df72d..6c0474dc6cfb6 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/util/ReflectionCache.java @@ -134,11 +134,34 @@ public Method lookupMethod( return lookupMethod(builderPair.getFirst(), method, parameters); } + /** + * Convert class name to the pdl model names. Upper case first letter unless the 3rd character is + * upper case. Reverse of {link ReflectionCache.toUpperFirst} i.e. MLModel -> mlModel Dataset -> + * dataset DataProduct -> dataProduct + * + * @param s input string + * @return class name + */ public static String toLowerFirst(String s) { - return s.substring(0, 1).toLowerCase() + s.substring(1); + if (s.length() > 2 && s.substring(2, 3).equals(s.substring(2, 3).toUpperCase())) { + return s.substring(0, 2).toLowerCase() + s.substring(2); + } else { + return s.substring(0, 1).toLowerCase() + s.substring(1); + } } + /** + * Convert the pdl model names to desired class names. Upper case first letter unless the 3rd + * character is upper case. i.e. mlModel -> MLModel dataset -> Dataset dataProduct -> DataProduct + * + * @param s input string + * @return class name + */ public static String toUpperFirst(String s) { - return s.substring(0, 1).toUpperCase() + s.substring(1); + if (s.length() > 2 && s.substring(2, 3).equals(s.substring(2, 3).toUpperCase())) { + return s.substring(0, 2).toUpperCase() + s.substring(2); + } else { + return s.substring(0, 1).toUpperCase() + s.substring(1); + } } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java new file mode 100644 index 0000000000000..503330fdc8a2e --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -0,0 +1,507 @@ +package io.datahubproject.openapi.v2.controller; + +import static io.datahubproject.openapi.v2.utils.ControllerUtil.checkAuthorized; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizerChain; +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.ByteString; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.metadata.aspect.batch.UpsertItem; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.UpdateAspectResult; +import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; +import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchService; +import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.metadata.utils.SearchUtil; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import io.datahubproject.openapi.v2.models.GenericEntity; +import io.datahubproject.openapi.v2.models.GenericScrollResult; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PatchMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/v2/entity") +@Slf4j +public class EntityController { + private static final SearchFlags DEFAULT_SEARCH_FLAGS = + new SearchFlags().setFulltext(false).setSkipAggregates(true).setSkipHighlighting(true); + @Autowired private EntityRegistry entityRegistry; + @Autowired private SearchService searchService; + @Autowired private EntityService entityService; + @Autowired private AuthorizerChain authorizationChain; + @Autowired private boolean restApiAuthorizationEnabled; + @Autowired private ObjectMapper objectMapper; + + @Tag(name = "Generic Entities", description = "API for interacting with generic entities.") + @GetMapping(value = "/{entityName}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll entities") + public ResponseEntity> getEntities( + @PathVariable("entityName") String entityName, + @RequestParam(value = "aspectNames", defaultValue = "") Set aspectNames, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "query", defaultValue = "*") String query, + @RequestParam(value = "scrollId", required = false) String scrollId, + @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField, + @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING") + String sortOrder, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata) + throws URISyntaxException { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + // TODO: support additional and multiple sort params + SortCriterion sortCriterion = SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder)); + + ScrollResult result = + searchService.scrollAcrossEntities( + List.of(entitySpec.getName()), + query, + null, + sortCriterion, + scrollId, + null, + count, + DEFAULT_SEARCH_FLAGS); + + return ResponseEntity.ok( + GenericScrollResult.builder() + .results(toRecordTemplates(result.getEntities(), aspectNames, withSystemMetadata)) + .scrollId(result.getScrollId()) + .build()); + } + + @Tag(name = "Generic Entities") + @GetMapping(value = "/{entityName}/{entityUrn}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Get an entity") + public ResponseEntity getEntity( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @RequestParam(value = "aspectNames", defaultValue = "") Set aspectNames, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata) + throws URISyntaxException { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return ResponseEntity.of( + toRecordTemplates(List.of(UrnUtils.getUrn(entityUrn)), aspectNames, withSystemMetadata) + .stream() + .findFirst()); + } + + @Tag(name = "Generic Entities") + @RequestMapping( + value = "/{entityName}/{entityUrn}", + method = {RequestMethod.HEAD}) + @Operation(summary = "Entity exists") + public ResponseEntity headEntity( + @PathVariable("entityName") String entityName, @PathVariable("entityUrn") String entityUrn) { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return exists(UrnUtils.getUrn(entityUrn), null) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @Tag(name = "Generic Aspects", description = "API for generic aspects.") + @GetMapping( + value = "/{entityName}/{entityUrn}/{aspectName}", + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Get an entity's generic aspect.") + public ResponseEntity getAspect( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName) + throws URISyntaxException { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return ResponseEntity.of( + toRecordTemplates(List.of(UrnUtils.getUrn(entityUrn)), Set.of(aspectName), true).stream() + .findFirst() + .flatMap(e -> e.getAspects().values().stream().findFirst())); + } + + @Tag(name = "Generic Aspects") + @RequestMapping( + value = "/{entityName}/{entityUrn}/{aspectName}", + method = {RequestMethod.HEAD}) + @Operation(summary = "Whether an entity aspect exists.") + public ResponseEntity headAspect( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName) { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return exists(UrnUtils.getUrn(entityUrn), aspectName) + ? ResponseEntity.noContent().build() + : ResponseEntity.notFound().build(); + } + + @Tag(name = "Generic Entities") + @DeleteMapping(value = "/{entityName}/{entityUrn}") + @Operation(summary = "Delete an entity") + public void deleteEntity( + @PathVariable("entityName") String entityName, @PathVariable("entityUrn") String entityUrn) { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE.getType())); + } + + entityService.deleteAspect(entityUrn, entitySpec.getKeyAspectName(), Map.of(), true); + } + + @Tag(name = "Generic Aspects") + @DeleteMapping(value = "/{entityName}/{entityUrn}/{aspectName}") + @Operation(summary = "Delete an entity aspect.") + public void deleteAspect( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName) { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE.getType())); + } + + entityService.deleteAspect(entityUrn, aspectName, Map.of(), true); + } + + @Tag(name = "Generic Aspects") + @PostMapping( + value = "/{entityName}/{entityUrn}/{aspectName}", + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Create an entity aspect.") + public ResponseEntity createAspect( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestBody @Nonnull String jsonAspect) + throws URISyntaxException { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Authentication authentication = AuthenticationContext.getAuthentication(); + + if (restApiAuthorizationEnabled) { + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType())); + } + + AspectSpec aspectSpec = entitySpec.getAspectSpec(aspectName); + UpsertItem upsert = + toUpsertItem(UrnUtils.getUrn(entityUrn), aspectSpec, jsonAspect, authentication.getActor()); + + List results = + entityService.ingestAspects( + AspectsBatchImpl.builder().items(List.of(upsert)).build(), true, true); + + return ResponseEntity.of( + results.stream() + .findFirst() + .map( + result -> + GenericEntity.builder() + .urn(result.getUrn().toString()) + .build( + objectMapper, + Map.of( + aspectName, + Pair.of( + result.getNewValue(), + withSystemMetadata ? result.getNewSystemMetadata() : null))))); + } + + @Tag(name = "Generic Aspects") + @PatchMapping( + value = "/{entityName}/{entityUrn}/{aspectName}", + consumes = "application/json-patch+json", + produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Patch an entity aspect. (Experimental)") + public ResponseEntity patchAspect( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestBody @Nonnull GenericJsonPatch patch) + throws URISyntaxException, + NoSuchMethodException, + InvocationTargetException, + InstantiationException, + IllegalAccessException { + + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + Authentication authentication = AuthenticationContext.getAuthentication(); + + if (restApiAuthorizationEnabled) { + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpec, + entityUrn, + ImmutableList.of(PoliciesConfig.EDIT_ENTITY_PRIVILEGE.getType())); + } + + RecordTemplate currentValue = + entityService.getAspect(UrnUtils.getUrn(entityUrn), aspectName, 0); + + AspectSpec aspectSpec = entitySpec.getAspectSpec(aspectName); + GenericPatchTemplate genericPatchTemplate = + GenericPatchTemplate.builder() + .genericJsonPatch(patch) + .templateType(aspectSpec.getDataTemplateClass()) + .templateDefault( + aspectSpec.getDataTemplateClass().getDeclaredConstructor().newInstance()) + .build(); + UpsertItem upsert = + toUpsertItem( + UrnUtils.getUrn(entityUrn), + aspectSpec, + currentValue, + genericPatchTemplate, + authentication.getActor()); + + List results = + entityService.ingestAspects( + AspectsBatchImpl.builder().items(List.of(upsert)).build(), true, true); + + return ResponseEntity.of( + results.stream() + .findFirst() + .map( + result -> + GenericEntity.builder() + .urn(result.getUrn().toString()) + .build( + objectMapper, + Map.of( + aspectName, + Pair.of( + result.getNewValue(), + withSystemMetadata ? result.getNewSystemMetadata() : null))))); + } + + private List toRecordTemplates( + SearchEntityArray searchEntities, Set aspectNames, boolean withSystemMetadata) + throws URISyntaxException { + return toRecordTemplates( + searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), + aspectNames, + withSystemMetadata); + } + + private Boolean exists(Urn urn, @Nullable String aspect) { + return aspect == null ? entityService.exists(urn, true) : entityService.exists(urn, aspect); + } + + private List toRecordTemplates( + List urns, Set aspectNames, boolean withSystemMetadata) + throws URISyntaxException { + if (urns.isEmpty()) { + return List.of(); + } else { + Set urnsSet = new HashSet<>(urns); + + Map> aspects = + entityService.getLatestEnvelopedAspects( + urnsSet, resolveAspectNames(urnsSet, aspectNames)); + + return urns.stream() + .map( + u -> + GenericEntity.builder() + .urn(u.toString()) + .build( + objectMapper, + toAspectMap(u, aspects.getOrDefault(u, List.of()), withSystemMetadata))) + .collect(Collectors.toList()); + } + } + + private Set resolveAspectNames(Set urns, Set requestedNames) { + if (requestedNames.isEmpty()) { + return urns.stream() + .flatMap(u -> entityRegistry.getEntitySpec(u.getEntityType()).getAspectSpecs().stream()) + .map(AspectSpec::getName) + .collect(Collectors.toSet()); + } else { + // ensure key is always present + return Stream.concat( + requestedNames.stream(), + urns.stream() + .map(u -> entityRegistry.getEntitySpec(u.getEntityType()).getKeyAspectName())) + .collect(Collectors.toSet()); + } + } + + private Map> toAspectMap( + Urn urn, List aspects, boolean withSystemMetadata) { + return aspects.stream() + .map( + a -> + Map.entry( + a.getName(), + Pair.of( + toRecordTemplate(lookupAspectSpec(urn, a.getName()), a), + withSystemMetadata ? a.getSystemMetadata() : null))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private AspectSpec lookupAspectSpec(Urn urn, String aspectName) { + return entityRegistry.getEntitySpec(urn.getEntityType()).getAspectSpec(aspectName); + } + + private RecordTemplate toRecordTemplate(AspectSpec aspectSpec, EnvelopedAspect envelopedAspect) { + return RecordUtils.toRecordTemplate( + aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data()); + } + + private UpsertItem toUpsertItem( + Urn entityUrn, AspectSpec aspectSpec, String jsonAspect, Actor actor) + throws URISyntaxException { + return MCPUpsertBatchItem.builder() + .urn(entityUrn) + .aspectName(aspectSpec.getName()) + .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) + .aspect( + GenericRecordUtils.deserializeAspect( + ByteString.copyString(jsonAspect, StandardCharsets.UTF_8), + GenericRecordUtils.JSON, + aspectSpec)) + .build(entityService); + } + + private UpsertItem toUpsertItem( + @Nonnull Urn urn, + @Nonnull AspectSpec aspectSpec, + @Nullable RecordTemplate currentValue, + @Nonnull GenericPatchTemplate genericPatchTemplate, + @Nonnull Actor actor) + throws URISyntaxException { + return MCPUpsertBatchItem.fromPatch( + urn, + aspectSpec, + currentValue, + genericPatchTemplate, + AuditStampUtils.createAuditStamp(actor.toUrnStr()), + entityService); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java new file mode 100644 index 0000000000000..3550a86163f51 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/RelationshipController.java @@ -0,0 +1,228 @@ +package io.datahubproject.openapi.v2.controller; + +import static io.datahubproject.openapi.v2.utils.ControllerUtil.checkAuthorized; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizerChain; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.graph.RelatedEntities; +import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; +import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.utils.QueryUtils; +import com.linkedin.metadata.utils.SearchUtil; +import io.datahubproject.openapi.v2.models.GenericRelationship; +import io.datahubproject.openapi.v2.models.GenericScrollResult; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/v2/relationship") +@Slf4j +@Tag( + name = "Generic Relationships", + description = "APIs for ingesting and accessing entity relationships.") +public class RelationshipController { + + private static final String[] SORT_FIELDS = {"source.urn", "destination.urn", "relationshipType"}; + private static final String[] SORT_ORDERS = {"ASCENDING", "ASCENDING", "ASCENDING"}; + private static final List EDGE_SORT_CRITERION; + + static { + EDGE_SORT_CRITERION = + IntStream.range(0, SORT_FIELDS.length) + .mapToObj( + idx -> SearchUtil.sortBy(SORT_FIELDS[idx], SortOrder.valueOf(SORT_ORDERS[idx]))) + .collect(Collectors.toList()); + } + + @Autowired private EntityRegistry entityRegistry; + @Autowired private ElasticSearchGraphService graphService; + @Autowired private AuthorizerChain authorizationChain; + + @Autowired private boolean restApiAuthorizationEnabled; + + /** + * Returns relationship edges by type + * + * @param relationshipType the relationship type + * @param count number of results + * @param scrollId scrolling id + * @return list of relation edges + */ + @GetMapping(value = "/{relationshipType}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll relationships of the given type.") + public ResponseEntity> getRelationshipsByType( + @PathVariable("relationshipType") String relationshipType, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "scrollId", required = false) String scrollId) { + + RelatedEntitiesScrollResult result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + List.of(relationshipType), + new RelationshipFilter().setDirection(RelationshipDirection.UNDIRECTED), + EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + Set entitySpecs = + result.getEntities().stream() + .flatMap( + relatedEntity -> + Stream.of( + entityRegistry.getEntitySpec( + UrnUtils.getUrn(relatedEntity.getUrn()).getEntityType()), + entityRegistry.getEntitySpec( + UrnUtils.getUrn(relatedEntity.getSourceUrn()).getEntityType()))) + .collect(Collectors.toSet()); + + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpecs, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return ResponseEntity.ok( + GenericScrollResult.builder() + .results(toGenericRelationships(result.getEntities())) + .scrollId(result.getScrollId()) + .build()); + } + + /** + * Returns edges for a given urn + * + * @param relationshipTypes types of edges + * @param direction direction of the edges + * @param count number of results + * @param scrollId scroll id + * @return urn edges + */ + @GetMapping(value = "/{entityName}/{entityUrn}", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll relationships from a given entity.") + public ResponseEntity> getRelationshipsByEntity( + @PathVariable("entityName") String entityName, + @PathVariable("entityUrn") String entityUrn, + @RequestParam(value = "relationshipType[]", required = false, defaultValue = "*") + String[] relationshipTypes, + @RequestParam(value = "direction", defaultValue = "OUTGOING") String direction, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "scrollId", required = false) String scrollId) { + + final RelatedEntitiesScrollResult result; + + switch (RelationshipDirection.valueOf(direction.toUpperCase())) { + case INCOMING -> result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") + ? Arrays.stream(relationshipTypes).toList() + : List.of(), + new RelationshipFilter() + .setDirection(RelationshipDirection.UNDIRECTED) + .setOr(QueryUtils.newFilter("destination.urn", entityUrn).getOr()), + EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + case OUTGOING -> result = + graphService.scrollRelatedEntities( + null, + null, + null, + null, + relationshipTypes.length > 0 && !relationshipTypes[0].equals("*") + ? Arrays.stream(relationshipTypes).toList() + : List.of(), + new RelationshipFilter() + .setDirection(RelationshipDirection.UNDIRECTED) + .setOr(QueryUtils.newFilter("source.urn", entityUrn).getOr()), + EDGE_SORT_CRITERION, + scrollId, + count, + null, + null); + default -> throw new IllegalArgumentException("Direction must be INCOMING or OUTGOING"); + } + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + Set entitySpecs = + result.getEntities().stream() + .flatMap( + relatedEntity -> + Stream.of( + entityRegistry.getEntitySpec( + UrnUtils.getUrn(relatedEntity.getDestinationUrn()).getEntityType()), + entityRegistry.getEntitySpec( + UrnUtils.getUrn(relatedEntity.getSourceUrn()).getEntityType()))) + .collect(Collectors.toSet()); + + checkAuthorized( + authorizationChain, + authentication.getActor(), + entitySpecs, + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + return ResponseEntity.ok( + GenericScrollResult.builder() + .results(toGenericRelationships(result.getEntities())) + .scrollId(result.getScrollId()) + .build()); + } + + private List toGenericRelationships(List relatedEntities) { + return relatedEntities.stream() + .map( + result -> { + Urn source = UrnUtils.getUrn(result.getSourceUrn()); + Urn dest = UrnUtils.getUrn(result.getDestinationUrn()); + return GenericRelationship.builder() + .relationshipType(result.getRelationshipType()) + .source(GenericRelationship.GenericNode.fromUrn(source)) + .destination(GenericRelationship.GenericNode.fromUrn(dest)) + .build(); + }) + .collect(Collectors.toList()); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java new file mode 100644 index 0000000000000..ab12b68339011 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/TimeseriesController.java @@ -0,0 +1,115 @@ +package io.datahubproject.openapi.v2.controller; + +import static io.datahubproject.openapi.v2.utils.ControllerUtil.checkAuthorized; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationContext; +import com.datahub.authorization.AuthorizerChain; +import com.google.common.collect.ImmutableList; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.timeseries.GenericTimeseriesDocument; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.timeseries.TimeseriesScrollResult; +import com.linkedin.metadata.utils.SearchUtil; +import io.datahubproject.openapi.v2.models.GenericScrollResult; +import io.datahubproject.openapi.v2.models.GenericTimeseriesAspect; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.net.URISyntaxException; +import java.util.List; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/v2/timeseries") +@Slf4j +@Tag( + name = "Generic Timeseries Aspects", + description = "APIs for ingesting and accessing timeseries aspects") +public class TimeseriesController { + + @Autowired private EntityRegistry entityRegistry; + + @Autowired private TimeseriesAspectService timeseriesAspectService; + + @Autowired private AuthorizerChain authorizationChain; + + @Autowired private boolean restApiAuthorizationEnabled; + + @GetMapping(value = "/{entityName}/{aspectName}", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> getAspects( + @PathVariable("entityName") String entityName, + @PathVariable("aspectName") String aspectName, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "scrollId", required = false) String scrollId, + @RequestParam(value = "startTimeMillis", required = false) Long startTimeMillis, + @RequestParam(value = "endTimeMillis", required = false) Long endTimeMillis, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata) + throws URISyntaxException { + + if (restApiAuthorizationEnabled) { + Authentication authentication = AuthenticationContext.getAuthentication(); + checkAuthorized( + authorizationChain, + authentication.getActor(), + entityRegistry.getEntitySpec(entityName), + ImmutableList.of(PoliciesConfig.GET_ENTITY_PRIVILEGE.getType())); + } + + AspectSpec aspectSpec = entityRegistry.getEntitySpec(entityName).getAspectSpec(aspectName); + if (!aspectSpec.isTimeseries()) { + throw new IllegalArgumentException("Only timeseries aspects are supported."); + } + + List sortCriterion = + List.of( + SearchUtil.sortBy("timestampMillis", SortOrder.DESCENDING), + SearchUtil.sortBy("messageId", SortOrder.DESCENDING)); + + TimeseriesScrollResult result = + timeseriesAspectService.scrollAspects( + entityName, + aspectName, + null, + sortCriterion, + scrollId, + count, + startTimeMillis, + endTimeMillis); + + return ResponseEntity.ok( + GenericScrollResult.builder() + .scrollId(result.getScrollId()) + .results(toGenericTimeseriesAspect(result.getDocuments(), withSystemMetadata)) + .build()); + } + + private static List toGenericTimeseriesAspect( + List docs, boolean withSystemMetadata) { + return docs.stream() + .map( + doc -> + GenericTimeseriesAspect.builder() + .urn(doc.getUrn()) + .messageId(doc.getMessageId()) + .timestampMillis(doc.getTimestampMillis()) + .systemMetadata(withSystemMetadata ? doc.getSystemMetadata() : null) + .event(doc.getEvent()) + .build()) + .collect(Collectors.toList()); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java new file mode 100644 index 0000000000000..f1e965ca05464 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericEntity.java @@ -0,0 +1,57 @@ +package io.datahubproject.openapi.v2.models; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.mxe.SystemMetadata; +import com.linkedin.util.Pair; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class GenericEntity { + private String urn; + private Map aspects; + + public static class GenericEntityBuilder { + + public GenericEntity build( + ObjectMapper objectMapper, Map> aspects) { + Map jsonObjectMap = + aspects.entrySet().stream() + .map( + e -> { + try { + Map valueMap = + Map.of( + "value", + objectMapper.readTree( + RecordUtils.toJsonString(e.getValue().getFirst()) + .getBytes(StandardCharsets.UTF_8))); + + if (e.getValue().getSecond() != null) { + return Map.entry( + e.getKey(), + Map.of( + "systemMetadata", e.getValue().getSecond(), + "value", valueMap.get("value"))); + } else { + return Map.entry(e.getKey(), Map.of("value", valueMap.get("value"))); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return new GenericEntity(urn, jsonObjectMap); + } + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericRelationship.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericRelationship.java new file mode 100644 index 0000000000000..a4fb429c1eb18 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericRelationship.java @@ -0,0 +1,36 @@ +package io.datahubproject.openapi.v2.models; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.linkedin.common.urn.Urn; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class GenericRelationship { + @Nonnull private String relationshipType; + @Nonnull private GenericNode destination; + @Nonnull private GenericNode source; + @Nullable private NodeProperties properties; + + @Data + @Builder + public static class GenericNode { + @Nonnull private String entityType; + @Nonnull private String urn; + + public static GenericNode fromUrn(@Nonnull Urn urn) { + return GenericNode.builder().entityType(urn.getEntityType()).urn(urn.toString()).build(); + } + } + + @Data + @Builder + public static class NodeProperties { + private List source; + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java new file mode 100644 index 0000000000000..2befc83c00363 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericScrollResult.java @@ -0,0 +1,12 @@ +package io.datahubproject.openapi.v2.models; + +import java.util.List; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GenericScrollResult { + private String scrollId; + private List results; +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericTimeseriesAspect.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericTimeseriesAspect.java new file mode 100644 index 0000000000000..9d52ed28b2066 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/GenericTimeseriesAspect.java @@ -0,0 +1,18 @@ +package io.datahubproject.openapi.v2.models; + +import com.fasterxml.jackson.annotation.JsonInclude; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class GenericTimeseriesAspect { + private long timestampMillis; + @Nonnull private String urn; + @Nonnull private Object event; + @Nullable private String messageId; + @Nullable private Object systemMetadata; +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/PatchOperation.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/PatchOperation.java new file mode 100644 index 0000000000000..c5323dfe68369 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/models/PatchOperation.java @@ -0,0 +1,26 @@ +package io.datahubproject.openapi.v2.models; + +import com.fasterxml.jackson.databind.JsonNode; +import com.linkedin.metadata.aspect.patch.PatchOperationType; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class PatchOperation { + @Nonnull private String op; + @Nonnull private String path; + @Nullable private JsonNode value; + @Nullable private List arrayMapKey; + + public PatchOperationType getOp() { + return PatchOperationType.valueOf(op.toUpperCase()); + } +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/utils/ControllerUtil.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/utils/ControllerUtil.java new file mode 100644 index 0000000000000..70d588721d3b3 --- /dev/null +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/utils/ControllerUtil.java @@ -0,0 +1,67 @@ +package io.datahubproject.openapi.v2.utils; + +import com.datahub.authentication.Actor; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.datahub.plugins.auth.authorization.Authorizer; +import com.google.common.collect.ImmutableList; +import com.linkedin.metadata.models.EntitySpec; +import io.datahubproject.openapi.exception.UnauthorizedException; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class ControllerUtil { + private ControllerUtil() {} + + public static void checkAuthorized( + @Nonnull Authorizer authorizationChain, + @Nonnull Actor actor, + @Nonnull EntitySpec entitySpec, + @Nonnull List privileges) { + checkAuthorized(authorizationChain, actor, entitySpec, null, privileges); + } + + public static void checkAuthorized( + @Nonnull Authorizer authorizationChain, + @Nonnull Actor actor, + @Nonnull Set entitySpecs, + @Nonnull List privileges) { + DisjunctivePrivilegeGroup orGroup = + new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup(privileges))); + List> resourceSpecs = + entitySpecs.stream() + .map( + entitySpec -> + Optional.of(new com.datahub.authorization.EntitySpec(entitySpec.getName(), ""))) + .collect(Collectors.toList()); + if (!AuthUtil.isAuthorizedForResources( + authorizationChain, actor.toUrnStr(), resourceSpecs, orGroup)) { + throw new UnauthorizedException(actor.toUrnStr() + " is unauthorized to get entities."); + } + } + + public static void checkAuthorized( + @Nonnull Authorizer authorizationChain, + @Nonnull Actor actor, + @Nonnull EntitySpec entitySpec, + @Nullable String entityUrn, + @Nonnull List privileges) { + DisjunctivePrivilegeGroup orGroup = + new DisjunctivePrivilegeGroup(ImmutableList.of(new ConjunctivePrivilegeGroup(privileges))); + + List> resourceSpecs = + List.of( + Optional.of( + new com.datahub.authorization.EntitySpec( + entitySpec.getName(), entityUrn != null ? entityUrn : ""))); + if (!AuthUtil.isAuthorizedForResources( + authorizationChain, actor.toUrnStr(), resourceSpecs, orGroup)) { + throw new UnauthorizedException(actor.toUrnStr() + " is unauthorized to get entities."); + } + } +} diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index bca3e7161c8b8..ee45b8921143a 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -319,6 +319,7 @@ "default" : { }, "Searchable" : { "/*" : { + "fieldType" : "TEXT", "queryByDefault" : true } } @@ -993,6 +994,11 @@ "filterNameOverride" : "Glossary Term", "hasValuesFieldName" : "hasGlossaryTerms" } + }, { + "name" : "actor", + "type" : "Urn", + "doc" : "The user URN which will be credited for adding associating this term to the entity", + "optional" : true }, { "name" : "context", "type" : "string", @@ -2049,6 +2055,7 @@ "name" : "GlossaryNodeInfo", "namespace" : "com.linkedin.glossary", "doc" : "Properties associated with a GlossaryNode", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "definition", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 69184856e4f9e..505f44c52d583 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -61,6 +61,7 @@ "default" : { }, "Searchable" : { "/*" : { + "fieldType" : "TEXT", "queryByDefault" : true } } @@ -993,6 +994,11 @@ "filterNameOverride" : "Glossary Term", "hasValuesFieldName" : "hasGlossaryTerms" } + }, { + "name" : "actor", + "type" : "Urn", + "doc" : "The user URN which will be credited for adding associating this term to the entity", + "optional" : true }, { "name" : "context", "type" : "string", @@ -5084,6 +5090,7 @@ "name" : "GlossaryNodeInfo", "namespace" : "com.linkedin.glossary", "doc" : "Properties associated with a GlossaryNode", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "definition", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 09c0185f74f3a..e8c15d1b4ca04 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -61,6 +61,7 @@ "default" : { }, "Searchable" : { "/*" : { + "fieldType" : "TEXT", "queryByDefault" : true } } @@ -735,6 +736,11 @@ "filterNameOverride" : "Glossary Term", "hasValuesFieldName" : "hasGlossaryTerms" } + }, { + "name" : "actor", + "type" : "Urn", + "doc" : "The user URN which will be credited for adding associating this term to the entity", + "optional" : true }, { "name" : "context", "type" : "string", @@ -1783,6 +1789,7 @@ "name" : "GlossaryNodeInfo", "namespace" : "com.linkedin.glossary", "doc" : "Properties associated with a GlossaryNode", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "definition", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json index eae0eed2dd50b..67f70d40e010c 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.operations.operations.snapshot.json @@ -61,6 +61,7 @@ "default" : { }, "Searchable" : { "/*" : { + "fieldType" : "TEXT", "queryByDefault" : true } } @@ -735,6 +736,11 @@ "filterNameOverride" : "Glossary Term", "hasValuesFieldName" : "hasGlossaryTerms" } + }, { + "name" : "actor", + "type" : "Urn", + "doc" : "The user URN which will be credited for adding associating this term to the entity", + "optional" : true }, { "name" : "context", "type" : "string", @@ -1777,6 +1783,7 @@ "name" : "GlossaryNodeInfo", "namespace" : "com.linkedin.glossary", "doc" : "Properties associated with a GlossaryNode", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "definition", "type" : "string", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index cb253c458e6c4..4c8cd1f20d476 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -61,6 +61,7 @@ "default" : { }, "Searchable" : { "/*" : { + "fieldType" : "TEXT", "queryByDefault" : true } } @@ -993,6 +994,11 @@ "filterNameOverride" : "Glossary Term", "hasValuesFieldName" : "hasGlossaryTerms" } + }, { + "name" : "actor", + "type" : "Urn", + "doc" : "The user URN which will be credited for adding associating this term to the entity", + "optional" : true }, { "name" : "context", "type" : "string", @@ -5078,6 +5084,7 @@ "name" : "GlossaryNodeInfo", "namespace" : "com.linkedin.glossary", "doc" : "Properties associated with a GlossaryNode", + "include" : [ "com.linkedin.common.CustomProperties" ], "fields" : [ { "name" : "definition", "type" : "string", diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index 64ae3632c353a..2f470dca01f2a 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -1,6 +1,9 @@ package com.linkedin.entity.client; +import static com.linkedin.metadata.utils.GenericRecordUtils.entityResponseToAspectMap; + import com.datahub.authentication.Authentication; +import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; @@ -11,7 +14,6 @@ import com.linkedin.entity.EntityResponse; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.browse.BrowseResultV2; import com.linkedin.metadata.graph.LineageDirection; @@ -40,7 +42,7 @@ import javax.annotation.Nullable; // Consider renaming this to datahub client. -public interface EntityClient extends AspectRetriever { +public interface EntityClient { @Nullable public EntityResponse getV2( @@ -623,14 +625,26 @@ public void producePlatformEvent( @Nonnull Authentication authentication) throws Exception; - public void rollbackIngestion(@Nonnull String runId, @Nonnull Authentication authentication) + public void rollbackIngestion( + @Nonnull String runId, @Nonnull Authorizer authorizer, @Nonnull Authentication authentication) throws Exception; - default Aspect getLatestAspectObject(@Nonnull Urn urn, @Nonnull String aspectName) + @Nullable + default Aspect getLatestAspectObject( + @Nonnull Urn urn, @Nonnull String aspectName, @Nonnull Authentication authentication) + throws RemoteInvocationException, URISyntaxException { + return getLatestAspects(Set.of(urn), Set.of(aspectName), authentication) + .getOrDefault(urn, Map.of()) + .get(aspectName); + } + + @Nonnull + default Map> getLatestAspects( + @Nonnull Set urns, + @Nonnull Set aspectNames, + @Nonnull Authentication authentication) throws RemoteInvocationException, URISyntaxException { - return getV2(urn.getEntityType(), urn, Set.of(aspectName), null) - .getAspects() - .get(aspectName) - .getValue(); + String entityName = urns.stream().findFirst().map(Urn::getEntityType).get(); + return entityResponseToAspectMap(batchGetV2(entityName, urns, aspectNames, authentication)); } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index d68c472ea9170..3108345bd3937 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -1,6 +1,7 @@ package com.linkedin.entity.client; import com.datahub.authentication.Authentication; +import com.datahub.plugins.auth.authorization.Authorizer; import com.datahub.util.RecordUtils; import com.google.common.collect.ImmutableList; import com.linkedin.common.VersionedUrn; @@ -539,7 +540,9 @@ public SearchResult search( if (searchFlags != null) { requestBuilder.searchFlagsParam(searchFlags); - requestBuilder.fulltextParam(searchFlags.isFulltext()); + if (searchFlags.isFulltext() != null) { + requestBuilder.fulltextParam(searchFlags.isFulltext()); + } } return sendClientRequest(requestBuilder, authentication).getEntity(); @@ -1057,7 +1060,10 @@ public void producePlatformEvent( } @Override - public void rollbackIngestion(@Nonnull String runId, @Nonnull final Authentication authentication) + public void rollbackIngestion( + @Nonnull String runId, + @Nonnull Authorizer authorizer, + @Nonnull final Authentication authentication) throws Exception { final RunsDoRollbackRequestBuilder requestBuilder = RUNS_REQUEST_BUILDERS.actionRollback().runIdParam(runId).dryRunParam(false); diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemEntityClient.java index dfad20b5f52b2..243e8a40bf4b7 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemEntityClient.java @@ -4,24 +4,60 @@ import com.linkedin.common.urn.Urn; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; -import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.ScrollResult; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.PlatformEvent; import com.linkedin.r2.RemoteInvocationException; import java.net.URISyntaxException; +import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; import javax.annotation.Nullable; -/** Adds entity/aspect cache and assumes system authentication */ -public interface SystemEntityClient extends EntityClient, AspectRetriever { +/** Adds entity/aspect cache and assumes **system** authentication */ +public interface SystemEntityClient extends EntityClient { EntityClientCache getEntityClientCache(); Authentication getSystemAuthentication(); + /** + * Searches for entities matching to a given query and filters across multiple entity types + * + * @param entities entity types to search (if empty, searches all entities) + * @param input search query + * @param filter search filters + * @param scrollId opaque scroll ID indicating offset + * @param keepAlive string representation of time to keep point in time alive, ex: 5m + * @param count max number of search results requested + * @return Snapshot key + * @throws RemoteInvocationException + */ + @Nonnull + default ScrollResult scrollAcrossEntities( + @Nonnull List entities, + @Nonnull String input, + @Nullable Filter filter, + @Nullable String scrollId, + @Nullable String keepAlive, + int count, + @Nullable SearchFlags searchFlags) + throws RemoteInvocationException { + return scrollAcrossEntities( + entities, + input, + filter, + scrollId, + keepAlive, + count, + searchFlags, + getSystemAuthentication()); + } + /** * Builds the cache * @@ -101,11 +137,16 @@ default void setWritable(boolean canWrite) throws RemoteInvocationException { setWritable(canWrite, getSystemAuthentication()); } + @Nullable default Aspect getLatestAspectObject(@Nonnull Urn urn, @Nonnull String aspectName) throws RemoteInvocationException, URISyntaxException { - return getV2(urn.getEntityType(), urn, Set.of(aspectName), getSystemAuthentication()) - .getAspects() - .get(aspectName) - .getValue(); + return getLatestAspectObject(urn, aspectName, getSystemAuthentication()); + } + + @Nonnull + default Map> getLatestAspects( + @Nonnull Set urns, @Nonnull Set aspectNames) + throws RemoteInvocationException, URISyntaxException { + return getLatestAspects(urns, aspectNames, getSystemAuthentication()); } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java index a2f5596af9f4e..0f179c4da7b74 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/SystemRestliEntityClient.java @@ -17,7 +17,7 @@ public SystemRestliEntityClient( @Nonnull final Client restliClient, @Nonnull final BackoffPolicy backoffPolicy, int retryCount, - Authentication systemAuthentication, + @Nonnull Authentication systemAuthentication, EntityClientCacheConfig cacheConfig) { super(restliClient, backoffPolicy, retryCount); this.systemAuthentication = systemAuthentication; diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle index ec5b645ee233c..8d21bdd489505 100644 --- a/metadata-service/restli-servlet-impl/build.gradle +++ b/metadata-service/restli-servlet-impl/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'java' + id 'java-library' id 'pegasus' } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index c5b019e85e0c9..ffa3abe6806f9 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -252,14 +252,14 @@ public Task ingestProposal( if (asyncBool) { // if async we'll expand the getAdditionalChanges later, no need to do this early batch = AspectsBatchImpl.builder() - .mcps(List.of(metadataChangeProposal), auditStamp, _entityService.getEntityRegistry(), _entityService.getSystemEntityClient()) + .mcps(List.of(metadataChangeProposal), auditStamp, _entityService) .build(); } else { Stream proposalStream = Stream.concat(Stream.of(metadataChangeProposal), AspectUtils.getAdditionalChanges(metadataChangeProposal, _entityService).stream()); batch = AspectsBatchImpl.builder() - .mcps(proposalStream.collect(Collectors.toList()), auditStamp, _entityService.getEntityRegistry(), _entityService.getSystemEntityClient()) + .mcps(proposalStream.collect(Collectors.toList()), auditStamp, _entityService) .build(); } diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java index 294ded8a1e255..869cfc7afdee8 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/BatchIngestionRunResource.java @@ -1,40 +1,25 @@ package com.linkedin.metadata.resources.entity; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.resources.restli.RestliUtils.*; +import static com.linkedin.metadata.service.RollbackService.ROLLBACK_FAILED_STATUS; import com.codahale.metrics.MetricRegistry; import com.datahub.authentication.Authentication; import com.datahub.authentication.AuthenticationContext; -import com.datahub.authorization.EntitySpec; +import com.datahub.authentication.AuthenticationException; import com.datahub.plugins.auth.authorization.Authorizer; -import com.google.common.collect.ImmutableList; -import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.execution.ExecutionRequestResult; -import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.entity.RollbackRunResult; -import com.linkedin.metadata.key.ExecutionRequestKey; import com.linkedin.metadata.restli.RestliUtil; import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.run.AspectRowSummaryArray; import com.linkedin.metadata.run.IngestionRunSummary; import com.linkedin.metadata.run.IngestionRunSummaryArray; import com.linkedin.metadata.run.RollbackResponse; -import com.linkedin.metadata.run.UnsafeEntityInfo; -import com.linkedin.metadata.run.UnsafeEntityInfoArray; -import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.metadata.service.RollbackService; import com.linkedin.metadata.systemmetadata.SystemMetadataService; -import com.linkedin.metadata.timeseries.TimeseriesAspectService; -import com.linkedin.metadata.utils.EntityKeyUtils; -import com.linkedin.metadata.utils.GenericRecordUtils; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.parseq.Task; import com.linkedin.restli.common.HttpStatus; import com.linkedin.restli.server.RestLiServiceException; @@ -43,13 +28,8 @@ import com.linkedin.restli.server.annotations.Optional; import com.linkedin.restli.server.annotations.RestLiCollection; import com.linkedin.restli.server.resources.CollectionResourceTaskTemplate; -import com.linkedin.timeseries.DeleteAspectValuesResult; import io.opentelemetry.extension.annotations.WithSpan; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; @@ -64,14 +44,8 @@ public class BatchIngestionRunResource private static final Integer DEFAULT_OFFSET = 0; private static final Integer DEFAULT_PAGE_SIZE = 100; - private static final Integer DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE = 1000000; private static final boolean DEFAULT_INCLUDE_SOFT_DELETED = false; private static final boolean DEFAULT_HARD_DELETE = false; - private static final Integer ELASTIC_MAX_PAGE_SIZE = 10000; - private static final Integer ELASTIC_BATCH_DELETE_SLEEP_SEC = 5; - private static final String ROLLING_BACK_STATUS = "ROLLING_BACK"; - private static final String ROLLED_BACK_STATUS = "ROLLED_BACK"; - private static final String ROLLBACK_FAILED_STATUS = "ROLLBACK_FAILED"; @Inject @Named("systemMetadataService") @@ -79,15 +53,15 @@ public class BatchIngestionRunResource @Inject @Named("entityService") - private EntityService _entityService; + private EntityService _entityService; @Inject - @Named("timeseriesAspectService") - private TimeseriesAspectService _timeseriesAspectService; + @Named("rollbackService") + private RollbackService rollbackService; - @Inject - @Named("authorizerChain") - private Authorizer _authorizer; + @Inject + @Named("authorizerChain") + private Authorizer _authorizer; /** Rolls back an ingestion run */ @Action(name = "rollback") @@ -111,274 +85,23 @@ public Task rollback( try { return RestliUtil.toTask( () -> { - if (runId.equals(DEFAULT_RUN_ID)) { - throw new IllegalArgumentException( - String.format( - "%s is a default run-id provided for non labeled ingestion runs. You cannot delete using this reserved run-id", - runId)); - } - if (!dryRun) { - updateExecutionRequestStatus(runId, ROLLING_BACK_STATUS); - } - - RollbackResponse response = new RollbackResponse(); - List aspectRowsToDelete; - aspectRowsToDelete = - _systemMetadataService.findByRunId(runId, doHardDelete, 0, ESUtils.MAX_RESULT_SIZE); - Set urns = - aspectRowsToDelete.stream() - .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) - .keySet(); - List> resourceSpecs = - urns.stream() - .map(UrnUtils::getUrn) - .map( - urn -> - java.util.Optional.of( - new EntitySpec(urn.getEntityType(), urn.toString()))) - .collect(Collectors.toList()); - Authentication auth = AuthenticationContext.getAuthentication(); - if (Boolean.parseBoolean(System.getenv(REST_API_AUTHORIZATION_ENABLED_ENV)) - && !isAuthorized( - auth, - _authorizer, - ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE), - resourceSpecs)) { - throw new RestLiServiceException( - HttpStatus.S_401_UNAUTHORIZED, "User is unauthorized to delete entities."); - } - log.info("found {} rows to delete...", stringifyRowCount(aspectRowsToDelete.size())); - if (dryRun) { - - final Map> aspectsSplitByIsKeyAspects = - aspectRowsToDelete.stream() - .collect(Collectors.partitioningBy(AspectRowSummary::isKeyAspect)); - - final List keyAspects = aspectsSplitByIsKeyAspects.get(true); - - long entitiesDeleted = keyAspects.size(); - long aspectsReverted = aspectRowsToDelete.size(); - final long affectedEntities = - aspectRowsToDelete.stream() - .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) - .keySet() - .size(); - - final AspectRowSummaryArray rowSummaries = - new AspectRowSummaryArray( - aspectRowsToDelete.subList(0, Math.min(100, aspectRowsToDelete.size()))); - - // If we are soft deleting, remove key aspects from count of aspects being deleted - if (!doHardDelete) { - aspectsReverted -= keyAspects.size(); - rowSummaries.removeIf(AspectRowSummary::isKeyAspect); + Authentication auth = AuthenticationContext.getAuthentication(); + try { + return rollbackService.rollbackIngestion(runId, dryRun, doHardDelete, _authorizer, auth); + } catch (AuthenticationException authException) { + throw new RestLiServiceException( + HttpStatus.S_401_UNAUTHORIZED, authException.getMessage()); } - // Compute the aspects that exist referencing the key aspects we are deleting - final List affectedAspectsList = - keyAspects.stream() - .map( - (AspectRowSummary urn) -> - _systemMetadataService.findByUrn( - urn.getUrn(), false, 0, ESUtils.MAX_RESULT_SIZE)) - .flatMap(List::stream) - .filter( - row -> - !row.getRunId().equals(runId) - && !row.isKeyAspect() - && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) - .collect(Collectors.toList()); - - long affectedAspects = affectedAspectsList.size(); - long unsafeEntitiesCount = - affectedAspectsList.stream() - .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) - .keySet() - .size(); - - final List unsafeEntityInfos = - affectedAspectsList.stream() - .map(AspectRowSummary::getUrn) - .distinct() - .map( - urn -> { - UnsafeEntityInfo unsafeEntityInfo = new UnsafeEntityInfo(); - unsafeEntityInfo.setUrn(urn); - return unsafeEntityInfo; - }) - // Return at most 1 million rows - .limit(DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE) - .collect(Collectors.toList()); - - return response - .setAspectsAffected(affectedAspects) - .setAspectsReverted(aspectsReverted) - .setEntitiesAffected(affectedEntities) - .setEntitiesDeleted(entitiesDeleted) - .setUnsafeEntitiesCount(unsafeEntitiesCount) - .setUnsafeEntities(new UnsafeEntityInfoArray(unsafeEntityInfos)) - .setAspectRowSummaries(rowSummaries); - } - - RollbackRunResult rollbackRunResult = - _entityService.rollbackRun(aspectRowsToDelete, runId, doHardDelete); - final List deletedRows = rollbackRunResult.getRowsRolledBack(); - int rowsDeletedFromEntityDeletion = - rollbackRunResult.getRowsDeletedFromEntityDeletion(); - - // since elastic limits how many rows we can access at once, we need to iteratively - // delete - while (aspectRowsToDelete.size() >= ELASTIC_MAX_PAGE_SIZE) { - sleep(ELASTIC_BATCH_DELETE_SLEEP_SEC); - aspectRowsToDelete = - _systemMetadataService.findByRunId( - runId, doHardDelete, 0, ESUtils.MAX_RESULT_SIZE); - log.info( - "{} remaining rows to delete...", stringifyRowCount(aspectRowsToDelete.size())); - log.info("deleting..."); - rollbackRunResult = - _entityService.rollbackRun(aspectRowsToDelete, runId, doHardDelete); - deletedRows.addAll(rollbackRunResult.getRowsRolledBack()); - rowsDeletedFromEntityDeletion += rollbackRunResult.getRowsDeletedFromEntityDeletion(); - } - - // Rollback timeseries aspects - DeleteAspectValuesResult timeseriesRollbackResult = - _timeseriesAspectService.rollbackTimeseriesAspects(runId); - rowsDeletedFromEntityDeletion += timeseriesRollbackResult.getNumDocsDeleted(); - - log.info("finished deleting {} rows", deletedRows.size()); - int aspectsReverted = deletedRows.size() + rowsDeletedFromEntityDeletion; - - final Map> aspectsSplitByIsKeyAspects = - aspectRowsToDelete.stream() - .collect(Collectors.partitioningBy(AspectRowSummary::isKeyAspect)); - - final List keyAspects = aspectsSplitByIsKeyAspects.get(true); - - final long entitiesDeleted = keyAspects.size(); - final long affectedEntities = - deletedRows.stream() - .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) - .keySet() - .size(); - - final AspectRowSummaryArray rowSummaries = - new AspectRowSummaryArray( - aspectRowsToDelete.subList(0, Math.min(100, aspectRowsToDelete.size()))); - - log.info("computing aspects affected by this rollback..."); - // Compute the aspects that exist referencing the key aspects we are deleting - final List affectedAspectsList = - keyAspects.stream() - .map( - (AspectRowSummary urn) -> - _systemMetadataService.findByUrn( - urn.getUrn(), false, 0, ESUtils.MAX_RESULT_SIZE)) - .flatMap(List::stream) - .filter( - row -> - !row.getRunId().equals(runId) - && !row.isKeyAspect() - && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) - .collect(Collectors.toList()); - - long affectedAspects = affectedAspectsList.size(); - long unsafeEntitiesCount = - affectedAspectsList.stream() - .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) - .keySet() - .size(); - - final List unsafeEntityInfos = - affectedAspectsList.stream() - .map(AspectRowSummary::getUrn) - .distinct() - .map( - urn -> { - UnsafeEntityInfo unsafeEntityInfo = new UnsafeEntityInfo(); - unsafeEntityInfo.setUrn(urn); - return unsafeEntityInfo; - }) - // Return at most 1 million rows - .limit(DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE) - .collect(Collectors.toList()); - - log.info("calculation done."); - - updateExecutionRequestStatus(runId, ROLLED_BACK_STATUS); - - return response - .setAspectsAffected(affectedAspects) - .setAspectsReverted(aspectsReverted) - .setEntitiesAffected(affectedEntities) - .setEntitiesDeleted(entitiesDeleted) - .setUnsafeEntitiesCount(unsafeEntitiesCount) - .setUnsafeEntities(new UnsafeEntityInfoArray(unsafeEntityInfos)) - .setAspectRowSummaries(rowSummaries); }, MetricRegistry.name(this.getClass(), "rollback")); } catch (Exception e) { - updateExecutionRequestStatus(runId, ROLLBACK_FAILED_STATUS); + rollbackService.updateExecutionRequestStatus(runId, ROLLBACK_FAILED_STATUS); throw new RuntimeException( String.format("There was an issue rolling back ingestion run with runId %s", runId), e); } } - private String stringifyRowCount(int size) { - if (size < ELASTIC_MAX_PAGE_SIZE) { - return String.valueOf(size); - } else { - return "at least " + size; - } - } - - private void sleep(Integer seconds) { - try { - TimeUnit.SECONDS.sleep(seconds); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - private void updateExecutionRequestStatus(String runId, String status) { - try { - final Urn executionRequestUrn = - EntityKeyUtils.convertEntityKeyToUrn( - new ExecutionRequestKey().setId(runId), Constants.EXECUTION_REQUEST_ENTITY_NAME); - EnvelopedAspect aspect = - _entityService.getLatestEnvelopedAspect( - executionRequestUrn.getEntityType(), - executionRequestUrn, - Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME); - if (aspect == null) { - log.warn("Aspect for execution request with runId {} not found", runId); - } else { - final MetadataChangeProposal proposal = new MetadataChangeProposal(); - ExecutionRequestResult requestResult = new ExecutionRequestResult(aspect.getValue().data()); - requestResult.setStatus(status); - proposal.setEntityUrn(executionRequestUrn); - proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); - proposal.setAspectName(Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME); - proposal.setAspect(GenericRecordUtils.serializeAspect(requestResult)); - proposal.setChangeType(ChangeType.UPSERT); - - _entityService.ingestProposal( - proposal, - new AuditStamp() - .setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)) - .setTime(System.currentTimeMillis()), - false); - } - } catch (Exception e) { - log.error( - String.format( - "Not able to update execution result aspect with runId %s and new status %s.", - runId, status), - e); - } - } - /** Retrieves the value for an entity that is made up of latest versions of specified aspects. */ @Action(name = "list") @Nonnull diff --git a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java index e3534875c6cd2..d6130e05b77bd 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java +++ b/metadata-service/restli-servlet-impl/src/test/java/com/linkedin/metadata/resources/entity/AspectResourceTest.java @@ -87,7 +87,7 @@ public void testAsyncDefaultAspects() throws URISyntaxException { .aspect(mcp.getAspect()) .auditStamp(new AuditStamp()) .metadataChangeProposal(mcp) - .build(_entityRegistry, _entityService.getSystemEntityClient()); + .build(_entityService); when(_aspectDao.runInTransactionWithRetry(any(), any(), anyInt())) .thenReturn( List.of( diff --git a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java index 2a12ecf6866bb..5187cba0b9151 100644 --- a/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java +++ b/metadata-service/restli-servlet-impl/src/test/java/mock/MockTimeseriesAspectService.java @@ -7,6 +7,7 @@ import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.timeseries.BatchWriteOperationsOptions; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.timeseries.TimeseriesScrollResult; import com.linkedin.timeseries.AggregationSpec; import com.linkedin.timeseries.DeleteAspectValuesResult; import com.linkedin.timeseries.GenericTable; @@ -118,4 +119,18 @@ public void upsertDocument( public List getIndexSizes() { return List.of(); } + + @Nonnull + @Override + public TimeseriesScrollResult scrollAspects( + @Nonnull String entityName, + @Nonnull String aspectName, + @Nullable Filter filter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis) { + return TimeseriesScrollResult.builder().build(); + } } diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle index c683b0c75f40a..78d651c05e4d9 100644 --- a/metadata-service/services/build.gradle +++ b/metadata-service/services/build.gradle @@ -1,6 +1,6 @@ plugins { id 'org.hidetake.swagger.generator' - id 'java' + id 'java-library' } configurations { @@ -14,7 +14,9 @@ dependencies { implementation project(':metadata-events:mxe-avro') implementation project(':metadata-events:mxe-registration') implementation project(':metadata-events:mxe-utils-avro') - implementation project(':metadata-models') + api project(path: ':metadata-models', configuration: 'dataTemplate') + api project(':metadata-models') + implementation project(':metadata-service:restli-client') implementation project(':metadata-service:configuration') diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java index 518b5f28a5b99..ea86d2c0c9842 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/datahubusage/DataHubUsageEventType.java @@ -43,6 +43,7 @@ public enum DataHubUsageEventType { CREATE_RESET_CREDENTIALS_LINK_EVENT("CreateResetCredentialsLinkEvent"), DELETE_ENTITY_EVENT("DeleteEntityEvent"), SELECT_USER_ROLE_EVENT("SelectUserRoleEvent"), + SELECT_GROUP_ROLE_EVENT("SelectGroupRoleEvent"), BATCH_SELECT_USER_ROLE_EVENT("BatchSelectUserRoleEvent"), CREATE_POLICY_EVENT("CreatePolicyEvent"), UPDATE_POLICY_EVENT("UpdatePolicyEvent"), diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java index c4216962c134c..2c1596474fb21 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/AspectUtils.java @@ -88,7 +88,7 @@ public static List getAdditionalChanges( public static List getAdditionalChanges( @Nonnull MetadataChangeProposal metadataChangeProposal, - @Nonnull EntityService entityService) { + @Nonnull EntityService entityService) { return getAdditionalChanges(metadataChangeProposal, entityService, false); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java index 71573aa2b10e0..94ab69e895920 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -9,11 +9,11 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.VersionedAspect; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.UpsertItem; +import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesResult; import com.linkedin.metadata.models.AspectSpec; @@ -35,7 +35,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -public interface EntityService { +public interface EntityService extends AspectRetriever { /** * Just whether the entity/aspect exists @@ -287,6 +287,8 @@ Pair>> generateDefaultAspectsOnFirstW Set getEntityAspectNames(final String entityName); + @Override + @Nonnull EntityRegistry getEntityRegistry(); RollbackResult deleteAspect( @@ -349,15 +351,5 @@ default boolean exists(@Nonnull Urn urn, boolean includeSoftDelete) { BrowsePathsV2 buildDefaultBrowsePathV2(final @Nonnull Urn urn, boolean useContainerPaths) throws URISyntaxException; - /** - * Allow internal use of the system entity client. Solves recursive dependencies between the - * EntityService and the SystemJavaEntityClient - * - * @param systemEntityClient system entity client - */ - void setSystemEntityClient(SystemEntityClient systemEntityClient); - - SystemEntityClient getSystemEntityClient(); - RecordTemplate getLatestAspect(@Nonnull final Urn urn, @Nonnull final String aspectName); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java index b3e713a906d01..625353eeb6820 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -5,6 +5,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; +import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.search.utils.QueryUtils; import java.net.URISyntaxException; import java.util.ArrayList; @@ -322,4 +323,18 @@ void removeEdgesFromNode( default boolean supportsMultiHop() { return false; } + + @Nonnull + RelatedEntitiesScrollResult scrollRelatedEntities( + @Nullable List sourceTypes, + @Nonnull Filter sourceEntityFilter, + @Nullable List destinationTypes, + @Nonnull Filter destinationEntityFilter, + @Nonnull List relationshipTypes, + @Nonnull RelationshipFilter relationshipFilter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntities.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntities.java new file mode 100644 index 0000000000000..0c6f8a0d65d5c --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntities.java @@ -0,0 +1,31 @@ +package com.linkedin.metadata.graph; + +import com.linkedin.metadata.query.filter.RelationshipDirection; +import javax.annotation.Nonnull; +import lombok.Getter; + +/** Preserves directionality as well as the generic `related` urn concept */ +@Getter +public class RelatedEntities extends RelatedEntity { + /** source Urn * */ + @Nonnull String sourceUrn; + + /** Destination Urn associated with the related entity. */ + @Nonnull String destinationUrn; + + public RelatedEntities( + @Nonnull String relationshipType, + @Nonnull String sourceUrn, + @Nonnull String destinationUrn, + @Nonnull RelationshipDirection relationshipDirection) { + super( + relationshipType, + relationshipDirection == RelationshipDirection.OUTGOING ? destinationUrn : sourceUrn); + this.sourceUrn = sourceUrn; + this.destinationUrn = destinationUrn; + } + + public RelatedEntity asRelatedEntity() { + return new RelatedEntity(relationshipType, urn); + } +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesScrollResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesScrollResult.java new file mode 100644 index 0000000000000..b0b5394ca5808 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/RelatedEntitiesScrollResult.java @@ -0,0 +1,16 @@ +package com.linkedin.metadata.graph; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@AllArgsConstructor +@Data +@Builder +public class RelatedEntitiesScrollResult { + int numResults; + int pageSize; + String scrollId; + List entities; +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java index 546c2856c28ac..0a29ebfe46415 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/recommendation/candidatesource/EntityRecommendationSource.java @@ -8,6 +8,7 @@ import com.linkedin.metadata.recommendation.RecommendationParams; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -29,7 +30,7 @@ default Stream buildContent( entityUrns.stream() .map(UrnUtils::getUrn) .filter(urn -> getSupportedEntityTypes().contains(urn.getEntityType())) - .toList(); + .collect(Collectors.toList()); Set existingNonRemoved = entityService.exists(entities, false); return entities.stream().filter(existingNonRemoved::contains).map(this::buildContent); diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java index 189ae09e1b938..2fec88ad221fd 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/search/EntitySearchService.java @@ -161,7 +161,7 @@ AutoCompleteResult autoComplete( * @param field the field name for aggregate * @param requestParams filters to apply before aggregating * @param limit the number of aggregations to return - * @return + * @return a map of the value to the count of documents having the value */ @Nonnull Map aggregateByValue( diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java new file mode 100644 index 0000000000000..59d40b29e7383 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/FormService.java @@ -0,0 +1,1107 @@ +package com.linkedin.metadata.service; + +import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.FORMS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.FORM_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTIES_ASPECT_NAME; +import static com.linkedin.metadata.entity.AspectUtils.buildMetadataChangeProposal; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.FieldFormPromptAssociation; +import com.linkedin.common.FieldFormPromptAssociationArray; +import com.linkedin.common.FormAssociation; +import com.linkedin.common.FormAssociationArray; +import com.linkedin.common.FormPromptAssociation; +import com.linkedin.common.FormPromptAssociationArray; +import com.linkedin.common.FormPromptFieldAssociations; +import com.linkedin.common.FormVerificationAssociation; +import com.linkedin.common.FormVerificationAssociationArray; +import com.linkedin.common.Forms; +import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.form.FormActorAssignment; +import com.linkedin.form.FormInfo; +import com.linkedin.form.FormPrompt; +import com.linkedin.form.FormType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.authorization.OwnershipUtils; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.service.util.SearchBasedFormAssignmentRunner; +import com.linkedin.metadata.utils.FormUtils; +import com.linkedin.metadata.utils.SchemaFieldUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.structured.PrimitivePropertyValueArray; +import com.linkedin.structured.StructuredProperties; +import com.linkedin.structured.StructuredPropertyValueAssignment; +import com.linkedin.structured.StructuredPropertyValueAssignmentArray; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +/** + * This class is used to execute CRUD operations around forms and submitting responses to forms and + * their prompts. + * + *

Note that no Authorization is performed within the service. The expectation is that the caller + * has already verified the permissions of the active Actor. + */ +@Slf4j +public class FormService extends BaseService { + private static final int BATCH_FORM_ENTITY_COUNT = 500; + + public FormService( + @Nonnull final EntityClient entityClient, + @Nonnull final Authentication systemAuthentication) { + super(entityClient, systemAuthentication); + } + + /** Batch associated a form to a given set of entities by urn. */ + public void batchAssignFormToEntities( + @Nonnull final List entityUrns, @Nonnull final Urn formUrn) throws Exception { + batchAssignFormToEntities(entityUrns, formUrn, this.systemAuthentication); + } + + /** Batch associated a form to a given set of entities by urn. */ + public void batchAssignFormToEntities( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) + throws Exception { + verifyEntityExists(formUrn, authentication); + verifyEntitiesExist(entityUrns, authentication); + final List changes = + buildAssignFormChanges(entityUrns, formUrn, authentication); + ingestChangeProposals(changes, authentication); + } + + /** Batch remove a form from a given entity by urn. */ + public void batchUnassignFormForEntities( + @Nonnull final List entityUrns, @Nonnull final Urn formUrn) throws Exception { + batchUnassignFormForEntities(entityUrns, formUrn, this.systemAuthentication); + } + + /** Batch remove a form from a given entity by urn. */ + public void batchUnassignFormForEntities( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) + throws Exception { + verifyEntityExists(formUrn, authentication); + verifyEntitiesExist(entityUrns, authentication); + final List changes = + buildUnassignFormChanges(entityUrns, formUrn, authentication); + ingestChangeProposals(changes, authentication); + } + + /** Mark a specific form prompt as incomplete */ + public void batchSetFormPromptIncomplete( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId) + throws Exception { + batchSetFormPromptIncomplete(entityUrns, formUrn, formPromptId, this.systemAuthentication); + } + + /** Mark a specific form prompt as incomplete */ + public void batchSetFormPromptIncomplete( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) + throws Exception { + verifyEntityExists(formUrn, authentication); + verifyEntitiesExist(entityUrns, authentication); + final FormInfo formInfo = getFormInfo(formUrn, authentication); + final List changes = + buildUnsetFormPromptChanges(entityUrns, formUrn, formPromptId, formInfo, authentication); + ingestChangeProposals(changes, authentication); + } + + /** Create a dynamic form assignment for a particular form. */ + public void createDynamicFormAssignment( + @Nonnull final DynamicFormAssignment dynamicFormAssignment, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) + throws RemoteInvocationException { + if (!entityClient.exists(formUrn, authentication)) { + throw new RuntimeException( + String.format("Form %s does not exist. Skipping dynamic form assignment", formUrn)); + } + + try { + this.entityClient.ingestProposal( + AspectUtils.buildMetadataChangeProposal( + formUrn, Constants.DYNAMIC_FORM_ASSIGNMENT_ASPECT_NAME, dynamicFormAssignment), + authentication, + false); + } catch (Exception e) { + throw new RuntimeException("Failed to create form", e); + } + } + + /** Assigns the form to an entity for completion. */ + public void upsertFormAssignmentRunner( + @Nonnull final Urn formUrn, @Nonnull final DynamicFormAssignment formFilters) { + try { + SearchBasedFormAssignmentRunner.assign( + formFilters, formUrn, BATCH_FORM_ENTITY_COUNT, entityClient, systemAuthentication); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to dynamically assign form with urn: %s", formUrn), e); + } + } + + /** Submit a response for a structured property type prompt. */ + public Boolean batchSubmitStructuredPropertyPromptResponse( + @Nonnull final List entityUrns, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) + throws Exception { + entityUrns.forEach( + urnStr -> { + Urn urn = UrnUtils.getUrn(urnStr); + try { + submitStructuredPropertyPromptResponse( + urn, structuredPropertyUrn, values, formUrn, formPromptId, authentication); + } catch (Exception e) { + throw new RuntimeException("Failed to batch submit structured property prompt", e); + } + }); + + return true; + } + + /** Submit a response for a structured property type prompt. */ + public Boolean submitStructuredPropertyPromptResponse( + @Nonnull final Urn entityUrn, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) + throws Exception { + + // First, let's apply the action and add the structured property. + ingestStructuredProperties(entityUrn, structuredPropertyUrn, values, authentication); + + // Then, let's apply the change to the entity's form status. + ingestCompletedFormResponse(entityUrn, formUrn, formPromptId, authentication); + + return true; + } + + /** Submit a response for a field-level structured property type prompt. */ + public Boolean batchSubmitFieldStructuredPropertyPromptResponse( + @Nonnull final List entityUrns, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final String fieldPath, + @Nonnull final Authentication authentication) + throws Exception { + entityUrns.forEach( + urnStr -> { + Urn urn = UrnUtils.getUrn(urnStr); + try { + submitFieldStructuredPropertyPromptResponse( + urn, + structuredPropertyUrn, + values, + formUrn, + formPromptId, + fieldPath, + authentication); + } catch (Exception e) { + throw new RuntimeException( + "Failed to batch submit field structured property prompt", e); + } + }); + + return true; + } + + /** Submit a response for a field-level structured property type prompt. */ + public Boolean submitFieldStructuredPropertyPromptResponse( + @Nonnull final Urn entityUrn, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final String fieldPath, + @Nonnull final Authentication authentication) + throws Exception { + + // First, let's apply the action and add the structured property. + ingestSchemaFieldStructuredProperties( + entityUrn, structuredPropertyUrn, values, fieldPath, authentication); + + // Then, let's apply the change to the entity's form status. + ingestCompletedFieldFormResponse(entityUrn, formUrn, formPromptId, fieldPath, authentication); + + return true; + } + + private void ingestCompletedFieldFormResponse( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final String fieldPath, + @Nonnull final Authentication authentication) + throws Exception { + final Forms forms = getEntityForms(entityUrn, authentication); + final FormAssociation formAssociation = getFormWithUrn(forms, formUrn); + if (formAssociation == null) { + throw new RuntimeException( + String.format("Form %s has not been assigned to entity %s", formUrn, entityUrn)); + } + final FormPromptAssociation formPromptAssociation = + getOrDefaultFormPromptAssociation(formAssociation, formPromptId, authentication); + + // update the prompt association to have this fieldFormPromptAssociation marked as complete + updateFieldPromptToComplete( + formPromptAssociation, fieldPath, UrnUtils.getUrn(authentication.getActor().toUrnStr())); + + // field prompt is complete if all fields in entity's schema metadata are marked complete + if (isFieldPromptComplete(entityUrn, formPromptAssociation, authentication)) { + // if this is complete, the prompt as a whole should be marked as complete + ingestCompletedFormResponse(entityUrn, formUrn, formPromptId, forms, authentication); + } else { + // regardless, ingest forms to save state of this aspect + ingestForms(entityUrn, forms, authentication); + } + } + + private void ingestCompletedFormResponse( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) + throws Exception { + final Forms forms = getEntityForms(entityUrn, authentication); + ingestCompletedFormResponse(entityUrn, formUrn, formPromptId, forms, authentication); + } + + private void ingestCompletedFormResponse( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Forms forms, + @Nonnull final Authentication authentication) + throws Exception { + // Next, get all the information we need to update the forms for the entity. + final FormInfo formInfo = getFormInfo(formUrn, authentication); + final FormAssociation formAssociation = getFormWithUrn(forms, formUrn); + + if (formAssociation == null) { + throw new RuntimeException( + String.format("Form %s has not been assigned to entity %s", formUrn, entityUrn)); + } + + // First, mark the prompt as completed in forms aspect. + updatePromptToComplete(formAssociation, entityUrn, formUrn, formPromptId, authentication); + + // Then, update the completed forms fields based on which prompts remain incomplete. + updateFormCompletion(forms, formAssociation, formInfo); + + // Finally, ingest the newly updated forms aspect. + ingestForms(entityUrn, forms, authentication); + } + + private void ingestSchemaFieldStructuredProperties( + @Nonnull final Urn entityUrn, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final String fieldPath, + @Nonnull final Authentication authentication) + throws Exception { + Urn schemaFieldUrn = SchemaFieldUtils.generateSchemaFieldUrn(entityUrn.toString(), fieldPath); + ingestStructuredProperties(schemaFieldUrn, structuredPropertyUrn, values, authentication); + } + + private void ingestStructuredProperties( + @Nonnull final Urn entityUrn, + @Nonnull final Urn structuredPropertyUrn, + @Nonnull final PrimitivePropertyValueArray values, + @Nonnull final Authentication authentication) + throws Exception { + final EntityResponse response = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(STRUCTURED_PROPERTIES_ASPECT_NAME), + authentication); + + StructuredProperties structuredProperties = new StructuredProperties(); + structuredProperties.setProperties(new StructuredPropertyValueAssignmentArray()); + if (response != null && response.getAspects().containsKey(STRUCTURED_PROPERTIES_ASPECT_NAME)) { + structuredProperties = + new StructuredProperties( + response.getAspects().get(STRUCTURED_PROPERTIES_ASPECT_NAME).getValue().data()); + } + + // Since we upsert assignments for this structuredProperty, + // remove anything from this structured property and add to this list + List filteredAssignments = + structuredProperties.getProperties().stream() + .filter(assignment -> !assignment.getPropertyUrn().equals(structuredPropertyUrn)) + .collect(Collectors.toList()); + + StructuredPropertyValueAssignment assignment = new StructuredPropertyValueAssignment(); + assignment.setValues(values); + assignment.setPropertyUrn(structuredPropertyUrn); + assignment.setCreated( + new AuditStamp() + .setActor(UrnUtils.getUrn(authentication.getActor().toUrnStr())) + .setTime(System.currentTimeMillis())); + assignment.setLastModified( + new AuditStamp() + .setActor(UrnUtils.getUrn(authentication.getActor().toUrnStr())) + .setTime(System.currentTimeMillis())); + filteredAssignments.add(assignment); + + StructuredPropertyValueAssignmentArray assignments = + new StructuredPropertyValueAssignmentArray(filteredAssignments); + structuredProperties.setProperties(assignments); + + final MetadataChangeProposal structuredPropertiesProposal = + AspectUtils.buildMetadataChangeProposal( + entityUrn, STRUCTURED_PROPERTIES_ASPECT_NAME, structuredProperties); + try { + this.entityClient.ingestProposal(structuredPropertiesProposal, authentication, false); + } catch (Exception e) { + throw new RuntimeException("Failed to submit form response", e); + } + } + + private void ingestForms( + @Nonnull final Urn entityUrn, + @Nonnull final Forms forms, + @Nonnull final Authentication authentication) { + try { + ingestChangeProposals( + ImmutableList.of( + AspectUtils.buildMetadataChangeProposal(entityUrn, FORMS_ASPECT_NAME, forms)), + authentication); + } catch (Exception e) { + log.warn(String.format("Failed to ingest forms for entity with urn %s", entityUrn), e); + } + } + + private Forms getEntityForms( + @Nonnull final Urn entityUrn, @Nonnull final Authentication authentication) throws Exception { + final EntityResponse response = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(FORMS_ASPECT_NAME), + authentication); + if (response != null && response.getAspects().containsKey(FORMS_ASPECT_NAME)) { + return new Forms(response.getAspects().get(FORMS_ASPECT_NAME).getValue().data()); + } + // No entity forms found. + throw new RuntimeException( + String.format( + "Entity is missing forms aspect, form is not assigned to entity with urn %s", + entityUrn)); + } + + /** + * Checks schema metadata for an entity and ensures there's a completed field prompt for every + * field. If there is no schema metadata, raise an error. + */ + private boolean isFieldPromptComplete( + @Nonnull final Urn entityUrn, + @Nonnull final FormPromptAssociation formPromptAssociation, + @Nonnull final Authentication authentication) + throws Exception { + final Set completedFieldPaths = + Objects.requireNonNull(formPromptAssociation.getFieldAssociations()) + .getCompletedFieldPrompts() + .stream() + .map(FieldFormPromptAssociation::getFieldPath) + .collect(Collectors.toSet()); + final SchemaMetadata schemaMetadata = getSchemaMetadata(entityUrn, authentication); + final List fieldPaths = + schemaMetadata.getFields().stream() + .map(SchemaField::getFieldPath) + .collect(Collectors.toList()); + + return completedFieldPaths.containsAll(fieldPaths); + } + + /** + * Performs the operation of changing the status of a form field prompt from incomplete to + * complete. + */ + private void updateFieldPromptToComplete( + @Nonnull final FormPromptAssociation formPromptAssociation, + @Nonnull final String fieldPath, + @Nonnull final Urn actor) { + final FieldFormPromptAssociation completedFieldPromptAssociation = + new FieldFormPromptAssociation(); + completedFieldPromptAssociation.setFieldPath(fieldPath); + completedFieldPromptAssociation.setLastModified(createAuditStamp(actor)); + + FormPromptFieldAssociations fieldAssociations = + formPromptAssociation.getFieldAssociations() != null + ? formPromptAssociation.getFieldAssociations() + : new FormPromptFieldAssociations(); + + if (fieldAssociations.getCompletedFieldPrompts() == null) { + fieldAssociations.setCompletedFieldPrompts(new FieldFormPromptAssociationArray()); + } + if (fieldAssociations.getIncompleteFieldPrompts() == null) { + fieldAssociations.setIncompleteFieldPrompts(new FieldFormPromptAssociationArray()); + } + + // add this prompt association to list of completed prompts, removing its previous association + // if it was already in there + FieldFormPromptAssociationArray completedFieldPrompts = + new FieldFormPromptAssociationArray( + fieldAssociations.getCompletedFieldPrompts().stream() + .filter(fieldPrompt -> !fieldPrompt.getFieldPath().equals(fieldPath)) + .collect(Collectors.toList())); + completedFieldPrompts.add(completedFieldPromptAssociation); + fieldAssociations.setCompletedFieldPrompts(completedFieldPrompts); + + // remove this prompt association from list of incomplete prompts + FieldFormPromptAssociationArray incompleteFieldPrompts = new FieldFormPromptAssociationArray(); + fieldAssociations + .getIncompleteFieldPrompts() + .forEach( + incompleteFieldPrompt -> { + if (!incompleteFieldPrompt.getFieldPath().equals(fieldPath)) { + incompleteFieldPrompts.add(incompleteFieldPrompt); + } + }); + fieldAssociations.setIncompleteFieldPrompts(incompleteFieldPrompts); + + formPromptAssociation.setFieldAssociations(fieldAssociations); + } + + /** Performs the operation of changing the status of a form prompt from incomplete to complete. */ + private void updatePromptToComplete( + @Nonnull final FormAssociation formAssociation, + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) { + final FormPromptAssociation formPromptAssociation = + getOrDefaultFormPromptAssociation(formAssociation, formPromptId, authentication); + + // add this prompt association to list of completed prompts, removing its previous association + // if it was already in there + FormPromptAssociationArray completedPrompts = + new FormPromptAssociationArray( + formAssociation.getCompletedPrompts().stream() + .filter(prompt -> !prompt.getId().equals(formPromptId)) + .collect(Collectors.toList())); + completedPrompts.add(formPromptAssociation); + formAssociation.setCompletedPrompts(completedPrompts); + + // remove this prompt association from list of incomplete prompts + FormPromptAssociationArray incompletePrompts = new FormPromptAssociationArray(); + formAssociation + .getIncompletePrompts() + .forEach( + incompletePrompt -> { + if (!incompletePrompt.getId().equals(formPromptId)) { + incompletePrompts.add(incompletePrompt); + } + }); + formAssociation.setIncompletePrompts(incompletePrompts); + } + + /** Performs the operation of changing the status of a form prompt from complete to incomplete. */ + private void updatePromptToIncomplete( + @Nonnull final FormAssociation form, + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId) { + // Remove the prompt from completed. + final List newCompletedPrompts = + form.getCompletedPrompts().stream() + .filter(prompt -> !prompt.getId().equals(formPromptId)) + .collect(Collectors.toList()); + form.setCompletedPrompts(new FormPromptAssociationArray(newCompletedPrompts)); + + // Add the prompt to in-completed. + if (form.getIncompletePrompts().stream() + .anyMatch(prompt -> prompt.getId().equals(formPromptId))) { + log.warn( + String.format( + "Attempting to unset a prompt that is already incomplete. Skipping... Form: %s, Prompt: %s, Entity: %s", + formUrn, formPromptId, entityUrn)); + return; + } + final List newIncompletePrompts = + new ArrayList<>(form.getIncompletePrompts()); + newIncompletePrompts.add( + new FormPromptAssociation().setId(formPromptId).setLastModified(createSystemAuditStamp())); + form.setIncompletePrompts(new FormPromptAssociationArray(newIncompletePrompts)); + } + + private List buildAssignFormChanges( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) { + final List results = new ArrayList<>(); + entityUrns.forEach( + entityUrn -> { + try { + MetadataChangeProposal maybeChange = + buildAssignFormChange(entityUrn, formUrn, authentication); + if (maybeChange != null) { + results.add(maybeChange); + } + } catch (Exception e) { + log.warn( + String.format( + "Failed to retrieve form %s for entity %s. Skipping form assignment", + formUrn, entityUrn), + e); + } + }); + return results; + } + + @Nullable + private MetadataChangeProposal buildAssignFormChange( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) + throws Exception { + + final EntityResponse response = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(FORMS_ASPECT_NAME), + authentication); + + Forms formsAspect = new Forms(); + formsAspect.setIncompleteForms(new FormAssociationArray()); + formsAspect.setCompletedForms(new FormAssociationArray()); + if (response != null && response.getAspects().containsKey(FORMS_ASPECT_NAME)) { + formsAspect = new Forms(response.getAspects().get(FORMS_ASPECT_NAME).getValue().data()); + } + + // if this form is already assigned to this entity, leave it and move on + Optional formAssociation = + Stream.concat( + formsAspect.getCompletedForms().stream(), formsAspect.getIncompleteForms().stream()) + .filter(form -> form.getUrn().equals(formUrn)) + .findAny(); + + if (formAssociation.isPresent()) { + return null; + } + + // add this form to the entity's incomplete form associations. + FormAssociationArray incompleteForms = formsAspect.getIncompleteForms(); + FormAssociation newAssociation = new FormAssociation(); + newAssociation.setUrn(formUrn); + + // set all prompts as incomplete when assigning this form + FormInfo formInfo = getFormInfo(formUrn, authentication); + FormPromptAssociationArray formPromptAssociations = new FormPromptAssociationArray(); + formInfo + .getPrompts() + .forEach( + prompt -> { + FormPromptAssociation association = new FormPromptAssociation(); + association.setId(prompt.getId()); + association.setLastModified(createAuditStamp(authentication)); + formPromptAssociations.add(association); + }); + newAssociation.setIncompletePrompts(formPromptAssociations); + newAssociation.setCompletedPrompts(new FormPromptAssociationArray()); + incompleteForms.add(newAssociation); + formsAspect.setIncompleteForms(incompleteForms); + return buildMetadataChangeProposal(entityUrn, FORMS_ASPECT_NAME, formsAspect); + } + + private List buildUnassignFormChanges( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) { + final List results = new ArrayList<>(); + entityUrns.forEach( + entityUrn -> { + try { + MetadataChangeProposal maybeChange = + buildUnassignFormChange(entityUrn, formUrn, authentication); + if (maybeChange != null) { + results.add(maybeChange); + } + } catch (Exception e) { + log.warn( + String.format( + "Failed to retrieve form %s for entity %s. Skipping form unassignment.", + formUrn, entityUrn), + e); + } + }); + return results; + } + + @Nullable + private MetadataChangeProposal buildUnassignFormChange( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final Authentication authentication) + throws Exception { + final EntityResponse response = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(FORMS_ASPECT_NAME), + authentication); + Forms formsAspect = new Forms(); + formsAspect.setCompletedForms(new FormAssociationArray()); + formsAspect.setIncompleteForms(new FormAssociationArray()); + if (response != null && response.getAspects().containsKey(FORMS_ASPECT_NAME)) { + formsAspect = new Forms(response.getAspects().get(FORMS_ASPECT_NAME).getValue().data()); + } + + List newCompleted = + new ArrayList<>( + new FormAssociationArray( + formsAspect.getCompletedForms().stream() + .filter(form -> !form.getUrn().equals(formUrn)) + .collect(Collectors.toList()))); + List newIncomplete = + new ArrayList<>( + new FormAssociationArray( + formsAspect.getIncompleteForms().stream() + .filter(form -> !form.getUrn().equals(formUrn)) + .collect(Collectors.toList()))); + + if (newCompleted.size() == formsAspect.getCompletedForms().size() + && newIncomplete.size() == formsAspect.getIncompleteForms().size()) { + // No metadata to change. Skip ingestion. + return null; + } + + formsAspect.setCompletedForms(new FormAssociationArray(newCompleted)); + formsAspect.setIncompleteForms(new FormAssociationArray(newIncomplete)); + + return buildMetadataChangeProposal(entityUrn, FORMS_ASPECT_NAME, formsAspect); + } + + private List buildUnsetFormPromptChanges( + @Nonnull final List entityUrns, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final FormInfo formDefinition, + @Nonnull final Authentication authentication) { + final List results = new ArrayList<>(); + entityUrns.forEach( + entityUrn -> { + try { + MetadataChangeProposal maybeChange = + buildUnsetFormPromptChange( + entityUrn, formUrn, formPromptId, formDefinition, authentication); + if (maybeChange != null) { + results.add(maybeChange); + } + } catch (Exception e) { + log.warn( + String.format( + "Failed to retrieve form %s for entity %s. Skipping form unassignment.", + formUrn, entityUrn), + e); + } + }); + return results; + } + + @Nullable + private MetadataChangeProposal buildUnsetFormPromptChange( + @Nonnull final Urn entityUrn, + @Nonnull final Urn formUrn, + @Nonnull final String formPromptId, + @Nonnull final FormInfo formDefinition, + @Nonnull final Authentication authentication) + throws Exception { + + // Retrieve entity forms state + final Forms forms = getEntityForms(entityUrn, authentication); + + // First, find the form with the provided urn. + final FormAssociation formAssociation = getFormWithUrn(forms, formUrn); + + if (formAssociation != null) { + // 1. Find and mark the provided form prompt as incomplete. + updatePromptToIncomplete(formAssociation, entityUrn, formUrn, formPromptId); + + // 2. Update the form's completion status given the incomplete prompt. + updateFormCompletion(forms, formAssociation, formDefinition); + + // 3. Update the form status aspect for the entity. + return buildMetadataChangeProposal(entityUrn, FORMS_ASPECT_NAME, forms); + } else { + // Form not assigned to the entity! Let's warn and do nothing. + log.warn( + String.format( + "Failed to find form with urn %s associated with entity urn %s while attempting to unset form prompt %s. Skipping...", + formUrn, entityUrn, formPromptId)); + } + + return null; + } + + private void updateFormCompletion( + @Nonnull final Forms forms, + @Nonnull final FormAssociation form, + @Nonnull final FormInfo formDefinition) { + + final boolean isFormCompleted = isFormCompleted(form, formDefinition); + + if (isFormCompleted) { + // If the form is complete, we want to add it to completed forms. + + // 1. Remove from incomplete. + forms.setIncompleteForms( + new FormAssociationArray( + forms.getIncompleteForms().stream() + .filter(incompleteForm -> !incompleteForm.getUrn().equals(form.getUrn())) + .collect(Collectors.toList()))); + + // 2. Add to complete (if not already present) + if (forms.getCompletedForms().stream() + .noneMatch(completedForm -> completedForm.getUrn().equals(form.getUrn()))) { + // Not found in completed, let's update it. + List newCompleted = new ArrayList<>(forms.getCompletedForms()); + newCompleted.add(form); + forms.setCompletedForms(new FormAssociationArray(newCompleted)); + } + } else { + // If the form is incomplete, we want to remove it from the completed forms. + // If the form implies verification, we also ensure that the verification status is + // un-applied. + + // 1. Remove from complete. + forms.setCompletedForms( + new FormAssociationArray( + forms.getCompletedForms().stream() + .filter(completedForm -> !completedForm.getUrn().equals(form.getUrn())) + .collect(Collectors.toList()))); + + // 2. Add to incomplete (if not already present) + if (forms.getIncompleteForms().stream() + .noneMatch(incompleteForm -> incompleteForm.getUrn().equals(form.getUrn()))) { + // Not found in incompleted. Let's updated + List newIncomplete = new ArrayList<>(forms.getIncompleteForms()); + newIncomplete.add(form); + forms.setIncompleteForms(new FormAssociationArray(newIncomplete)); + } + + // 3. Remove verification as required. + if (FormType.VERIFICATION.equals(formDefinition.getType())) { + removeFormVerification(form.getUrn(), forms); + } + } + } + + /** + * Returns true if a form is considered completed, false otherwise. This is a function of whether + * all required prompts are marked as completed. + * + *

If none or some required prompts are marked as completed, then the form will be considered + * NOT completed. + * + * @param form the form status, as completed for a specific entity. + * @param formDefinition the form definition, which contains information about which prompts are + * required. + */ + private boolean isFormCompleted( + @Nonnull final FormAssociation form, @Nonnull final FormInfo formDefinition) { + final List requiredPromptsIds = + formDefinition.getPrompts().stream() + .filter(FormPrompt::isRequired) + .map(FormPrompt::getId) + .collect(Collectors.toList()); + + final List completedPromptIds = + form.getCompletedPrompts().stream() + .map(FormPromptAssociation::getId) + .collect(Collectors.toList()); + + // If all required prompts are completed, then the form is completed. + return completedPromptIds.containsAll(requiredPromptsIds); + } + + @Nullable + private FormAssociation getFormWithUrn( + @Nonnull final Forms existingForms, @Nonnull final Urn formUrn) { + // First check in the completed set. + Optional maybeForm = + existingForms.getCompletedForms().stream() + .filter(form -> form.getUrn().equals(formUrn)) + .findFirst(); + if (maybeForm.isPresent()) { + return maybeForm.get(); + } + + // Then check the incomplete set. + maybeForm = + existingForms.getIncompleteForms().stream() + .filter(form -> form.getUrn().equals(formUrn)) + .findFirst(); + if (maybeForm.isPresent()) { + return maybeForm.get(); + } + + // No form found, return null. + return null; + } + + @Nullable + private FormPromptAssociation getFormPromptAssociation( + @Nonnull final FormAssociation formAssociation, @Nonnull final String formPromptId) { + // First check in the completed set. + Optional maybePromptAssociation = + formAssociation.getCompletedPrompts().stream() + .filter(prompt -> prompt.getId().equals(formPromptId)) + .findFirst(); + if (maybePromptAssociation.isPresent()) { + return maybePromptAssociation.get(); + } + + // Then check the incomplete set. + maybePromptAssociation = + formAssociation.getIncompletePrompts().stream() + .filter(prompt -> prompt.getId().equals(formPromptId)) + .findFirst(); + if (maybePromptAssociation.isPresent()) { + return maybePromptAssociation.get(); + } + + // No prompt association found, return null. + return null; + } + + /** + * Gets a form prompt association by the prompt ID. If none exists (could happen as a form was + * changed after assigned or some other reason), then create the association and add it to the + * formAssociation's list of incomplete prompts. + */ + private FormPromptAssociation getOrDefaultFormPromptAssociation( + @Nonnull final FormAssociation formAssociation, + @Nonnull final String formPromptId, + @Nonnull final Authentication authentication) { + final FormPromptAssociation existingPromptAssociation = + getFormPromptAssociation(formAssociation, formPromptId); + final FormPromptAssociation formPromptAssociation = + existingPromptAssociation != null ? existingPromptAssociation : new FormPromptAssociation(); + formPromptAssociation.setId(formPromptId); + formPromptAssociation.setLastModified( + createAuditStamp(UrnUtils.getUrn(authentication.getActor().toUrnStr()))); + if (existingPromptAssociation == null) { + FormPromptAssociationArray incompletePrompts = + new FormPromptAssociationArray(formAssociation.getIncompletePrompts()); + incompletePrompts.add(formPromptAssociation); + formAssociation.setIncompletePrompts(incompletePrompts); + } + return formPromptAssociation; + } + + private void removeFormVerification(@Nonnull final Urn formUrn, @Nonnull final Forms forms) { + if (!forms.hasVerifications()) { + // Nothing to do. + return; + } + + // Remove verification of given urn. + final List newVerifications = + forms.getVerifications().stream() + .filter(verification -> !formUrn.equals(verification.getForm())) + .collect(Collectors.toList()); + + // Update verifications for forms aspect. + forms.setVerifications(new FormVerificationAssociationArray(newVerifications)); + } + + /** + * A form is assigned to a user if either the user or a group the user is in is explicitly set on + * the actors field on a form. Otherwise, if the actors field says that owners are assigned, + * ensure this actor, or a group they're in, is an owner of this entity. + */ + public boolean isFormAssignedToUser( + @Nonnull final Urn formUrn, + @Nonnull final Urn entityUrn, + @Nonnull final Urn actorUrn, + @Nonnull final List groupsForUser, + @Nonnull final Authentication authentication) + throws Exception { + final FormInfo formInfo = getFormInfo(formUrn, authentication); + final FormActorAssignment formActorAssignment = formInfo.getActors(); + if (FormUtils.isFormAssignedToUser(formActorAssignment, actorUrn, groupsForUser)) { + return true; + } + + if (formActorAssignment.isOwners()) { + Ownership entityOwnership = getEntityOwnership(entityUrn, authentication); + return OwnershipUtils.isOwnerOfEntity(entityOwnership, actorUrn, groupsForUser); + } + + return false; + } + + /** + * Adds a new form verification association for an entity for this form on their forms aspect. If + * there was an existing verification association for this form, remove and replace it. First, + * ensure this form is of VERIFICATION type and that this form is in completedForms. + */ + public boolean verifyFormForEntity( + @Nonnull final Urn formUrn, + @Nonnull final Urn entityUrn, + @Nonnull final Authentication authentication) + throws Exception { + final FormInfo formInfo = getFormInfo(formUrn, authentication); + if (!formInfo.getType().equals(FormType.VERIFICATION)) { + throw new UnsupportedOperationException( + String.format("Form %s is not of type VERIFICATION. Cannot verify form.", formUrn)); + } + final Forms formsAspect = getEntityForms(entityUrn, authentication); + if (!isFormInCompletedForms(formUrn, formsAspect)) { + throw new RuntimeException( + String.format( + "Form %s is not in the list of completed forms for this entity. Skipping verification.", + formUrn)); + } + + // Remove any existing verifications for this form to patch a new one + List formVerifications = + formsAspect.getVerifications().stream() + .filter(verification -> !verification.getForm().equals(formUrn)) + .collect(Collectors.toList()); + FormVerificationAssociation newAssociation = new FormVerificationAssociation(); + newAssociation.setForm(formUrn); + newAssociation.setLastModified(createAuditStamp(authentication)); + formVerifications.add(newAssociation); + + formsAspect.setVerifications(new FormVerificationAssociationArray(formVerifications)); + + ingestForms(entityUrn, formsAspect, authentication); + return true; + } + + private boolean isFormInCompletedForms( + @Nonnull final Urn formUrn, @Nonnull final Forms formsAspect) { + return formsAspect.getCompletedForms().stream() + .anyMatch(completedForm -> completedForm.getUrn().equals(formUrn)); + } + + public FormInfo getFormInfo( + @Nonnull final Urn formUrn, @Nonnull final Authentication authentication) + throws URISyntaxException, RemoteInvocationException { + final EntityResponse formInfoResponse = + entityClient.getV2( + formUrn.getEntityType(), + formUrn, + ImmutableSet.of(FORM_INFO_ASPECT_NAME), + authentication); + if (formInfoResponse != null + && formInfoResponse.getAspects().containsKey(FORM_INFO_ASPECT_NAME)) { + return new FormInfo( + formInfoResponse.getAspects().get(FORM_INFO_ASPECT_NAME).getValue().data()); + } else { + throw new RuntimeException(String.format("Form %s does not exist.", formUrn)); + } + } + + private SchemaMetadata getSchemaMetadata( + @Nonnull final Urn entityUrn, @Nonnull final Authentication authentication) + throws URISyntaxException, RemoteInvocationException { + final EntityResponse response = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(SCHEMA_METADATA_ASPECT_NAME), + authentication); + if (response != null && response.getAspects().containsKey(SCHEMA_METADATA_ASPECT_NAME)) { + return new SchemaMetadata( + response.getAspects().get(SCHEMA_METADATA_ASPECT_NAME).getValue().data()); + } else { + throw new RuntimeException( + String.format("Schema metadata does not exist on entity %s.", entityUrn)); + } + } + + private Ownership getEntityOwnership( + @Nonnull final Urn entityUrn, @Nonnull final Authentication authentication) + throws URISyntaxException, RemoteInvocationException { + final EntityResponse entityResponse = + entityClient.getV2( + entityUrn.getEntityType(), + entityUrn, + ImmutableSet.of(OWNERSHIP_ASPECT_NAME), + authentication); + if (entityResponse != null && entityResponse.getAspects().containsKey(OWNERSHIP_ASPECT_NAME)) { + return new Ownership( + entityResponse.getAspects().get(OWNERSHIP_ASPECT_NAME).getValue().data()); + } else { + throw new RuntimeException(String.format("Ownership %s does not exist.", entityUrn)); + } + } + + private void verifyEntitiesExist( + @Nonnull final List entityUrns, @Nonnull final Authentication authentication) { + entityUrns.forEach( + entityUrn -> { + try { + verifyEntityExists(entityUrn, authentication); + } catch (Exception e) { + throw new RuntimeException( + String.format( + "Issue verifying whether entity exists when assigning form to it. Entity urn: %s", + entityUrn)); + } + }); + } + + private void verifyEntityExists( + @Nonnull final Urn entityUrn, @Nonnull final Authentication authentication) + throws RemoteInvocationException { + if (!entityClient.exists(entityUrn, authentication)) { + throw new RuntimeException( + String.format("Entity %s does not exist. Skipping batch form assignment", entityUrn)); + } + } + + private AuditStamp createSystemAuditStamp() { + return createAuditStamp(UrnUtils.getUrn(SYSTEM_ACTOR)); + } + + private AuditStamp createAuditStamp(@Nonnull final Authentication authentication) { + return createAuditStamp(UrnUtils.getUrn(authentication.getActor().toUrnStr())); + } + + private AuditStamp createAuditStamp(@Nonnull final Urn actor) { + return new AuditStamp().setTime(System.currentTimeMillis()).setActor(actor); + } +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java new file mode 100644 index 0000000000000..666fe23a93187 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/RollbackService.java @@ -0,0 +1,328 @@ +package com.linkedin.metadata.service; + +import static com.linkedin.metadata.Constants.DEFAULT_RUN_ID; + +import com.datahub.authentication.Authentication; +import com.datahub.authentication.AuthenticationException; +import com.datahub.authorization.AuthUtil; +import com.datahub.authorization.ConjunctivePrivilegeGroup; +import com.datahub.authorization.DisjunctivePrivilegeGroup; +import com.datahub.authorization.EntitySpec; +import com.datahub.plugins.auth.authorization.Authorizer; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.execution.ExecutionRequestResult; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.authorization.PoliciesConfig; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.entity.RollbackRunResult; +import com.linkedin.metadata.key.ExecutionRequestKey; +import com.linkedin.metadata.run.AspectRowSummary; +import com.linkedin.metadata.run.AspectRowSummaryArray; +import com.linkedin.metadata.run.RollbackResponse; +import com.linkedin.metadata.run.UnsafeEntityInfo; +import com.linkedin.metadata.run.UnsafeEntityInfoArray; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.timeseries.DeleteAspectValuesResult; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** Extracts logic historically in the Restli service which acts across multiple services */ +@Slf4j +@AllArgsConstructor +public class RollbackService { + public static final String ROLLING_BACK_STATUS = "ROLLING_BACK"; + public static final String ROLLED_BACK_STATUS = "ROLLED_BACK"; + public static final String ROLLBACK_FAILED_STATUS = "ROLLBACK_FAILED"; + + public static final int MAX_RESULT_SIZE = 10000; + public static final int ELASTIC_MAX_PAGE_SIZE = 10000; + public static final int DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE = 1000000; + public static final int ELASTIC_BATCH_DELETE_SLEEP_SEC = 5; + + private final EntityService entityService; + private final SystemMetadataService systemMetadataService; + private final TimeseriesAspectService timeseriesAspectService; + private final boolean restApiAuthorizationEnabled; + + public List rollbackTargetAspects(@Nonnull String runId, boolean hardDelete) { + return systemMetadataService.findByRunId(runId, hardDelete, 0, MAX_RESULT_SIZE); + } + + public RollbackResponse rollbackIngestion( + @Nonnull String runId, + boolean dryRun, + boolean hardDelete, + Authorizer authorizer, + @Nonnull Authentication authentication) + throws AuthenticationException { + + if (runId.equals(DEFAULT_RUN_ID)) { + throw new IllegalArgumentException( + String.format( + "%s is a default run-id provided for non labeled ingestion runs. You cannot delete using this reserved run-id", + runId)); + } + + if (!dryRun) { + updateExecutionRequestStatus(runId, ROLLING_BACK_STATUS); + } + + List aspectRowsToDelete = rollbackTargetAspects(runId, hardDelete); + if (!isAuthorized(authorizer, aspectRowsToDelete, authentication)) { + throw new AuthenticationException("User is NOT unauthorized to delete entities."); + } + + log.info("found {} rows to delete...", stringifyRowCount(aspectRowsToDelete.size())); + if (dryRun) { + + final Map> aspectsSplitByIsKeyAspects = + aspectRowsToDelete.stream() + .collect(Collectors.partitioningBy(AspectRowSummary::isKeyAspect)); + + final List keyAspects = aspectsSplitByIsKeyAspects.get(true); + + long entitiesDeleted = keyAspects.size(); + long aspectsReverted = aspectRowsToDelete.size(); + + final long affectedEntities = + aspectRowsToDelete.stream() + .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) + .keySet() + .size(); + + final AspectRowSummaryArray rowSummaries = + new AspectRowSummaryArray( + aspectRowsToDelete.subList(0, Math.min(100, aspectRowsToDelete.size()))); + + // If we are soft deleting, remove key aspects from count of aspects being deleted + if (!hardDelete) { + aspectsReverted -= keyAspects.size(); + rowSummaries.removeIf(AspectRowSummary::isKeyAspect); + } + // Compute the aspects that exist referencing the key aspects we are deleting + final List affectedAspectsList = + keyAspects.stream() + .map( + (AspectRowSummary urn) -> + systemMetadataService.findByUrn(urn.getUrn(), false, 0, MAX_RESULT_SIZE)) + .flatMap(List::stream) + .filter( + row -> + !row.getRunId().equals(runId) + && !row.isKeyAspect() + && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) + .collect(Collectors.toList()); + + long unsafeEntitiesCount = + affectedAspectsList.stream() + .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) + .keySet() + .size(); + + final List unsafeEntityInfos = + affectedAspectsList.stream() + .map(AspectRowSummary::getUrn) + .distinct() + .map( + urn -> { + UnsafeEntityInfo unsafeEntityInfo = new UnsafeEntityInfo(); + unsafeEntityInfo.setUrn(urn); + return unsafeEntityInfo; + }) + // Return at most 1 million rows + .limit(DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE) + .collect(Collectors.toList()); + + return new RollbackResponse() + .setAspectsReverted(aspectsReverted) + .setEntitiesAffected(affectedEntities) + .setEntitiesDeleted(entitiesDeleted) + .setUnsafeEntitiesCount(unsafeEntitiesCount) + .setUnsafeEntities(new UnsafeEntityInfoArray(unsafeEntityInfos)) + .setAspectRowSummaries(rowSummaries); + } + + RollbackRunResult rollbackRunResult = + entityService.rollbackRun(aspectRowsToDelete, runId, hardDelete); + final List deletedRows = rollbackRunResult.getRowsRolledBack(); + int rowsDeletedFromEntityDeletion = rollbackRunResult.getRowsDeletedFromEntityDeletion(); + + // since elastic limits how many rows we can access at once, we need to iteratively + // delete + while (aspectRowsToDelete.size() >= ELASTIC_MAX_PAGE_SIZE) { + sleep(ELASTIC_BATCH_DELETE_SLEEP_SEC); + aspectRowsToDelete = systemMetadataService.findByRunId(runId, hardDelete, 0, MAX_RESULT_SIZE); + log.info("{} remaining rows to delete...", stringifyRowCount(aspectRowsToDelete.size())); + log.info("deleting..."); + rollbackRunResult = entityService.rollbackRun(aspectRowsToDelete, runId, hardDelete); + deletedRows.addAll(rollbackRunResult.getRowsRolledBack()); + rowsDeletedFromEntityDeletion += rollbackRunResult.getRowsDeletedFromEntityDeletion(); + } + + // Rollback timeseries aspects + DeleteAspectValuesResult timeseriesRollbackResult = + timeseriesAspectService.rollbackTimeseriesAspects(runId); + rowsDeletedFromEntityDeletion += timeseriesRollbackResult.getNumDocsDeleted(); + + log.info("finished deleting {} rows", deletedRows.size()); + int aspectsReverted = deletedRows.size() + rowsDeletedFromEntityDeletion; + + final Map> aspectsSplitByIsKeyAspects = + aspectRowsToDelete.stream() + .collect(Collectors.partitioningBy(AspectRowSummary::isKeyAspect)); + + final List keyAspects = aspectsSplitByIsKeyAspects.get(true); + + final long entitiesDeleted = keyAspects.size(); + final long affectedEntities = + deletedRows.stream() + .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) + .keySet() + .size(); + + final AspectRowSummaryArray rowSummaries = + new AspectRowSummaryArray( + aspectRowsToDelete.subList(0, Math.min(100, aspectRowsToDelete.size()))); + + log.info("computing aspects affected by this rollback..."); + // Compute the aspects that exist referencing the key aspects we are deleting + final List affectedAspectsList = + keyAspects.stream() + .map( + (AspectRowSummary urn) -> + systemMetadataService.findByUrn(urn.getUrn(), false, 0, MAX_RESULT_SIZE)) + .flatMap(List::stream) + .filter( + row -> + !row.getRunId().equals(runId) + && !row.isKeyAspect() + && !row.getAspectName().equals(Constants.STATUS_ASPECT_NAME)) + .collect(Collectors.toList()); + + long affectedAspects = affectedAspectsList.size(); + long unsafeEntitiesCount = + affectedAspectsList.stream() + .collect(Collectors.groupingBy(AspectRowSummary::getUrn)) + .keySet() + .size(); + + final List unsafeEntityInfos = + affectedAspectsList.stream() + .map(AspectRowSummary::getUrn) + .distinct() + .map( + urn -> { + UnsafeEntityInfo unsafeEntityInfo = new UnsafeEntityInfo(); + unsafeEntityInfo.setUrn(urn); + return unsafeEntityInfo; + }) + // Return at most 1 million rows + .limit(DEFAULT_UNSAFE_ENTITIES_PAGE_SIZE) + .collect(Collectors.toList()); + + log.info("calculation done."); + + updateExecutionRequestStatus(runId, ROLLED_BACK_STATUS); + + return new RollbackResponse() + .setAspectsAffected(affectedAspects) + .setAspectsReverted(aspectsReverted) + .setEntitiesAffected(affectedEntities) + .setEntitiesDeleted(entitiesDeleted) + .setUnsafeEntitiesCount(unsafeEntitiesCount) + .setUnsafeEntities(new UnsafeEntityInfoArray(unsafeEntityInfos)) + .setAspectRowSummaries(rowSummaries); + } + + public void updateExecutionRequestStatus(@Nonnull String runId, @Nonnull String status) { + try { + final Urn executionRequestUrn = + EntityKeyUtils.convertEntityKeyToUrn( + new ExecutionRequestKey().setId(runId), Constants.EXECUTION_REQUEST_ENTITY_NAME); + EnvelopedAspect aspect = + entityService.getLatestEnvelopedAspect( + executionRequestUrn.getEntityType(), + executionRequestUrn, + Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME); + if (aspect == null) { + log.warn("Aspect for execution request with runId {} not found", runId); + } else { + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + ExecutionRequestResult requestResult = new ExecutionRequestResult(aspect.getValue().data()); + requestResult.setStatus(status); + proposal.setEntityUrn(executionRequestUrn); + proposal.setEntityType(Constants.EXECUTION_REQUEST_ENTITY_NAME); + proposal.setAspectName(Constants.EXECUTION_REQUEST_RESULT_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(requestResult)); + proposal.setChangeType(ChangeType.UPSERT); + + entityService.ingestProposal( + proposal, + new AuditStamp() + .setActor(UrnUtils.getUrn(Constants.SYSTEM_ACTOR)) + .setTime(System.currentTimeMillis()), + false); + } + } catch (Exception e) { + log.error( + String.format( + "Not able to update execution result aspect with runId %s and new status %s.", + runId, status), + e); + } + } + + private boolean isAuthorized( + final Authorizer authorizer, + @Nonnull List rowSummaries, + @Nonnull Authentication authentication) { + DisjunctivePrivilegeGroup orGroup = + new DisjunctivePrivilegeGroup( + ImmutableList.of( + new ConjunctivePrivilegeGroup( + ImmutableList.of(PoliciesConfig.DELETE_ENTITY_PRIVILEGE.getType())))); + + List> resourceSpecs = + rowSummaries.stream() + .map(AspectRowSummary::getUrn) + .map(UrnUtils::getUrn) + .map(urn -> Optional.of(new EntitySpec(urn.getEntityType(), urn.toString()))) + .distinct() + .collect(Collectors.toList()); + + return !restApiAuthorizationEnabled + || AuthUtil.isAuthorizedForResources( + authorizer, authentication.getActor().toUrnStr(), resourceSpecs, orGroup); + } + + private static String stringifyRowCount(int size) { + if (size < ELASTIC_MAX_PAGE_SIZE) { + return String.valueOf(size); + } else { + return "at least " + size; + } + } + + private static void sleep(int seconds) { + try { + TimeUnit.SECONDS.sleep(seconds); + } catch (InterruptedException e) { + log.error("Rollback sleep exception", e); + } + } +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java new file mode 100644 index 0000000000000..73e3bc130ac9d --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentManager.java @@ -0,0 +1,94 @@ +package com.linkedin.metadata.service.util; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.DynamicFormAssignment; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.search.ScrollResult; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.service.FormService; +import com.linkedin.r2.RemoteInvocationException; +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SearchBasedFormAssignmentManager { + + private static final ImmutableList ENTITY_TYPES = + ImmutableList.of(Constants.DATASET_ENTITY_NAME); + + public static void apply( + DynamicFormAssignment formFilters, + Urn formUrn, + int batchFormEntityCount, + EntityClient entityClient, + Authentication authentication) + throws Exception { + + try { + int totalResults = 0; + int numResults = 0; + String scrollId = null; + FormService formService = new FormService(entityClient, authentication); + + do { + + ScrollResult results = + entityClient.scrollAcrossEntities( + ENTITY_TYPES, + "*", + formFilters.getFilter(), + scrollId, + "5m", + batchFormEntityCount, + null, + authentication); + + if (!results.hasEntities() + || results.getNumEntities() == 0 + || results.getEntities().isEmpty()) { + break; + } + + log.info("Search across entities results: {}.", results); + + if (results.hasEntities()) { + final List entityUrns = + results.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()); + + formService.batchAssignFormToEntities(entityUrns, formUrn); + + if (!entityUrns.isEmpty()) { + log.info("Batch assign {} entities to form {}.", entityUrns.size(), formUrn); + } + + numResults = results.getEntities().size(); + totalResults += numResults; + scrollId = results.getScrollId(); + + log.info( + "Starting batch assign forms, count: {} running total: {}, size: {}", + batchFormEntityCount, + totalResults, + results.getEntities().size()); + + } else { + break; + } + } while (scrollId != null); + + log.info("Successfully assigned {} entities to form {}.", totalResults, formUrn); + + } catch (RemoteInvocationException e) { + log.error("Error while assigning form to entities.", e); + throw new RuntimeException(e); + } + } + + private SearchBasedFormAssignmentManager() {} +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentRunner.java b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentRunner.java new file mode 100644 index 0000000000000..a20f71f550c65 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/service/util/SearchBasedFormAssignmentRunner.java @@ -0,0 +1,45 @@ +package com.linkedin.metadata.service.util; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.form.DynamicFormAssignment; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SearchBasedFormAssignmentRunner { + + public static void assign( + DynamicFormAssignment formFilters, + Urn formUrn, + int batchFormEntityCount, + EntityClient entityClient, + Authentication authentication) { + Runnable runnable = + new Runnable() { + @Override + public void run() { + try { + SearchBasedFormAssignmentManager.apply( + formFilters, formUrn, batchFormEntityCount, entityClient, authentication); + } catch (Exception e) { + log.error( + "SearchBasedFormAssignmentRunner failed to run. " + + "Options: formFilters: {}, " + + "formUrn: {}, " + + "batchFormCount: {}, " + + "entityClient: {}, ", + formFilters, + formUrn, + batchFormEntityCount, + entityClient); + throw new RuntimeException("Form assignment runner error.", e); + } + } + }; + + new Thread(runnable).start(); + } + + private SearchBasedFormAssignmentRunner() {} +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java b/metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java index 71c4d357ad1eb..b6bef33df1d7f 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/shared/ValidationUtils.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.shared; +import com.codahale.metrics.Timer; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.AbstractArrayTemplate; @@ -19,6 +20,7 @@ import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.utils.metrics.MetricUtils; import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -33,25 +35,27 @@ public class ValidationUtils { public static SearchResult validateSearchResult( final SearchResult searchResult, @Nonnull final EntityService entityService) { - if (searchResult == null) { - return null; + try (Timer.Context ignored = + MetricUtils.timer(ValidationUtils.class, "validateSearchResult").time()) { + if (searchResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + SearchResult validatedSearchResult = + new SearchResult() + .setFrom(searchResult.getFrom()) + .setMetadata(searchResult.getMetadata()) + .setPageSize(searchResult.getPageSize()) + .setNumEntities(searchResult.getNumEntities()); + + SearchEntityArray validatedEntities = + validatedUrns(searchResult.getEntities(), SearchEntity::getEntity, entityService, true) + .collect(Collectors.toCollection(SearchEntityArray::new)); + validatedSearchResult.setEntities(validatedEntities); + + return validatedSearchResult; } - Objects.requireNonNull(entityService, "entityService must not be null"); - - SearchResult validatedSearchResult = - new SearchResult() - .setFrom(searchResult.getFrom()) - .setMetadata(searchResult.getMetadata()) - .setPageSize(searchResult.getPageSize()) - .setNumEntities(searchResult.getNumEntities()); - - SearchEntityArray validatedEntities = - validatedUrns(searchResult.getEntities(), SearchEntity::getEntity, entityService, true) - .collect(Collectors.toCollection(SearchEntityArray::new)); - - validatedSearchResult.setEntities(validatedEntities); - - return validatedSearchResult; } public static ScrollResult validateScrollResult( @@ -81,78 +85,85 @@ public static ScrollResult validateScrollResult( public static BrowseResult validateBrowseResult( final BrowseResult browseResult, @Nonnull final EntityService entityService) { - if (browseResult == null) { - return null; + try (Timer.Context ignored = + MetricUtils.timer(ValidationUtils.class, "validateBrowseResult").time()) { + if (browseResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + BrowseResult validatedBrowseResult = + new BrowseResult() + .setGroups(browseResult.getGroups()) + .setMetadata(browseResult.getMetadata()) + .setFrom(browseResult.getFrom()) + .setPageSize(browseResult.getPageSize()) + .setNumGroups(browseResult.getNumGroups()) + .setNumEntities(browseResult.getNumEntities()) + .setNumElements(browseResult.getNumElements()); + + BrowseResultEntityArray validatedEntities = + validatedUrns(browseResult.getEntities(), BrowseResultEntity::getUrn, entityService, true) + .collect(Collectors.toCollection(BrowseResultEntityArray::new)); + validatedBrowseResult.setEntities(validatedEntities); + + return validatedBrowseResult; } - Objects.requireNonNull(entityService, "entityService must not be null"); - - BrowseResult validatedBrowseResult = - new BrowseResult() - .setGroups(browseResult.getGroups()) - .setMetadata(browseResult.getMetadata()) - .setFrom(browseResult.getFrom()) - .setPageSize(browseResult.getPageSize()) - .setNumGroups(browseResult.getNumGroups()) - .setNumEntities(browseResult.getNumEntities()) - .setNumElements(browseResult.getNumElements()); - - BrowseResultEntityArray validatedEntities = - validatedUrns(browseResult.getEntities(), BrowseResultEntity::getUrn, entityService, true) - .collect(Collectors.toCollection(BrowseResultEntityArray::new)); - - validatedBrowseResult.setEntities(validatedEntities); - - return validatedBrowseResult; } public static ListResult validateListResult( final ListResult listResult, @Nonnull final EntityService entityService) { - if (listResult == null) { - return null; + try (Timer.Context ignored = + MetricUtils.timer(ValidationUtils.class, "validateListResult").time()) { + if (listResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + ListResult validatedListResult = + new ListResult() + .setStart(listResult.getStart()) + .setCount(listResult.getCount()) + .setTotal(listResult.getTotal()); + + UrnArray validatedEntities = + validatedUrns(listResult.getEntities(), Function.identity(), entityService, true) + .collect(Collectors.toCollection(UrnArray::new)); + validatedListResult.setEntities(validatedEntities); + + return validatedListResult; } - Objects.requireNonNull(entityService, "entityService must not be null"); - - ListResult validatedListResult = - new ListResult() - .setStart(listResult.getStart()) - .setCount(listResult.getCount()) - .setTotal(listResult.getTotal()); - - UrnArray validatedEntities = - validatedUrns(listResult.getEntities(), Function.identity(), entityService, true) - .collect(Collectors.toCollection(UrnArray::new)); - - validatedListResult.setEntities(validatedEntities); - - return validatedListResult; } public static LineageSearchResult validateLineageSearchResult( final LineageSearchResult lineageSearchResult, @Nonnull final EntityService entityService) { - if (lineageSearchResult == null) { - return null; + try (Timer.Context ignored = + MetricUtils.timer(ValidationUtils.class, "validateLineageResult").time()) { + if (lineageSearchResult == null) { + return null; + } + Objects.requireNonNull(entityService, "entityService must not be null"); + + LineageSearchResult validatedLineageSearchResult = + new LineageSearchResult() + .setMetadata(lineageSearchResult.getMetadata()) + .setFrom(lineageSearchResult.getFrom()) + .setPageSize(lineageSearchResult.getPageSize()) + .setNumEntities(lineageSearchResult.getNumEntities()); + + LineageSearchEntityArray validatedEntities = + validatedUrns( + lineageSearchResult.getEntities(), + LineageSearchEntity::getEntity, + entityService, + true) + .collect(Collectors.toCollection(LineageSearchEntityArray::new)); + validatedLineageSearchResult.setEntities(validatedEntities); + + log.debug("Returning validated lineage search results"); + return validatedLineageSearchResult; } - Objects.requireNonNull(entityService, "entityService must not be null"); - - LineageSearchResult validatedLineageSearchResult = - new LineageSearchResult() - .setMetadata(lineageSearchResult.getMetadata()) - .setFrom(lineageSearchResult.getFrom()) - .setPageSize(lineageSearchResult.getPageSize()) - .setNumEntities(lineageSearchResult.getNumEntities()); - - LineageSearchEntityArray validatedEntities = - validatedUrns( - lineageSearchResult.getEntities(), - LineageSearchEntity::getEntity, - entityService, - true) - .collect(Collectors.toCollection(LineageSearchEntityArray::new)); - - validatedLineageSearchResult.setEntities(validatedEntities); - - return validatedLineageSearchResult; } public static EntityLineageResult validateEntityLineageResult( diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/GenericTimeseriesDocument.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/GenericTimeseriesDocument.java new file mode 100644 index 0000000000000..1442f099c4703 --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/GenericTimeseriesDocument.java @@ -0,0 +1,26 @@ +package com.linkedin.metadata.timeseries; + +import com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class GenericTimeseriesDocument { + @Nonnull private String urn; + private long timestampMillis; + + @JsonProperty("@timestamp") + private long timestamp; + + @Nonnull private Object event; + @Nullable private String messageId; + @Nullable private Object systemMetadata; + @Nullable private String eventGranularity; + private boolean isExploded; + @Nullable private String runId; + @Nullable private String partition; + @Nullable private Object partitionSpec; +} diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java index 54480bb700398..529e8e00ecf57 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesAspectService.java @@ -201,4 +201,15 @@ void upsertDocument( @Nonnull final JsonNode document); List getIndexSizes(); + + @Nonnull + TimeseriesScrollResult scrollAspects( + @Nonnull final String entityName, + @Nonnull final String aspectName, + @Nullable Filter filter, + @Nonnull List sortCriterion, + @Nullable String scrollId, + int count, + @Nullable Long startTimeMillis, + @Nullable Long endTimeMillis); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesScrollResult.java b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesScrollResult.java new file mode 100644 index 0000000000000..200db2dfde8eb --- /dev/null +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/timeseries/TimeseriesScrollResult.java @@ -0,0 +1,18 @@ +package com.linkedin.metadata.timeseries; + +import com.linkedin.metadata.aspect.EnvelopedAspect; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@AllArgsConstructor +@Data +@Builder +public class TimeseriesScrollResult { + int numResults; + int pageSize; + String scrollId; + List events; + List documents; +} diff --git a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java index 970235fc88c87..27aa9ee04cc75 100644 --- a/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java +++ b/metadata-service/servlet/src/main/java/com/datahub/gms/servlet/ConfigSearchExport.java @@ -4,7 +4,7 @@ import static com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder.KEYWORD_ANALYZER; import com.datahub.gms.util.CSVWriter; -import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; diff --git a/metadata-service/war/src/main/resources/boot/data_types.json b/metadata-service/war/src/main/resources/boot/data_types.json new file mode 100644 index 0000000000000..2d7294e45bd7a --- /dev/null +++ b/metadata-service/war/src/main/resources/boot/data_types.json @@ -0,0 +1,42 @@ +[ + { + "urn": "urn:li:dataType:datahub.string", + "info": { + "qualifiedName":"datahub.string", + "displayName": "String", + "description": "A string of characters." + } + }, + { + "urn": "urn:li:dataType:datahub.number", + "info": { + "qualifiedName":"datahub.number", + "displayName": "Number", + "description": "An integer or decimal number." + } + }, + { + "urn": "urn:li:dataType:datahub.urn", + "info": { + "qualifiedName":"datahub.urn", + "displayName": "Urn", + "description": "An unique identifier for a DataHub entity." + } + }, + { + "urn": "urn:li:dataType:datahub.rich_text", + "info": { + "qualifiedName":"datahub.rich_text", + "displayName": "Rich Text", + "description": "An attributed string of characters." + } + }, + { + "urn": "urn:li:dataType:datahub.date", + "info": { + "qualifiedName":"datahub.date", + "displayName": "Date", + "description": "A specific day, without time." + } + } +] diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 3d65675219624..919d93c5f9fe1 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'pegasus' } dependencies { diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/OwnershipUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/OwnershipUtils.java new file mode 100644 index 0000000000000..140b64780918d --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/OwnershipUtils.java @@ -0,0 +1,20 @@ +package com.linkedin.metadata.authorization; + +import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; +import java.util.List; +import javax.annotation.Nonnull; + +public class OwnershipUtils { + + public static boolean isOwnerOfEntity( + @Nonnull final Ownership entityOwnership, + @Nonnull final Urn actorUrn, + @Nonnull final List groupsForUser) { + return entityOwnership.getOwners().stream() + .anyMatch( + owner -> owner.getOwner().equals(actorUrn) || groupsForUser.contains(owner.getOwner())); + } + + private OwnershipUtils() {} +} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/AuditStampUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/AuditStampUtils.java index 5f3975b066fde..6ba311cf166d4 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/AuditStampUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/AuditStampUtils.java @@ -3,8 +3,11 @@ import static com.linkedin.metadata.Constants.SYSTEM_ACTOR; import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import java.net.URISyntaxException; import java.time.Clock; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -16,4 +19,11 @@ public static AuditStamp createDefaultAuditStamp() { .setActor(UrnUtils.getUrn(SYSTEM_ACTOR)) .setTime(Clock.systemUTC().millis()); } + + public static AuditStamp createAuditStamp(@Nonnull String actorUrn) throws URISyntaxException { + AuditStamp auditStamp = new AuditStamp(); + auditStamp.setActor(Urn.createFromString(actorUrn)); + auditStamp.setTime(Clock.systemUTC().millis()); + return auditStamp; + } } diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/FormUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/FormUtils.java new file mode 100644 index 0000000000000..ebf2587418dae --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/FormUtils.java @@ -0,0 +1,49 @@ +package com.linkedin.metadata.utils; + +import com.linkedin.common.urn.Urn; +import com.linkedin.form.FormActorAssignment; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +public class FormUtils { + + private FormUtils() {} + + public static boolean isFormAssignedToUser( + @Nonnull final FormActorAssignment parent, + @Nonnull final Urn userUrn, + @Nonnull final List groupUrns) { + // Assigned urn and group urns + final Set assignedUserUrns = + parent.getUsers() != null + ? parent.getUsers().stream().map(Urn::toString).collect(Collectors.toSet()) + : Collections.emptySet(); + + final Set assignedGroupUrns = + parent.getGroups() != null + ? parent.getGroups().stream().map(Urn::toString).collect(Collectors.toSet()) + : Collections.emptySet(); + + // First check whether user is directly assigned. + if (assignedUserUrns.size() > 0) { + boolean isUserAssigned = assignedUserUrns.contains(userUrn.toString()); + if (isUserAssigned) { + return true; + } + } + + // Next check whether the user is assigned indirectly, by group. + if (assignedGroupUrns.size() > 0) { + boolean isUserGroupAssigned = + groupUrns.stream().anyMatch(groupUrn -> assignedGroupUrns.contains(groupUrn.toString())); + if (isUserGroupAssigned) { + return true; + } + } + + return false; + } +} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java index fc28367e6c7ee..ae061a2d0c090 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java @@ -1,12 +1,17 @@ package com.linkedin.metadata.utils; import com.datahub.util.RecordUtils; +import com.linkedin.common.urn.Urn; import com.linkedin.data.ByteString; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.GenericPayload; import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; import javax.annotation.Nonnull; public class GenericRecordUtils { @@ -66,4 +71,20 @@ public static GenericPayload serializePayload(@Nonnull RecordTemplate payload) { genericPayload.setContentType(GenericRecordUtils.JSON); return genericPayload; } + + @Nonnull + public static Map> entityResponseToAspectMap( + Map inputMap) { + return inputMap.entrySet().stream() + .map( + entry -> + Map.entry( + entry.getKey(), + entry.getValue().getAspects().entrySet().stream() + .map( + aspectEntry -> + Map.entry(aspectEntry.getKey(), aspectEntry.getValue().getValue())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } } diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SchemaFieldUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SchemaFieldUtils.java new file mode 100644 index 0000000000000..edf959d04a37b --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SchemaFieldUtils.java @@ -0,0 +1,22 @@ +package com.linkedin.metadata.utils; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.key.SchemaFieldKey; +import javax.annotation.Nonnull; + +public class SchemaFieldUtils { + + private SchemaFieldUtils() {} + + public static Urn generateSchemaFieldUrn( + @Nonnull final String resourceUrn, @Nonnull final String fieldPath) { + // we rely on schemaField fieldPaths to be encoded since we do that on the ingestion side + final String encodedFieldPath = + fieldPath.replaceAll("\\(", "%28").replaceAll("\\)", "%29").replaceAll(",", "%2C"); + final SchemaFieldKey key = + new SchemaFieldKey().setParent(UrnUtils.getUrn(resourceUrn)).setFieldPath(encodedFieldPath); + return EntityKeyUtils.convertEntityKeyToUrn(key, Constants.SCHEMA_FIELD_ENTITY_NAME); + } +} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java index eb58bc509838d..9df708c6e9fdc 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/SearchUtil.java @@ -7,14 +7,19 @@ import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; import com.linkedin.metadata.search.FilterValue; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -142,4 +147,25 @@ public static BoolQueryBuilder filterSoftDeletedByDefault( } return filterQuery; } + + public static SortCriterion sortBy(@Nonnull String field, @Nullable SortOrder direction) { + SortCriterion sortCriterion = new SortCriterion(); + sortCriterion.setField(field); + sortCriterion.setOrder( + com.linkedin.metadata.query.filter.SortOrder.valueOf( + Optional.ofNullable(direction).orElse(SortOrder.ASCENDING).toString())); + return sortCriterion; + } + + public static Filter andFilter(Criterion... criteria) { + Filter filter = new Filter(); + filter.setOr(andCriterion(Arrays.stream(criteria))); + return filter; + } + + public static ConjunctiveCriterionArray andCriterion(Stream criteria) { + return new ConjunctiveCriterionArray( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(criteria.collect(Collectors.toList())))); + } } diff --git a/mock-entity-registry/src/main/java/mock/MockEntityRegistry.java b/mock-entity-registry/src/main/java/mock/MockEntityRegistry.java index a324f9ce0195b..dfa8c627e0617 100644 --- a/mock-entity-registry/src/main/java/mock/MockEntityRegistry.java +++ b/mock-entity-registry/src/main/java/mock/MockEntityRegistry.java @@ -1,10 +1,10 @@ package mock; +import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.EventSpec; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.models.registry.template.AspectTemplateEngine; import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh index 93f03d36cbd19..b1c6571e1a065 100755 --- a/smoke-test/cypress-dev.sh +++ b/smoke-test/cypress-dev.sh @@ -15,7 +15,7 @@ python -c 'from tests.cypress.integration_test import ingest_data; ingest_data() cd tests/cypress npm install -source ../../set-cypress-creds.sh +source "$DIR/set-cypress-creds.sh" npx cypress open \ --env "ADMIN_DISPLAYNAME=$CYPRESS_ADMIN_DISPLAYNAME,ADMIN_USERNAME=$CYPRESS_ADMIN_USERNAME,ADMIN_PASSWORD=$CYPRESS_ADMIN_PASSWORD" diff --git a/smoke-test/requests_wrapper/__init__.py b/smoke-test/requests_wrapper/__init__.py index d9956e8434a89..c2f4190e6150d 100644 --- a/smoke-test/requests_wrapper/__init__.py +++ b/smoke-test/requests_wrapper/__init__.py @@ -1,3 +1,4 @@ from .utils_requests_wrapper import CustomSession as Session from .utils_requests_wrapper import get, post from .constants import * +from requests import exceptions diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js b/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js index cb67efe00b484..843a15d7430af 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/homePagePost.js @@ -1,65 +1,85 @@ -const title = 'Test Link Title' -const url = 'https://www.example.com' -const imagesURL = 'https://www.example.com/images/example-image.jpg' - const homePageRedirection = () => { - cy.visit('/') - cy.waitTextPresent("Welcome back,") -} + cy.visit('/'); + cy.waitTextPresent("Welcome back"); +}; -const addAnnouncement = () => { - cy.get('[id="posts-create-post"]').click({ force: true }); - cy.waitTextPresent('Create new Post') - cy.enterTextInTestId("create-post-title", "Test Announcement Title"); - cy.get('[id="description"]').type("Add Description to post announcement") - cy.get('[data-testid="create-post-button"]').click({ force: true }); - cy.reload() +const addOrEditAnnouncement = (text, title, description, testId) => { + cy.waitTextPresent(text); + cy.get('[data-testid="create-post-title"]').clear().type(title); + cy.get('[id="description"]').clear().type(description); + cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); + cy.reload(); homePageRedirection(); - cy.waitTextPresent("Test Announcement Title"); -} +}; -const addLink = (title,url,imagesURL) => { - cy.get('[id="posts-create-post"]').click({ force: true }); - cy.waitTextPresent('Create new Post') - cy.clickOptionWithText('Link'); - cy.enterTextInTestId('create-post-title', title); - cy.enterTextInTestId('create-post-link', url); - cy.enterTextInTestId('create-post-media-location', imagesURL) - cy.get('[data-testid="create-post-button"]').click({ force: true }); - cy.reload() +const addOrEditLink = (text, title, url, imagesURL, testId) => { + cy.waitTextPresent(text); + cy.get('[data-testid="create-post-title"]').clear().type(title); + cy.get('[data-testid="create-post-link"]').clear().type(url); + cy.get('[data-testid="create-post-media-location"]').clear().type(imagesURL); + cy.get(`[data-testid="${testId}-post-button"]`).click({ force: true }); + cy.reload(); homePageRedirection(); - cy.waitTextPresent(title) +}; + +const clickOnNewPost = () =>{ + cy.get('[id="posts-create-post"]').click({ force: true }); } -const deleteFromPostDropdown = () => { - cy.get('[aria-label="more"]').first().click() - cy.clickOptionWithText("Delete"); - cy.clickOptionWithText("Yes"); - cy.reload() - homePageRedirection(); +const clickOnMoreOption = () => { + cy.get('[aria-label="more"]').first().click(); } -describe("Create announcement and link posts", () => { +describe("create announcement and link post", () => { beforeEach(() => { cy.loginWithCredentials(); cy.goToHomePagePostSettings(); }); - it("Create and Verify Announcement Post", () => { - addAnnouncement(); - }) + it("create announcement post and verify", () => { + clickOnNewPost() + addOrEditAnnouncement("Create new Post", "Test Announcement Title", "Add Description to post announcement", "create"); + cy.waitTextPresent("Test Announcement Title"); + }); - it("Delete and Verify Announcement Post", () => { - deleteFromPostDropdown(); - cy.ensureTextNotPresent("Test Announcement Title") - }) + it("edit announced post and verify", () => { + clickOnMoreOption() + cy.clickOptionWithText("Edit"); + addOrEditAnnouncement("Edit Post", "Test Announcement Title Edited", "Decription Edited", "update"); + cy.waitTextPresent("Test Announcement Title Edited"); + }); + + it("delete announced post and verify", () => { + clickOnMoreOption() + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.reload(); + homePageRedirection(); + cy.ensureTextNotPresent("Test Announcement Title Edited"); + }); + + it("create link post and verify", () => { + clickOnNewPost() + cy.waitTextPresent('Create new Post'); + cy.contains('label', 'Link').click(); + addOrEditLink("Create new Post", "Test Link Title", 'https://www.example.com', 'https://www.example.com/images/example-image.jpg', "create"); + cy.waitTextPresent("Test Link Title"); + }); + + it("edit linked post and verify", () => { + clickOnMoreOption() + cy.clickOptionWithText("Edit"); + addOrEditLink("Edit Post", "Test Link Edited Title", 'https://www.updatedexample.com', 'https://www.updatedexample.com/images/example-image.jpg', "update"); + cy.waitTextPresent("Test Link Edited Title"); + }); - it("Create and Verify Link Post", () => { - addLink(title,url,imagesURL) - }) + it("delete linked post and verify", () => { + clickOnMoreOption() + cy.clickOptionWithText("Delete"); + cy.clickOptionWithText("Yes"); + cy.reload(); + homePageRedirection(); + cy.ensureTextNotPresent("Test Link Edited Title"); + }); +}); - it("Delete and Verify Link Post", () => { - deleteFromPostDropdown(); - cy.ensureTextNotPresent(title); - }) -}) \ No newline at end of file diff --git a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js index 978a245c3d9e3..8421bd288edf0 100644 --- a/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js +++ b/smoke-test/tests/cypress/cypress/e2e/settings/managing_groups.js @@ -106,10 +106,19 @@ describe("create and manage group", () => { cy.waitTextVisible(username); }); + it("assign role to group ", () => { + cy.loginWithCredentials(); + cy.visit("/settings/identities/groups"); + cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).next().click() + cy.get('.ant-select-item-option').contains('Admin').click() + cy.get('button.ant-btn-primary').contains('OK').click(); + cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).waitTextVisible('Admin'); + }); + it("remove group", () => { cy.loginWithCredentials(); cy.visit("/settings/identities/groups"); - cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).next().click(); + cy.get(`[href="/group/urn:li:corpGroup:${test_id}"]`).openThreeDotDropdown() cy.clickOptionWithText("Delete"); cy.clickOptionWithText("Yes"); cy.waitTextVisible("Deleted Group!"); diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js index 00de08e77a185..f89b70b7a7d23 100644 --- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js +++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js @@ -80,7 +80,7 @@ describe('siblings', () => { cy.login(); cy.visit('/search?page=1&query=raw_orders'); - cy.contains('Showing 1 - 10 of 14 results'); + cy.contains('Showing 1 - 10 of '); cy.get('.test-search-result').should('have.length', 5); cy.get('.test-search-result-sibling-section').should('have.length', 5); diff --git a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py index b5e408731334e..6d95f731f32b1 100644 --- a/smoke-test/tests/managed-ingestion/managed_ingestion_test.py +++ b/smoke-test/tests/managed-ingestion/managed_ingestion_test.py @@ -260,6 +260,27 @@ def test_create_list_get_remove_secret(frontend_session): # Get new count of secrets _ensure_secret_increased(frontend_session, before_count) + # Update existing secret + json_q = { + "query": """mutation updateSecret($input: UpdateSecretInput!) {\n + updateSecret(input: $input) + }""", + "variables": {"input": {"urn": secret_urn, "name": "SMOKE_TEST", "value": "mytestvalue.updated"}}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=json_q + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["updateSecret"] is not None + assert "errors" not in res_data + + secret_urn = res_data["data"]["updateSecret"] + # Get the secret value back json_q = { "query": """query getSecretValues($input: GetSecretValuesInput!) {\n @@ -285,7 +306,7 @@ def test_create_list_get_remove_secret(frontend_session): secret_values = res_data["data"]["getSecretValues"] secret_value = [x for x in secret_values if x["name"] == "SMOKE_TEST"][0] - assert secret_value["value"] == "mytestvalue" + assert secret_value["value"] == "mytestvalue.updated" # Now cleanup and remove the secret json_q = { diff --git a/smoke-test/tests/structured_properties/__init__.py b/smoke-test/tests/structured_properties/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/smoke-test/tests/structured_properties/click_event.avsc b/smoke-test/tests/structured_properties/click_event.avsc new file mode 100644 index 0000000000000..d959dcbbdeea1 --- /dev/null +++ b/smoke-test/tests/structured_properties/click_event.avsc @@ -0,0 +1,14 @@ +{ + "namespace": "io.datahubproject", + "type": "record", + "name": "ClickEvent", + "fields": [ + { "name": "ip", "type": "string" }, + { "name": "url", "type": "string" }, + { "name": "time", "type": "long" }, + { "name": "referer", "type": ["string", "null"] }, + { "name": "user_agent", "type": ["string", "null"] }, + { "name": "user_id", "type": ["string", "null"] }, + { "name": "session_id", "type": ["string", "null"] } + ] +} diff --git a/smoke-test/tests/structured_properties/test_dataset.yaml b/smoke-test/tests/structured_properties/test_dataset.yaml new file mode 100644 index 0000000000000..2ac1cca6c6dc2 --- /dev/null +++ b/smoke-test/tests/structured_properties/test_dataset.yaml @@ -0,0 +1,19 @@ +- id: user.clicks + platform: hive + # urn: urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD) # use urn instead of id and platform + subtype: table + schema: + file: tests/structured_properties/click_event.avsc + fields: + # - id: ip + - urn: urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD),ip) + structured_properties: + io.acryl.dataManagement.deprecationDate: "2023-01-01" + properties: + retention: 365 + structured_properties: + clusterType: primary + clusterName: gold + projectNames: + - Tracking + - DataHub diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py new file mode 100644 index 0000000000000..83994776076b0 --- /dev/null +++ b/smoke-test/tests/structured_properties/test_structured_properties.py @@ -0,0 +1,577 @@ +import logging +import os +from datahub.ingestion.graph.filters import SearchFilterRule +from tests.consistency_utils import wait_for_writes_to_sync +import tempfile +from random import randint +from tests.utilities.file_emitter import FileEmitter +from typing import Iterable, List, Optional, Union + +import pytest +# import tenacity +from datahub.api.entities.dataset.dataset import Dataset +from datahub.api.entities.structuredproperties.structuredproperties import \ + StructuredProperties +from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.metadata.schema_classes import ( + EntityTypeInfoClass, PropertyValueClass, StructuredPropertiesClass, + StructuredPropertyDefinitionClass, StructuredPropertyValueAssignmentClass) +from datahub.specific.dataset import DatasetPatchBuilder +from datahub.utilities.urns.structured_properties_urn import \ + StructuredPropertyUrn +from datahub.utilities.urns.urn import Urn + +from tests.utils import (delete_urns, delete_urns_from_file, get_gms_url, + get_sleep_info, ingest_file_via_rest, + wait_for_writes_to_sync) + +logger = logging.getLogger(__name__) + +start_index = randint(10, 10000) +dataset_urns = [ + make_dataset_urn("snowflake", f"table_foo_{i}") + for i in range(start_index, start_index + 10) +] + +schema_field_urns = [ + make_schema_field_urn(dataset_urn, "column_1") + for dataset_urn in dataset_urns +] + +generated_urns = [d for d in dataset_urns] + [f for f in schema_field_urns] + + +default_namespace = "io.acryl.privacy" + +def create_logical_entity( + entity_name: str, +) -> Iterable[MetadataChangeProposalWrapper]: + mcp = MetadataChangeProposalWrapper( + entityUrn="urn:li:entityType:" + entity_name, + aspect=EntityTypeInfoClass( + qualifiedName="io.datahubproject." + entity_name, + displayName=entity_name, + ), + ) + return [mcp] + + +def create_test_data(filename: str): + file_emitter = FileEmitter(filename) + for mcps in create_logical_entity("dataset"): + file_emitter.emit(mcps) + + file_emitter.close() + wait_for_writes_to_sync() + +sleep_sec, sleep_times = get_sleep_info() + + +@pytest.fixture(scope="module", autouse=False) +def graph() -> DataHubGraph: + graph: DataHubGraph = DataHubGraph( + config=DatahubClientConfig(server=get_gms_url()) + ) + return graph + + +@pytest.fixture(scope="module", autouse=False) +def ingest_cleanup_data(request): + new_file, filename = tempfile.mkstemp() + try: + create_test_data(filename) + print("ingesting structured properties test data") + ingest_file_via_rest(filename) + yield + print("removing structured properties test data") + delete_urns_from_file(filename) + delete_urns(generated_urns) + wait_for_writes_to_sync() + finally: + os.remove(filename) + + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +def create_property_definition( + property_name: str, + graph: DataHubGraph, + namespace: str = default_namespace, + value_type: str = "string", + cardinality: str = "SINGLE", + allowed_values: Optional[List[PropertyValueClass]] = None, + entity_types: Optional[List[str]] = None, +): + structured_property_definition = StructuredPropertyDefinitionClass( + qualifiedName=f"{namespace}.{property_name}", + valueType=Urn.make_data_type_urn(value_type), + description="The retention policy for the dataset", + entityTypes=[Urn.make_entity_type_urn(e) for e in entity_types] + if entity_types + else [Urn.make_entity_type_urn("dataset")], + cardinality=cardinality, + allowedValues=allowed_values, + ) + + mcp = MetadataChangeProposalWrapper( + entityUrn=f"urn:li:structuredProperty:{namespace}.{property_name}", + aspect=structured_property_definition, + ) + graph.emit(mcp) + wait_for_writes_to_sync() + + +def attach_property_to_entity( + urn: str, + property_name: str, + property_value: Union[str, float, List[str | float]], + graph: DataHubGraph, + namespace: str = default_namespace +): + if isinstance(property_value, list): + property_values: List[Union[str, float]] = property_value + else: + property_values = [property_value] + + mcp = MetadataChangeProposalWrapper( + entityUrn=urn, + aspect=StructuredPropertiesClass( + properties=[ + StructuredPropertyValueAssignmentClass( + propertyUrn=f"urn:li:structuredProperty:{namespace}.{property_name}", + values=property_values, + ) + ] + ), + ) + graph.emit_mcp(mcp) + wait_for_writes_to_sync() + + +def get_property_from_entity( + urn: str, + property_name: str, + graph: DataHubGraph, +): + structured_properties: Optional[ + StructuredPropertiesClass + ] = graph.get_aspect(urn, StructuredPropertiesClass) + assert structured_properties is not None + for property in structured_properties.properties: + if ( + property.propertyUrn + == f"urn:li:structuredProperty:{property_name}" + ): + return property.values + return None + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), +# wait=tenacity.wait_fixed(sleep_sec), +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_string(ingest_cleanup_data, graph): + property_name = "retentionPolicy" + + create_property_definition(property_name, graph) + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.retentionPolicy") + + attach_property_to_entity( + dataset_urns[0], property_name, ["30d"], graph=graph + ) + + try: + attach_property_to_entity( + dataset_urns[0], property_name, 200030, graph=graph + ) + raise AssertionError( + "Should not be able to attach a number to a string property" + ) + except Exception as e: + if not isinstance(e, AssertionError): + pass + else: + raise e + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), +# wait=tenacity.wait_fixed(sleep_sec), +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_double(ingest_cleanup_data, graph): + property_name = "expiryTime" + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.{property_name}") + create_property_definition(property_name, graph, value_type="number") + + attach_property_to_entity( + dataset_urns[0], property_name, 2000034, graph=graph + ) + + try: + attach_property_to_entity( + dataset_urns[0], property_name, "30 days", graph=graph + ) + raise AssertionError( + "Should not be able to attach a string to a number property" + ) + except Exception as e: + if not isinstance(e, AssertionError): + pass + else: + raise e + + try: + attach_property_to_entity( + dataset_urns[0], property_name, [2000034, 2000035], graph=graph + ) + raise AssertionError( + "Should not be able to attach a list to a number property" + ) + except Exception as e: + if not isinstance(e, AssertionError): + pass + else: + raise e + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), +# wait=tenacity.wait_fixed(sleep_sec), +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_double_multiple(ingest_cleanup_data, graph): + property_name = "versions" + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.{property_name}") + + create_property_definition( + property_name, graph, value_type="number", cardinality="MULTIPLE" + ) + + attach_property_to_entity( + dataset_urns[0], property_name, [1.0, 2.0], graph=graph + ) + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), +# wait=tenacity.wait_fixed(sleep_sec), +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_string_allowed_values( + ingest_cleanup_data, graph +): + property_name = "enumProperty" + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.{property_name}") + + create_property_definition( + property_name, + graph, + value_type="string", + cardinality="MULTIPLE", + allowed_values=[ + PropertyValueClass(value="foo"), + PropertyValueClass(value="bar"), + ], + ) + + attach_property_to_entity( + dataset_urns[0], property_name, ["foo", "bar"], graph=graph + ) + + try: + attach_property_to_entity( + dataset_urns[0], property_name, ["foo", "baz"], graph=graph + ) + raise AssertionError( + "Should not be able to attach a value not in allowed values" + ) + except Exception as e: + if "value: {string=baz} should be one of [" in str(e): + pass + else: + raise e + + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_definition_evolution( + ingest_cleanup_data, graph +): + property_name = "enumProperty1234" + + create_property_definition( + property_name, + graph, + value_type="string", + cardinality="MULTIPLE", + allowed_values=[ + PropertyValueClass(value="foo"), + PropertyValueClass(value="bar"), + ], + ) + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.{property_name}") + + try: + create_property_definition( + property_name, + graph, + value_type="string", + cardinality="SINGLE", + allowed_values=[ + PropertyValueClass(value="foo"), + PropertyValueClass(value="bar"), + ], + ) + raise AssertionError( + "Should not be able to change cardinality from MULTIPLE to SINGLE" + ) + except Exception as e: + if isinstance(e, AssertionError): + raise e + else: + pass + + +# @tenacity.retry( +# stop=tenacity.stop_after_attempt(sleep_times), +# wait=tenacity.wait_fixed(sleep_sec), +# ) +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_structured_property_schema_field(ingest_cleanup_data, graph): + property_name = ( + f"deprecationDate{randint(10, 10000)}" + ) + + create_property_definition( + property_name, + graph, + namespace="io.datahubproject.test", + value_type="date", + entity_types=["schemaField"], + ) + generated_urns.append(f"urn:li:structuredProperty:io.datahubproject.test.{property_name}") + + attach_property_to_entity( + schema_field_urns[0], property_name, "2020-10-01", graph=graph, namespace="io.datahubproject.test" + ) + + assert ( + get_property_from_entity( + schema_field_urns[0], f"io.datahubproject.test.{property_name}", graph=graph + ) + == ["2020-10-01"] + ) + + try: + attach_property_to_entity( + schema_field_urns[0], property_name, 200030, graph=graph, namespace="io.datahubproject.test" + ) + raise AssertionError( + "Should not be able to attach a number to a DATE property" + ) + except Exception as e: + if not isinstance(e, AssertionError): + pass + else: + raise e + + +def test_dataset_yaml_loader(ingest_cleanup_data, graph): + StructuredProperties.create( + "tests/structured_properties/test_structured_properties.yaml" + ) + + for dataset in Dataset.from_yaml( + "tests/structured_properties/test_dataset.yaml" + ): + for mcp in dataset.generate_mcp(): + graph.emit(mcp) + wait_for_writes_to_sync() + + property_name = "io.acryl.dataManagement.deprecationDate" + assert ( + get_property_from_entity( + make_schema_field_urn( + make_dataset_urn("hive", "user.clicks"), "ip" + ), + property_name, + graph=graph, + ) + == ["2023-01-01"] + ) + + dataset = Dataset.from_datahub( + graph=graph, + urn="urn:li:dataset:(urn:li:dataPlatform:hive,user.clicks,PROD)", + ) + field_name = "ip" + matching_fields = [ + f + for f in dataset.schema_metadata.fields + if Dataset._simplify_field_path(f.id) == field_name + ] + assert len(matching_fields) == 1 + assert ( + matching_fields[0].structured_properties[ + Urn.make_structured_property_urn( + "io.acryl.dataManagement.deprecationDate" + ) + ] + == ["2023-01-01"] + ) + + +def test_dataset_structured_property_validation( + ingest_cleanup_data, graph, caplog +): + from datahub.api.entities.dataset.dataset import Dataset + + property_name = "replicationSLA" + property_value = 30 + value_type = "number" + + create_property_definition( + property_name=property_name, graph=graph, value_type=value_type + ) + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.replicationSLA") + + attach_property_to_entity( + dataset_urns[0], property_name, [property_value], graph=graph + ) + + assert Dataset.validate_structured_property( + f"{default_namespace}.{property_name}", property_value + ) == ( + f"{default_namespace}.{property_name}", + float(property_value), + ) + + assert ( + Dataset.validate_structured_property("testName", "testValue") is None + ) + + bad_property_value = "2023-09-20" + assert ( + Dataset.validate_structured_property( + property_name, bad_property_value + ) + is None + ) + + +def test_structured_property_search(ingest_cleanup_data, graph: DataHubGraph, caplog): + + def to_es_name(property_name, namespace=default_namespace): + namespace_field = namespace.replace(".", "_") + return f"structuredProperties.{namespace_field}_{property_name}" + + # Attach structured property to entity and to field + field_property_name = f"deprecationDate{randint(10, 10000)}" + + create_property_definition( + namespace="io.datahubproject.test", + property_name=field_property_name, + graph=graph, value_type="date", entity_types=["schemaField"] + ) + generated_urns.append(f"urn:li:structuredProperty:io.datahubproject.test.{field_property_name}") + + attach_property_to_entity( + schema_field_urns[0], field_property_name, "2020-10-01", graph=graph, namespace="io.datahubproject.test" + ) + dataset_property_name = "replicationSLA" + property_value = 30 + value_type = "number" + + create_property_definition(property_name=dataset_property_name, graph=graph, value_type=value_type) + generated_urns.append(f"urn:li:structuredProperty:{default_namespace}.{dataset_property_name}") + + attach_property_to_entity(dataset_urns[0], dataset_property_name, [property_value], graph=graph) + + # [] = default entities which includes datasets, does not include fields + entity_urns = list(graph.get_urns_by_filter(extraFilters=[ + { + "field": to_es_name(dataset_property_name), + "negated": "false", + "condition": "EXISTS", + } + ])) + assert len(entity_urns) == 1 + assert entity_urns[0] == dataset_urns[0] + + # Search over schema field specifically + field_structured_prop = graph.get_aspect(entity_urn=schema_field_urns[0], aspect_type=StructuredPropertiesClass) + assert field_structured_prop == StructuredPropertiesClass( + properties=[ + StructuredPropertyValueAssignmentClass( + propertyUrn=f"urn:li:structuredProperty:io.datahubproject.test.{field_property_name}", + values=["2020-10-01"] + ) + ] + ) + + # Search over entities that do not include the field + field_urns = list(graph.get_urns_by_filter(entity_types=["tag"], + extraFilters=[ + { + "field": to_es_name(field_property_name, + namespace="io.datahubproject.test"), + "negated": "false", + "condition": "EXISTS", + } + ])) + assert len(field_urns) == 0 + + # OR the two properties together to return both results + field_urns = list(graph.get_urns_by_filter(entity_types=["dataset", "tag"], + extraFilters=[ + { + "field": to_es_name(dataset_property_name), + "negated": "false", + "condition": "EXISTS", + } + ])) + assert len(field_urns) == 1 + assert dataset_urns[0] in field_urns + + +def test_dataset_structured_property_patch( + ingest_cleanup_data, graph, caplog +): + property_name = "replicationSLA" + property_value = 30 + value_type = "number" + + create_property_definition( + property_name=property_name, + graph=graph, + value_type=value_type + ) + + dataset_patcher: DatasetPatchBuilder = DatasetPatchBuilder( + urn=dataset_urns[0] + ) + + dataset_patcher.set_structured_property(StructuredPropertyUrn.make_structured_property_urn( + f"{default_namespace}.{property_name}"), property_value) + + for mcp in dataset_patcher.build(): + graph.emit(mcp) + wait_for_writes_to_sync() + + dataset = Dataset.from_datahub(graph=graph, urn=dataset_urns[0]) + assert dataset.structured_properties is not None + assert ( + [int(float(k)) for k in dataset.structured_properties[ + StructuredPropertyUrn.make_structured_property_urn( + f"{default_namespace}.{property_name}" + ) + ]] + == [property_value] + ) diff --git a/smoke-test/tests/structured_properties/test_structured_properties.yaml b/smoke-test/tests/structured_properties/test_structured_properties.yaml new file mode 100644 index 0000000000000..569a3d185165d --- /dev/null +++ b/smoke-test/tests/structured_properties/test_structured_properties.yaml @@ -0,0 +1,33 @@ +- id: clusterType + type: STRING + display_name: Cluster's type + description: "Test Cluster Type Property" + entity_types: + - dataset +- id: clusterName + type: STRING + display_name: Cluster's name + description: "Test Cluster Name Property" + entity_types: + - dataset +- id: projectNames + type: STRING + cardinality: MULTIPLE + display_name: Project Name + entity_types: + - dataset # or urn:li:logicalEntity:metamodel.datahub.dataset + - dataflow + description: "Test property for project name" + allowed_values: + - value: Tracking + description: test value 1 for project + - value: DataHub + description: test value 2 for project +- id: io.acryl.dataManagement.deprecationDate + type: DATE + display_name: Deprecation Date + entity_types: + - dataset + - dataFlow + - dataJob + - schemaField diff --git a/smoke-test/tests/telemetry/telemetry_test.py b/smoke-test/tests/telemetry/telemetry_test.py index 3127061c9f506..b7cd6fa0517df 100644 --- a/smoke-test/tests/telemetry/telemetry_test.py +++ b/smoke-test/tests/telemetry/telemetry_test.py @@ -3,9 +3,19 @@ from datahub.cli.cli_utils import get_aspects_for_entity -def test_no_clientID(): +def test_no_client_id(): client_id_urn = "urn:li:telemetry:clientId" - aspect = ["telemetryClientId"] + aspect = ["clientId"] # this is checking for the removal of the invalid aspect RemoveClientIdAspectStep.java + + res_data = json.dumps( + get_aspects_for_entity(entity_urn=client_id_urn, aspects=aspect, typed=False) + ) + assert res_data == "{}" + + +def test_no_telemetry_client_id(): + client_id_urn = "urn:li:telemetry:clientId" + aspect = ["telemetryClientId"] # telemetry expected to be disabled for tests res_data = json.dumps( get_aspects_for_entity(entity_urn=client_id_urn, aspects=aspect, typed=False) diff --git a/smoke-test/tests/utilities/__init__.py b/smoke-test/tests/utilities/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/smoke-test/tests/utilities/file_emitter.py b/smoke-test/tests/utilities/file_emitter.py new file mode 100644 index 0000000000000..27a91c360af8a --- /dev/null +++ b/smoke-test/tests/utilities/file_emitter.py @@ -0,0 +1,21 @@ +from datahub.ingestion.sink.file import FileSink, FileSinkConfig +from datahub.ingestion.api.common import PipelineContext, RecordEnvelope +from datahub.ingestion.api.sink import NoopWriteCallback +import time + + +class FileEmitter: + def __init__(self, filename: str, run_id: str = f"test_{int(time.time()*1000.0)}") -> None: + self.sink: FileSink = FileSink( + ctx=PipelineContext(run_id=run_id), + config=FileSinkConfig(filename=filename), + ) + + def emit(self, event): + self.sink.write_record_async( + record_envelope=RecordEnvelope(record=event, metadata={}), + write_callback=NoopWriteCallback(), + ) + + def close(self): + self.sink.close() \ No newline at end of file