Merge branch 'datahub-project:master' into master

datahub-frontend/app/controllers/SsoCallbackController.java

@@ -66,7 +66,8 @@ public SsoCallbackController(
 
   public CompletionStage<Result> handleCallback(String protocol, Http.Request request) {
     if (shouldHandleCallback(protocol)) {
-      log.debug(String.format("Handling SSO callback. Protocol: %s", protocol));
+      log.debug("Handling SSO callback. Protocol: {}",
+        _ssoManager.getSsoProvider().protocol().getCommonName());
       return callback(request)
           .handle(
               (res, e) -> {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java

@@ -41,7 +41,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
           final QueryContext context = environment.getContext();
           final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class);
 
-          log.info("User {} revoking access token {}", context.getActorUrn(), tokenId);
+          log.info("User {} revoking access token", context.getActorUrn());
 
           if (isAuthorizedToRevokeToken(context, tokenId)) {
             try {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java

@@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) {
   public CompletableFuture<String> get(final DataFetchingEnvironment environment) throws Exception {
     final QueryContext context = environment.getContext();
     if (IngestionAuthUtils.canManageSecrets(context)) {
-      final String secretUrn = environment.getArgument("urn");
-      final Urn urn = Urn.createFromString(secretUrn);
+      final String inputUrn = environment.getArgument("urn");
+      final Urn urn = Urn.createFromString(inputUrn);
       return GraphQLConcurrencyUtils.supplyAsync(
           () -> {
             try {
               _entityClient.deleteEntity(context.getOperationContext(), urn);
-              return secretUrn;
+              return inputUrn;
             } catch (Exception e) {
               throw new RuntimeException(
-                  String.format("Failed to perform delete against secret with urn %s", secretUrn),
+                  String.format("Failed to perform delete against secret with urn %s", inputUrn),
                   e);
             }
           },

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java

@@ -25,7 +25,7 @@ static String encrypt(String value, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.ENCRYPT_MODE, secretKey);
       return Base64.getEncoder()
           .encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8)));
@@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.DECRYPT_MODE, secretKey);
       return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue)));
     } catch (Exception e) {

docs-website/src/pages/cloud/CompanyLogos/customersData.json

@@ -2,38 +2,50 @@
   "customers": [
     {
       "link": {
-        "href": "https://robinhood.com",
+        "href": "https://www.depop.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/acertus.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/345d124a249d43bf0f20b608f8bfa2f7683311fa-360x180.png"
         },
-        "alt": "Robinhood"
+        "alt": "depop"
       }
     },
     {
       "link": {
-        "href": "https://www.dpgmediagroup.com/",
+        "href": "https://riskified.com",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/autoscout24.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c982e0459bed565273a9b696d9d40aed76f84b1e-360x180.png"
         },
-        "alt": "DPG Media"
+        "alt": "Riskified"
       }
     },
     {
       "link": {
-        "href": "https://www.twilio.com",
+        "href": "https://get.betterup.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/betterup.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5988a55b3c090a12ddc3f3cae07b290ac3134771-360x180.png"
         },
-        "alt": "Twilio"
+        "alt": "Betterup"
+      }
+    },
+    {
+      "link": {
+        "href": "https://www.ovoenergy.com/",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5e7bd32dfbc769849dca136947ebd2fc2f5e91f3-540x270.png"
+        },
+        "alt": "OVO Energy"
       }
     },
     {
@@ -43,45 +55,69 @@
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/depop.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/3d7c10e1bd7c7a062250e092d6d9d0553fb57790-360x180.png"
         },
         "alt": "Myob"
       }
     },
     {
       "link": {
-        "href": "https://regeneron.com",
+        "href": "https://www.dpgmediagroup.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/dpg_media.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png"
         },
-        "alt": "Regeneron"
+        "alt": "DPG Megia"
       }
     },
     {
       "link": {
-        "href": "https://riskified.com",
+        "href": "https://www.notion.so/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/myob.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c2e84f93572cd1baf30ea7ab8da234ff44182eb6-540x270.png"
         },
-        "alt": "Riskified"
+        "alt": "Notion"
+      }
+    },
+    {
+      "link": {
+        "href": "https://www.sae.org/",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/a9e8586635cb4039cbfc5836a6a5cacdeba9e6b3-540x270.png"
+        },
+        "alt": "SAE International"
+      }
+    },
+    {
+      "link": {
+        "href": "https://viator.com",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/0c17141cad5baa053da18dffb17d9c182d242e69-1200x475.png"
+        },
+        "alt": "Viator"
       }
     },
     {
       "link": {
-        "href": "https://xero.com",
+        "href": "https://www.tripadvisor.co.uk/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/notion.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/28255a28d261a074a83d1ee8632f0338bf5cf57e-1112x256.png"
         },
-        "alt": "Xero"
+        "alt": "Trip Advisor"
       }
     }
   ]

docs-website/src/pages/cloud/CompanyLogos/index.js

@@ -32,7 +32,7 @@ const ScrollingCustomers = ({ noOverlay = true, spacing, ...rest }) => {
             to={customer.link.href}
             target={customer.link.blank ? '_blank' : '_self'}
             rel={customer.link.blank ? 'noopener noreferrer' : ''}
-            style={{ minWidth: 'max-content', padding: '0 3.25rem' }}
+            style={{ minWidth: 'max-content', padding: '0 1.8rem' }}
           >
             <img
               src={customer.logo.asset._ref}

docs-website/src/pages/cloud/CompanyLogos/logos.module.scss

@@ -33,7 +33,7 @@
 }
 
 .scrollingCustomers__inner img {
-  max-width: 100px!important;
+  max-width: 160px !important;
   min-width: unset;
   filter: invert(1) brightness(0) contrast(100); // make image black
 }

entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java

@@ -63,7 +63,7 @@ public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
         } else {
           List<Object> valueList = (List<Object>) value.get();
           // If the field is a nested list of values, flatten it
-          for (int i = 0; i < numArrayWildcards - 1; i++) {
+          for (long i = 0; i < numArrayWildcards - 1; i++) {
             valueList =
                 valueList.stream()
                     .flatMap(v -> ((List<Object>) v).stream())

metadata-ingestion/src/datahub/cli/cli_utils.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import time
 import typing
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
@@ -403,6 +404,8 @@ def ensure_has_system_metadata(
     if event.systemMetadata is None:
         event.systemMetadata = SystemMetadataClass()
     metadata = event.systemMetadata
+    if metadata.lastObserved == 0:
+        metadata.lastObserved = int(time.time() * 1000)
     if metadata.properties is None:
         metadata.properties = {}
     props = metadata.properties

metadata-ingestion/src/datahub/ingestion/source/abs/source.py

@@ -198,11 +198,6 @@ class TableData:
 @support_status(SupportStatus.INCUBATING)
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 @capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
-@capability(
-    SourceCapability.DELETION_DETECTION,
-    "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
-    supported=True,
-)
 class ABSSource(StatefulIngestionSourceBase):
     source_config: DataLakeSourceConfig
     report: DataLakeSourceReport

metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py

@@ -12,11 +12,12 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source_config.csv_enricher import CSVEnricherConfig
 from datahub.metadata.schema_classes import (
@@ -96,6 +97,10 @@ class CSVEnricherReport(SourceReport):
 @platform_name("CSV Enricher")
 @config_class(CSVEnricherConfig)
 @support_status(SupportStatus.INCUBATING)
+@capability(SourceCapability.DOMAINS, "Supported by default")
+@capability(SourceCapability.TAGS, "Supported by default")
+@capability(SourceCapability.DESCRIPTIONS, "Supported by default")
+@capability(SourceCapability.OWNERSHIP, "Supported by default")
 class CSVEnricherSource(Source):
     """
     :::tip Looking to ingest a CSV data file into DataHub, as an asset?

metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py

@@ -166,11 +166,6 @@ def report_dropped(self, name: str) -> None:
     SourceCapability.PLATFORM_INSTANCE,
     "By default, platform_instance will use the AWS account id",
 )
-@capability(
-    SourceCapability.DELETION_DETECTION,
-    "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
-    supported=True,
-)
 class DynamoDBSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:

metadata-ingestion/src/datahub/ingestion/source/feast.py

@@ -96,6 +96,8 @@ class FeastRepositorySourceConfig(ConfigModel):
 @platform_name("Feast")
 @config_class(FeastRepositorySourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 @dataclass
 class FeastRepositorySource(Source):

metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py

@@ -1125,6 +1125,7 @@ def transform_connector_config(
 @config_class(KafkaConnectSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class KafkaConnectSource(StatefulIngestionSourceBase):
     config: KafkaConnectSourceConfig

metadata-ingestion/src/datahub/ingestion/source/mode.py

@@ -210,8 +210,12 @@ def report_dropped_space(self, ent_name: str) -> None:
 @platform_name("Mode")
 @config_class(ModeConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
+@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
+@capability(SourceCapability.LINEAGE_FINE, "Supported by default")
+@capability(SourceCapability.OWNERSHIP, "Enabled by default")
 class ModeSource(StatefulIngestionSourceBase):
     """
 
@@ -756,7 +760,7 @@ def _replace_definitions(self, raw_query: str) -> str:
     def _parse_definition_name(self, definition_variable: str) -> Tuple[str, str]:
         name, alias = "", ""
         # i.e '{{ @join_on_definition as alias}}'
-        name_match = re.findall("@[a-zA-z]+", definition_variable)
+        name_match = re.findall("@[a-zA-Z]+", definition_variable)
         if len(name_match):
             name = name_match[0][1:]
         alias_match = re.findall(

metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py

@@ -1220,8 +1220,12 @@ def report_to_datahub_work_units(
 @platform_name("PowerBI")
 @config_class(PowerBiDashboardSourceConfig)
 @support_status(SupportStatus.CERTIFIED)
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
+@capability(SourceCapability.OWNERSHIP, "Enabled by default")
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
+@capability(SourceCapability.TAGS, "Enabled by default")
 @capability(
     SourceCapability.OWNERSHIP,
     "Disabled by default, configured using `extract_ownership`",

metadata-ingestion/src/datahub/ingestion/source/pulsar.py

@@ -91,6 +91,7 @@ def __init__(self, schema):
 @config_class(PulsarSourceConfig)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 @dataclass
 class PulsarSource(StatefulIngestionSourceBase):
     def __init__(self, config: PulsarSourceConfig, ctx: PipelineContext):

metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_sense.py

@@ -93,12 +93,22 @@
 @platform_name("Qlik Sense")
 @config_class(QlikSourceConfig)
 @support_status(SupportStatus.INCUBATING)
+@capability(SourceCapability.CONTAINERS, "Enabled by default")
 @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
+@capability(
+    SourceCapability.LINEAGE_COARSE,
+    "Enabled by default.",
+)
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Disabled by default. ",
+)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(
     SourceCapability.OWNERSHIP,
     "Enabled by default, configured using `ingest_owner`",
 )
+@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
 class QlikSenseSource(StatefulIngestionSourceBase, TestableSource):
     """
     This plugin extracts the following:

metadata-ingestion/src/datahub/ingestion/source/redash.py

@@ -18,12 +18,13 @@
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (  # SourceCapability,; capability,
     SupportStatus,
+    capability,
     config_class,
     platform_name,
     support_status,
 )
 from datahub.ingestion.api.registry import import_path
-from datahub.ingestion.api.source import Source, SourceReport
+from datahub.ingestion.api.source import Source, SourceCapability, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.com.linkedin.pegasus2avro.common import (
     AuditStamp,
@@ -308,6 +309,7 @@ def report_dropped(self, item: str) -> None:
 @platform_name("Redash")
 @config_class(RedashConfig)
 @support_status(SupportStatus.INCUBATING)
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
 class RedashSource(Source):
     """
     This plugin extracts the following: