Merge branch 'datahub-project:master' into master

acryldata · Dec 6, 2023 · 28585a6 · 28585a6
2 parents 180bba2 + 2eee333
commit 28585a6
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 12 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -702,7 +702,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"]
+        test_strategy: [
+          "no_cypress_suite0",
+          "no_cypress_suite1",
+          "cypress_suite1", 
+          "cypress_rest"
+        ]
     needs:
       [
         setup,
@@ -798,11 +803,6 @@ jobs:
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh
-      - name: sleep 60s
-        run: |
-          # we are doing this because gms takes time to get ready
-          # and we don't have a better readiness check when bootstrap is done
-          sleep 60s
       - name: Disk Check
         run: df -h . && docker images
       - name: Disable ES Disk Threshold

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py
@@ -10,7 +10,7 @@
 from datahub.configuration.source_common import DatasetLineageProviderConfigBase
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.ingestion.source.data_lake_common.path_spec import PathSpec
-from datahub.ingestion.source.sql.postgres import BasePostgresConfig
+from datahub.ingestion.source.sql.sql_config import BasicSQLAlchemyConfig
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulLineageConfigMixin,
     StatefulProfilingConfigMixin,
@@ -64,7 +64,7 @@ class RedshiftUsageConfig(BaseUsageConfig, StatefulUsageConfigMixin):
 
 
 class RedshiftConfig(
-    BasePostgresConfig,
+    BasicSQLAlchemyConfig,
     DatasetLineageProviderConfigBase,
     S3DatasetLineageProviderConfigBase,
     RedshiftUsageConfig,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py
@@ -381,7 +381,8 @@ def _get_upstream_lineages(
                 qualified_table_name = dataset_urn.DatasetUrn.create_from_string(
                     source.urn
                 ).get_entity_id()[1]
-                db, schema, table = qualified_table_name.split(".")
+                # -3 because platform instance is optional and that can cause the split to have more than 3 elements
+                db, schema, table = qualified_table_name.split(".")[-3:]
                 if db == raw_db_name:
                     db = alias_db_name
                     path = f"{db}.{schema}.{table}"

diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh
@@ -24,12 +24,14 @@ source venv/bin/activate
 
 source ./set-cypress-creds.sh
 
-# no_cypress, cypress_suite1, cypress_rest
+# no_cypress_suite0, no_cypress_suite1, cypress_suite1, cypress_rest
 if [[ -z "${TEST_STRATEGY}" ]]; then
     pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml
 else
-    if [ "$TEST_STRATEGY" == "no_cypress" ]; then
-        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress'
+    if [ "$TEST_STRATEGY" == "no_cypress_suite0" ]; then
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' -m 'not no_cypress_suite1'
+    elif [ "$TEST_STRATEGY" == "no_cypress_suite1" ]; then
+        pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -m 'no_cypress_suite1'
     else
         pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py
     fi

diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py
@@ -8,6 +8,8 @@
 import tenacity
 from datahub.ingestion.run.pipeline import Pipeline
 
+pytestmark = pytest.mark.no_cypress_suite1
+
 from tests.utils import (
     get_frontend_url,
     get_gms_url,

diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py
@@ -5,6 +5,8 @@
                         get_frontend_url, get_admin_credentials,get_sleep_info)
 from tests.privileges.utils import *
 
+pytestmark = pytest.mark.no_cypress_suite1
+
 sleep_sec, sleep_times = get_sleep_info()
 
 @pytest.fixture(scope="session")

diff --git a/smoke-test/tests/timeline/timeline_test.py b/smoke-test/tests/timeline/timeline_test.py
@@ -1,4 +1,5 @@
 import json
+import pytest
 from time import sleep
 
 from datahub.cli import timeline_cli
@@ -7,6 +8,7 @@
 from tests.utils import (get_datahub_graph, ingest_file_via_rest,
                          wait_for_writes_to_sync)
 
+pytestmark = pytest.mark.no_cypress_suite1
 
 def test_all():
     platform = "urn:li:dataPlatform:kafka"