diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 8bc6b8a6e5f7c..b3b990078ec0f 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -702,7 +702,12 @@ jobs: strategy: fail-fast: false matrix: - test_strategy: ["no_cypress", "cypress_suite1", "cypress_rest"] + test_strategy: [ + "no_cypress_suite0", + "no_cypress_suite1", + "cypress_suite1", + "cypress_rest" + ] needs: [ setup, @@ -798,11 +803,6 @@ jobs: ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" run: | ./smoke-test/run-quickstart.sh - - name: sleep 60s - run: | - # we are doing this because gms takes time to get ready - # and we don't have a better readiness check when bootstrap is done - sleep 60s - name: Disk Check run: df -h . && docker images - name: Disable ES Disk Threshold diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py index 51ad8a050adc2..540adbf4bfd15 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/config.py @@ -10,7 +10,7 @@ from datahub.configuration.source_common import DatasetLineageProviderConfigBase from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.ingestion.source.data_lake_common.path_spec import PathSpec -from datahub.ingestion.source.sql.postgres import BasePostgresConfig +from datahub.ingestion.source.sql.sql_config import BasicSQLAlchemyConfig from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulLineageConfigMixin, StatefulProfilingConfigMixin, @@ -64,7 +64,7 @@ class RedshiftUsageConfig(BaseUsageConfig, StatefulUsageConfigMixin): class RedshiftConfig( - BasePostgresConfig, + BasicSQLAlchemyConfig, DatasetLineageProviderConfigBase, S3DatasetLineageProviderConfigBase, RedshiftUsageConfig, diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index 05011b2d7a769..abed8505f168b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -381,7 +381,8 @@ def _get_upstream_lineages( qualified_table_name = dataset_urn.DatasetUrn.create_from_string( source.urn ).get_entity_id()[1] - db, schema, table = qualified_table_name.split(".") + # -3 because platform instance is optional and that can cause the split to have more than 3 elements + db, schema, table = qualified_table_name.split(".")[-3:] if db == raw_db_name: db = alias_db_name path = f"{db}.{schema}.{table}" diff --git a/smoke-test/smoke.sh b/smoke-test/smoke.sh index 3236a0e5c3f0c..db0389be1f489 100755 --- a/smoke-test/smoke.sh +++ b/smoke-test/smoke.sh @@ -24,12 +24,14 @@ source venv/bin/activate source ./set-cypress-creds.sh -# no_cypress, cypress_suite1, cypress_rest +# no_cypress_suite0, no_cypress_suite1, cypress_suite1, cypress_rest if [[ -z "${TEST_STRATEGY}" ]]; then pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke.xml else - if [ "$TEST_STRATEGY" == "no_cypress" ]; then - pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' + if [ "$TEST_STRATEGY" == "no_cypress_suite0" ]; then + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -k 'not test_run_cypress' -m 'not no_cypress_suite1' + elif [ "$TEST_STRATEGY" == "no_cypress_suite1" ]; then + pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_non_cypress.xml -m 'no_cypress_suite1' else pytest -rP --durations=20 -vv --continue-on-collection-errors --junit-xml=junit.smoke_cypress_${TEST_STRATEGY}.xml tests/cypress/integration_test.py fi diff --git a/smoke-test/test_e2e.py b/smoke-test/test_e2e.py index 4a0a122b79670..abb4841314c4a 100644 --- a/smoke-test/test_e2e.py +++ b/smoke-test/test_e2e.py @@ -8,6 +8,8 @@ import tenacity from datahub.ingestion.run.pipeline import Pipeline +pytestmark = pytest.mark.no_cypress_suite1 + from tests.utils import ( get_frontend_url, get_gms_url, diff --git a/smoke-test/tests/privileges/test_privileges.py b/smoke-test/tests/privileges/test_privileges.py index d0f00734ae9f3..aa54a50b04e7f 100644 --- a/smoke-test/tests/privileges/test_privileges.py +++ b/smoke-test/tests/privileges/test_privileges.py @@ -5,6 +5,8 @@ get_frontend_url, get_admin_credentials,get_sleep_info) from tests.privileges.utils import * +pytestmark = pytest.mark.no_cypress_suite1 + sleep_sec, sleep_times = get_sleep_info() @pytest.fixture(scope="session") diff --git a/smoke-test/tests/timeline/timeline_test.py b/smoke-test/tests/timeline/timeline_test.py index 4705343c1a2ba..c075d981487db 100644 --- a/smoke-test/tests/timeline/timeline_test.py +++ b/smoke-test/tests/timeline/timeline_test.py @@ -1,4 +1,5 @@ import json +import pytest from time import sleep from datahub.cli import timeline_cli @@ -7,6 +8,7 @@ from tests.utils import (get_datahub_graph, ingest_file_via_rest, wait_for_writes_to_sync) +pytestmark = pytest.mark.no_cypress_suite1 def test_all(): platform = "urn:li:dataPlatform:kafka"