Merge branch 'datahub-project:master' into master

acryldata · Jul 15, 2024 · fcc4115 · fcc4115
2 parents 54397ae + 43ae12f
commit fcc4115
Show file tree

Hide file tree

Showing 63 changed files with 2,960 additions and 813 deletions.
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -61,7 +61,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -67,7 +67,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -33,7 +33,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -125,7 +125,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Run lint on smoke test
         if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
         run: |
@@ -144,7 +144,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -208,7 +208,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -272,7 +272,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -336,7 +336,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -400,7 +400,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -642,7 +642,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
@@ -729,7 +729,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
@@ -856,7 +856,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -38,7 +38,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -51,7 +51,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -53,7 +53,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -34,7 +34,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -54,7 +54,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
@@ -35,7 +35,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/README.md b/README.md
@@ -118,6 +118,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Cabify](https://cabify.tech/)
 - [ClassDojo](https://www.classdojo.com/)
 - [Coursera](https://www.coursera.org/)
+- [CVS Health](https://www.cvshealth.com/)
 - [DefinedCrowd](http://www.definedcrowd.com)
 - [DFDS](https://www.dfds.com/)
 - [Digital Turbine](https://www.digitalturbine.com/)

diff --git a/docs/advanced/mcp-mcl.md b/docs/advanced/mcp-mcl.md
@@ -183,6 +183,9 @@ A writer can provide a header with the expected `version` when initiating the re
 match the actual `version` stored in the database, the write will fail. This prevents overwriting an aspect that has
 been modified by another process.
 
+Note: If the aspect doesn't exist yet, then the `version` is `-1`. A writer can use this `version` to only create
+an aspect if it doesn't. Also see _Change Types: [`CREATE`, `CREATE_ENTITY`]_ section below.
+
 #### If-Modified-Since / If-Unmodified-Since
 
 A writer may also specify time-based conditions using http header semantics. Similar to version based conditional writes
@@ -194,3 +197,12 @@ A writer can specify that the aspect must NOT have been modified after a specifi
 
 `If-Modified-Since`
 A writer can specify that the aspect must have been modified after a specific time, following [If-Modified-Since](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since) http headers.
+
+#### Change Types: [`CREATE`, `CREATE_ENTITY`]
+
+Another form of conditional writes which considers the existence of an aspect or entity uses the following Change Types.
+
+`CREATE` - Create the aspect if it doesn't already exist.
+
+`CREATE_ENTITY` - Create the aspect if no aspects exist for the entity.
+