Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Oct 8, 2024
2 parents 98b395f + 6fd36ef commit a74219b
Show file tree
Hide file tree
Showing 42 changed files with 1,178 additions and 126 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/docker-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ jobs:
with:
image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -249,7 +249,7 @@ jobs:
with:
image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -313,7 +313,7 @@ jobs:
with:
image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -377,7 +377,7 @@ jobs:
with:
image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -443,7 +443,7 @@ jobs:
with:
image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -708,7 +708,7 @@ jobs:
with:
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Slim Image
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down Expand Up @@ -796,7 +796,7 @@ jobs:
with:
image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }}
- name: Run Trivy vulnerability scanner Full Image
uses: aquasecurity/trivy-action@0.8.0
uses: aquasecurity/trivy-action@0.25.0
env:
TRIVY_OFFLINE_SCAN: true
with:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import { EditableSchemaFieldInfo, SchemaFieldEntity } from '../../../../../../../types.generated';
import {
EditableSchemaFieldInfo,
SchemaFieldEntity,
DocumentationAssociation,
} from '../../../../../../../types.generated';

interface Props {
schemaFieldEntity?: SchemaFieldEntity | null;
Expand All @@ -7,19 +11,29 @@ interface Props {
}

export function getFieldDescriptionDetails({ schemaFieldEntity, editableFieldInfo, defaultDescription }: Props) {
const documentation = schemaFieldEntity?.documentation?.documentations?.[0];
const isUsingDocumentationAspect = !editableFieldInfo?.description && !!documentation;
const documentations = schemaFieldEntity?.documentation?.documentations;
let latestDocumentation: DocumentationAssociation | undefined;

if (documentations && documentations.length > 0) {
latestDocumentation = documentations.reduce((latest, current) => {
const latestTime = latest.attribution?.time || 0;
const currentTime = current.attribution?.time || 0;
return currentTime > latestTime ? current : latest;
});
}

const isUsingDocumentationAspect = !editableFieldInfo?.description && !!latestDocumentation;
const isPropagated =
isUsingDocumentationAspect &&
!!documentation?.attribution?.sourceDetail?.find(
!!latestDocumentation?.attribution?.sourceDetail?.find(
(mapEntry) => mapEntry.key === 'propagated' && mapEntry.value === 'true',
);

const displayedDescription =
editableFieldInfo?.description || documentation?.documentation || defaultDescription || '';
editableFieldInfo?.description || latestDocumentation?.documentation || defaultDescription || '';

const sourceDetail = documentation?.attribution?.sourceDetail;
const propagatedDescription = documentation?.documentation;
const sourceDetail = latestDocumentation?.attribution?.sourceDetail;
const propagatedDescription = latestDocumentation?.documentation;

return { displayedDescription, isPropagated, sourceDetail, propagatedDescription };
}
6 changes: 6 additions & 0 deletions docker/profiles/docker-compose.actions.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
x-search-datastore-elasticsearch-env: &search-datastore-env
ELASTICSEARCH_HOST: search
ELASTICSEARCH_PORT: 9200
ELASTICSEARCH_PROTOCOL: http
ELASTICSEARCH_USE_SSL: ${ELASTICSEARCH_USE_SSL:-false}

x-datahub-actions-service: &datahub-actions-service
hostname: actions
Expand All @@ -7,6 +12,7 @@ x-datahub-actions-service: &datahub-actions-service
- ${DATAHUB_LOCAL_COMMON_ENV:-empty.env}
- ${DATAHUB_LOCAL_ACTIONS_ENV:-empty2.env}
environment:
<<: *search-datastore-env
ACTIONS_EXTRA_PACKAGES: ${ACTIONS_EXTRA_PACKAGES:-}
ACTIONS_CONFIG: ${ACTIONS_CONFIG:-}
KAFKA_BOOTSTRAP_SERVER: broker:29092
Expand Down
4 changes: 4 additions & 0 deletions docs-website/docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ module.exports = {
label: "Learn",
position: "right",
items: [
{
to: "https://www.acryldata.io/webinars/weekly-live-demo",
label: "Weekly Demo",
},
{
to: "/learn",
label: "Use Cases",
Expand Down
1 change: 1 addition & 0 deletions docs-website/sidebars.js
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ module.exports = {
},
{
"DataHub Cloud Release History": [
"docs/managed-datahub/release-notes/v_0_3_6",
"docs/managed-datahub/release-notes/v_0_3_5",
"docs/managed-datahub/release-notes/v_0_3_4",
"docs/managed-datahub/release-notes/v_0_3_3",
Expand Down
15 changes: 15 additions & 0 deletions docs-website/src/pages/_components/Hero/hero.module.scss
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,21 @@
display: flex;
}

.hero__content_footer {
align-items: center;
justify-content: center;
display: flex;
width: 100%;
color: #FFFFFFcc;
margin-top: -2.5rem;
font-size: 1rem;
font-weight: 400;
a, path {
color: white;
font-weight: 600;
}
}

// .hero__alert {
// margin-bottom: 2rem;

Expand Down
5 changes: 4 additions & 1 deletion docs-website/src/pages/_components/Hero/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import Link from "@docusaurus/Link";
import useBaseUrl from "@docusaurus/useBaseUrl";
// import Image from "@theme/IdealImage";
// import { useColorMode } from "@docusaurus/theme-common";
// import { QuestionCircleOutlined } from "@ant-design/icons";
import { HeartOutlined } from "@ant-design/icons";
import styles from "./hero.module.scss";
import { animate, motion, useMotionValue, useTransform } from "framer-motion";
// import CodeBlock from "@theme/CodeBlock";
Expand Down Expand Up @@ -87,6 +87,9 @@ const Hero = ({ onOpenTourModal }) => {
<img src={useBaseUrl("/img/hero.png")} />
</div>
</div>
<div className="hero__content_footer">
Built with&nbsp;<HeartOutlined />&nbsp;by&nbsp;<a href="https://acryldata.io" target="_blank">Acryl Data</a>&nbsp;and&nbsp;<a href="https://www.acryldata.io/press/founded-by-airbnb-and-linkedin-data-veterans-acryl-data-re-imagines-metadata-management-with-dollar9-million-in-seed-funding" target="_blank">LinkedIn</a>.
</div>
</div>
</header>
);
Expand Down
32 changes: 12 additions & 20 deletions docs/authorization/policies.md
Original file line number Diff line number Diff line change
Expand Up @@ -191,16 +191,18 @@ These privileges are for DataHub operators to access & manage the administrative

#### System Management

| Platform Privileges | Description |
|-----------------------------------------------|------------------------------------------------------------------------|
| Restore Indices API[^1] | Allow actor to use the Restore Indices API. | |
| Get Timeseries index sizes API[^1] | Allow actor to use the get Timeseries indices size API. |
| Truncate timeseries aspect index size API[^1] | Allow actor to use the API to truncate a timeseries index. |
| Get ES task status API[^1] | Allow actor to use the get task status API for an ElasticSearch task. |
| Enable/Disable Writeability API[^1] | Allow actor to enable or disable GMS writeability for data migrations. |
| Apply Retention API[^1] | Allow actor to apply retention using the API. |
| Analytics API access[^1] | Allow actor to use API read access to raw analytics data. |
| Manage System Operations | Allow actor to manage system operation controls. |
| Platform Privileges | Description |
|-----------------------------------------------|----------------------------------------------------------------------------------------------------------|
| Restore Indices API[^1] | Allow actor to use the Restore Indices API. | |
| Get Timeseries index sizes API[^1] | Allow actor to use the get Timeseries indices size API. |
| Truncate timeseries aspect index size API[^1] | Allow actor to use the API to truncate a timeseries index. |
| Get ES task status API[^1] | Allow actor to use the get task status API for an ElasticSearch task. |
| Enable/Disable Writeability API[^1] | Allow actor to enable or disable GMS writeability for data migrations. |
| Apply Retention API[^1] | Allow actor to apply retention using the API. |
| Analytics API access[^1] | Allow actor to use API read access to raw analytics data. |
| Explain ElasticSearch Query API[^1] | Allow actor to use the Operations API explain endpoint. |
| Produce Platform Event API[^1] | Allow actor to produce Platform Events using the API. |
| Manage System Operations | Allow actor to manage system operation controls. This setting includes all System Management privileges. |

[^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true
[^2]: DataHub Cloud only
Expand Down Expand Up @@ -262,16 +264,6 @@ These privileges are to view & modify any entity within DataHub.
[^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true
[^2]: DataHub Cloud only

#### System Management

| System Privileges | Description |
|-------------------------------------|--------------------------------------------------------------------------------------------|
| Explain ElasticSearch Query API[^1] | Allow actor to use the Operations API explain endpoint. |
| Produce Platform Event API[^1] | Allow actor to produce Platform Events using the API. |

[^1]: Only active if REST_API_AUTHORIZATION_ENABLED is true
[^2]: DataHub Cloud only

### Specific Entity-level Privileges
These privileges are not generalizable.

Expand Down
2 changes: 2 additions & 0 deletions docs/how/updating-datahub.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe

### Other Notable Changes

- #11549 - Manage Operations Privilege is extended from throttle control to all system management and operations APIs.

## 0.14.1

### Breaking Changes
Expand Down
85 changes: 85 additions & 0 deletions docs/managed-datahub/release-notes/v_0_3_6.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# v0.3.6
---

Release Availability Date
---
08-Oct-2024

Recommended CLI/SDK
---
- `v0.14.1.1` with release notes at https://github.com/datahub-project/datahub/releases/tag/v0.14.1

If you are using an older CLI/SDK version, then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, GitHub Actions, Airflow, in Python SDK somewhere, Java SDK, etc. This is a strong recommendation to upgrade, as we keep on pushing fixes in the CLI, and it helps us support you better.

## Release Changelog
---

- All changes in https://github.com/datahub-project/datahub/releases/tag/v0.14.1
- Note Breaking Changes: https://datahubproject.io/docs/how/updating-datahub/#0141

- Breaking Changes
- Automatic downgrade to a previous release is not supported. Please reach out to support if required.
- Metadata tests that are created/edited will not automatically run on a small batch (less than 10,000 assets) immediately anymore, but instead will rely on the scheduled run. Scheduled runs and metadata tests running on asset changes are unimpacted. This bug is fixed in 0.3.6.1.
- Rest API Authorization enabled by default

- Bug Fixes
- No longer push notifications for soft-deleted entities
- Fixed a bug in the share modal where sharing was disabled even if instances to share were selected
- Correctly show the lineage count of upstreams and downstreams when there are large numbers
- Combine sibling entities in all search lists throughout the app, and not just the main search page
- Fix view select for Impact Analysis Tab
- Remove "Remove from Domain", "Remove from Data Product", and "Remove from Glossary" buttons when adding assets to domains, data products, and glossary
- Add 'explore all' button below all autocomplete results
- Remove Ctrl-K as a mechanism to invoke the search bar on Mac (Command K only)
- Adding Nullable tag back to V2 UI Columns tab.
- Remove Resources section on Homepage
- Use Browse Path instead of Container Path for rendering the context paths for assets across Search, Entity Profile, and more.
- Minor fixes with Snowflake Tag Sync automation to prevent rate limiting on removals of tags that do not yet exist in Snowflake
- Minor improvements to the Home Page Settings experience (Settings > Home Page)
- Show the number of assets ingested when ingestion succeeds with warnings
- Fetch more than 20 related proposals for an asset when rendering profile page (Glossary term, Tag proposals)
- Add 'ALTER TABLE' expression to BigQuery Audit Log based Freshness Assertion
- Hide the 'For You' section on the home page if there is no applicable content
- Fixed mouse interactions with graphs on statistics pages
- Fixed rare issues with lineage visualization node positioning and edge drawing
- Fixed bug with changing advanced search filter condition
- Hide "Remove from \[X]" buttons when adding assets to domains, data products, and glossary terms
- Change lineage sidebar summaries to use the default lineage time range
- Impact analysis and data product assets pages properly show results based on the currently selected view
- The New Users section in the Analytics page now tracks actual new users instead of last login
- Data Contracts builder now shows custom assertions as expected

- Product changes
- Show total (nested) asset count on home page Domain cards
- Add the ability to copy an urn from the ML Model asset profile page
- Added the `deleteStructuredProperty` GraphQL API to delete structured properties
- Improved "matched" section on search cards to let users know if a column is matched by a search term
- The "Documentation" button under the "Govern" dropdown has been rebranded to the "Compliance Forms" button to more accurately reflect this feature.
- On the Compliance Forms page, we now show you an overview of your organization's forms created to help govern your data assets. Here, you'll be able to manage all of your forms from the UI. The previous content of this page still exists under the "Analytics" tab.
- We have several new question types available for forms - questions to collect documentation, ownership, domains, and glossary terms.
- You can also now submit a response on up to 10,000 assets in the bulk form submission screen even if you have more than 10,000 assets. You can always submit again once these answers are processed.
- We now show the bulk form submission screen first when entering a form from the Task Center.
- We added a search bar in the bulk form submission flow to allow users to search when selecting assets to submit a response for.
- Support Generating AI summaries of Queries via the UI - Request this to be enabled your by Acryl representative.
- Allow hiding "Pending Tasks" module on home page, allows horizontal expansion of content
- Moved "Pinned Links" up to the top of the home page as a separate category, from the left sidebar bottom (based on requests)
- Assertions page uplift! Redesigned Asset Assertions (Entity Profile > Quality) to include a new Summary tab as well as search, filtering, and grouping capabilities on the assertions list.
- Add toggle in lineage visualization for showing soft-deleted and non-existent entities
- Add "Last Updated (in Source)" filter to search
- Add "View In \[Source]" button to lineage visualization sidebar
- Send test notification button will always be visible in the subscriptions side bar if the user has set their memberID

- Ingestion changes

- Platform changes
- Python images based on ubuntu base image
- Postgres NPE fixed (upgraded ebean framework)
- Domains/Containers have additional conditions: [DESCENDANTS_INCL, ANCESTORS_INCL, RELATED_INCL]
- Removed legacy browsePathV2 GMS bootstrap job
- Graphql dataloaders are now lazy loaded
- Added fallback versioning strategy to aggregation query
- Authorization cache added per request
- API Throttling on Kafka lag extended to API for programmatic access
- New GraphQL API to create subscriptions on behalf of individual users. Simply provide a new optional 'userUrn' parameter to the `createSubscription` mutation.
- Several Timeline/Schema History bugs have been fixed which may impact version calculations in some scenarios
- OIDC callback does not update corpUserStatus if the user is already ACTIVE
2 changes: 1 addition & 1 deletion docs/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
:::tip DataHub Cloud

This guide provides instructions on deploying the open source DataHub locally.
If you're interested in a managed version, [Acryl Data](https://www.acryldata.io/product) provides a fully managed, premium version of DataHub. <br />
If you're interested in a managed version, [Acryl Data](https://www.acryldata.io) provides a fully managed, premium version of DataHub. <br />
**[Get Started with DataHub Cloud](./managed-datahub/welcome-acryl.md)**

:::
Expand Down
Loading

0 comments on commit a74219b

Please sign in to comment.