Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Jul 19, 2024
2 parents 3cb93d8 + 3733a40 commit 049776b
Show file tree
Hide file tree
Showing 34 changed files with 1,127 additions and 248 deletions.
12 changes: 6 additions & 6 deletions .github/workflows/airflow-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,19 @@ jobs:
extra_pip_requirements: "apache-airflow~=2.2.4"
extra_pip_extras: plugin-v1
- python-version: "3.10"
extra_pip_requirements: "apache-airflow==2.4.3"
extra_pip_requirements: "apache-airflow~=2.4.3"
extra_pip_extras: plugin-v2,test-airflow24
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow==2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow==2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
extra_pip_extras: plugin-v2
- python-version: "3.10"
extra_pip_requirements: 'apache-airflow==2.9.0 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.10.txt'
- python-version: "3.11"
extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt'
extra_pip_extras: plugin-v2
fail-fast: false
steps:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,4 +255,19 @@ public static Filter viewFilter(
Filter result = SearchUtils.combineFilters(null, viewInfo.getDefinition().getFilter());
return result;
}

/**
* Simply resolves the end time filter for the search across lineage query. If the start time is
* provided, but end time is not provided, we will default to the current time.
*/
public static Long getLineageEndTimeMillis(
@Nullable Long startTimeMillis, @Nullable Long endTimeMillis) {
if (endTimeMillis != null) {
return endTimeMillis;
}
if (startTimeMillis != null) {
return System.currentTimeMillis();
}
return null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.linkedin.datahub.graphql.generated.LineageInput;
import com.linkedin.datahub.graphql.generated.LineageRelationship;
import com.linkedin.datahub.graphql.generated.Restricted;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper;
import com.linkedin.metadata.graph.SiblingGraphService;
import graphql.schema.DataFetcher;
Expand Down Expand Up @@ -63,7 +64,10 @@ public CompletableFuture<EntityLineageResult> get(DataFetchingEnvironment enviro
@Nullable final Integer count = input.getCount(); // Optional!
@Nullable final Boolean separateSiblings = input.getSeparateSiblings(); // Optional!
@Nullable final Long startTimeMillis = input.getStartTimeMillis(); // Optional!
@Nullable final Long endTimeMillis = input.getEndTimeMillis(); // Optional!
@Nullable
final Long endTimeMillis =
ResolverUtils.getLineageEndTimeMillis(
input.getStartTimeMillis(), input.getEndTimeMillis()); // Optional!

com.linkedin.metadata.graph.LineageDirection resolvedDirection =
com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ public CompletableFuture<ScrollAcrossLineageResults> get(DataFetchingEnvironment
@Nullable
Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis();
@Nullable
Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis();
Long endTimeMillis =
ResolverUtils.getLineageEndTimeMillis(input.getStartTimeMillis(), input.getEndTimeMillis());

final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags());
if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
@Nullable
Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis();
@Nullable
Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis();
Long endTimeMillis =
ResolverUtils.getLineageEndTimeMillis(input.getStartTimeMillis(), input.getEndTimeMillis());

final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags());
if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.EntityTypeToPlatforms;
import com.linkedin.datahub.graphql.generated.LineageFlags;
import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper;
import com.linkedin.datahub.graphql.types.mappers.ModelMapper;
import java.util.Collections;
Expand Down Expand Up @@ -42,12 +43,16 @@ public com.linkedin.metadata.query.LineageFlags apply(
if (lineageFlags.getIgnoreAsHops() != null) {
result.setIgnoreAsHops(mapIgnoreAsHops(lineageFlags.getIgnoreAsHops()));
}
if (lineageFlags.getEndTimeMillis() != null) {
result.setEndTimeMillis(lineageFlags.getEndTimeMillis());
}
if (lineageFlags.getStartTimeMillis() != null) {
result.setStartTimeMillis(lineageFlags.getStartTimeMillis());
}
// Default to "now" if no end time is provided, but start time is provided.
Long endTimeMillis =
ResolverUtils.getLineageEndTimeMillis(
lineageFlags.getStartTimeMillis(), lineageFlags.getEndTimeMillis());
if (endTimeMillis != null) {
result.setEndTimeMillis(endTimeMillis);
}
if (lineageFlags.getEntitiesExploredPerHopLimit() != null) {
result.setEntitiesExploredPerHopLimit(lineageFlags.getEntitiesExploredPerHopLimit());
}
Expand Down
15 changes: 15 additions & 0 deletions datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
INGESTION_REFRESH_SOURCES_ID,
} from '../../onboarding/config/IngestionOnboardingConfig';
import { ONE_SECOND_IN_MS } from '../../entity/shared/tabs/Dataset/Queries/utils/constants';
import { useCommandS } from './hooks';

const PLACEHOLDER_URN = 'placeholder-urn';

Expand All @@ -51,6 +52,8 @@ const FilterWrapper = styled.div`
display: flex;
`;

const SYSTEM_INTERNAL_SOURCE_TYPE = 'SYSTEM';

export enum IngestionSourceType {
ALL,
UI,
Expand Down Expand Up @@ -102,6 +105,17 @@ export const IngestionSourceList = () => {
// Set of removed urns used to account for eventual consistency
const [removedUrns, setRemovedUrns] = useState<string[]>([]);
const [sourceFilter, setSourceFilter] = useState(IngestionSourceType.ALL);
const [hideSystemSources, setHideSystemSources] = useState(true);

/**
* Show or hide system ingestion sources using a hidden command S command.
*/
useCommandS(() => setHideSystemSources(!hideSystemSources));

// Ingestion Source Default Filters
const filters = hideSystemSources
? [{ field: 'sourceType', values: [SYSTEM_INTERNAL_SOURCE_TYPE], negated: true }]
: undefined;

// Ingestion Source Queries
const { loading, error, data, client, refetch } = useListIngestionSourcesQuery({
Expand All @@ -110,6 +124,7 @@ export const IngestionSourceList = () => {
start,
count: pageSize,
query: (query?.length && query) || undefined,
filters,
},
},
fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',
Expand Down
16 changes: 16 additions & 0 deletions datahub-web-react/src/app/ingest/source/hooks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import { useEffect } from 'react';

export const useCommandS = (onPress: () => void) => {
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.metaKey && event.key === 's') {
event.preventDefault();
onPress();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => {
window.removeEventListener('keydown', handleKeyDown);
};
}, [onPress]);
};
16 changes: 15 additions & 1 deletion docs-website/docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ module.exports = {
label: "Docs",
position: "right",
},
{
to: "/learn",
activeBasePath: "learn",
label: "Learn",
position: "right",
},
{
to: "/integrations",
activeBasePath: "integrations",
Expand Down Expand Up @@ -299,7 +305,15 @@ module.exports = {
showLastUpdateAuthor: false,
showLastUpdateTime: false,
},
blog: false,
blog: {
blogTitle: "DataHub Learn",
blogSidebarTitle: "DataHub Learn",
blogDescription: "Learn about the hot topics in the data ecosystem and how DataHub can help you with your data journey.",
path: "src/learn",
routeBasePath: "learn",
postsPerPage: "ALL",
blogListComponent: "../src/learn/_components/LearnListPage",
},
theme: {
customCss: [
isSaas ? require.resolve("./src/styles/acryl.scss") : require.resolve("./src/styles/datahub.scss"),
Expand Down
30 changes: 30 additions & 0 deletions docs-website/src/learn/_components/LearnItemCard/index.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import React from "react";
import clsx from "clsx";
import Link from "@docusaurus/Link";
import { useBlogPost } from "@docusaurus/theme-common/internal";
import styles from "./styles.module.scss";

export default function LearnItemCard() {
const { metadata } = useBlogPost();
const { permalink, title, description, formattedDate, frontMatter } = metadata;
return (
<div className={clsx("col col--4", styles.featureCol)}>
<Link to={permalink} className={clsx("card", styles.card)}>
{frontMatter?.image ? (
<div className={styles.card_image}>
<img src={frontMatter?.image} alt={title} />
</div>
) : (
<div className={clsx("card__header", styles.featureHeader)}>
<h2>{title}</h2>
</div>
)}
<div className={clsx("card__body", styles.featureBody)}>
<div>{description}</div>
</div>

<div className={clsx(styles.card_date)}>Published on {formattedDate}</div>
</Link>
</div>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
.featureCol {
display: flex;
}

.card_date {
padding: 1rem 2rem;
font-size: 0.8rem;
font-style: italic;
color: gray;
margin-top: auto;
}

.card_feature {
font-size: 2rem;
font-weight: 700;
}

.card {
color: var(--ifm-text-color);
text-decoration: none !important;
padding: 0rem;
margin-bottom: 2rem;
align-self: stretch;
flex-grow: 1;
&:hover {
border-color: var(--ifm-color-primary);
}
hr {
margin: 0;
}
}

.featureHeader {
h2 {
margin-bottom: 1rem !important;
font-size: 1.25rem;
}
padding: 1rem 2rem;
}

.featureBody {
padding: 0 2rem;
}

.card_image {
margin: 0;
margin-bottom: 0.5rem;

img {
width: 100%;
height: auto;
}
}
91 changes: 91 additions & 0 deletions docs-website/src/learn/_components/LearnListPage/index.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import React, { useState } from "react";
import clsx from "clsx";

import useDocusaurusContext from "@docusaurus/useDocusaurusContext";
import { PageMetadata, HtmlClassNameProvider, ThemeClassNames } from "@docusaurus/theme-common";
import BlogListPaginator from "@theme/BlogListPaginator";
import SearchMetadata from "@theme/SearchMetadata";
import { BlogPostProvider } from "@docusaurus/theme-common/internal";
import LearnItemCard from "../LearnItemCard";
import Layout from "@theme/Layout";
import styles from "./styles.module.scss";

function BlogListPageMetadata(props) {
const { metadata } = props;
const {
siteConfig: { title: siteTitle },
} = useDocusaurusContext();
const { blogDescription, blogTitle, permalink } = metadata;
const isBlogOnlyMode = permalink === "/";
const title = isBlogOnlyMode ? siteTitle : blogTitle;
return (
<>
<PageMetadata title={title} description={blogDescription} />
<SearchMetadata tag="blog_posts_list" />
</>
);
}

function BlogListPageContent(props) {
const { metadata, items } = props;
const [activeFilters, setActiveFilters] = useState([]);
// These are currently hardcoded, check the frontmatter of the blog posts to see what audiences are available
const audiences = ["Data Governance Leads", "Data Engineers", "Data Architects", "Data Platform Leads", "Data Analysts"];

const filteredItems = activeFilters?.length
? (items || []).filter((post) => activeFilters.some((activeFilter) => post?.content?.frontMatter?.audience?.some((a) => a === activeFilter)))
: items;

const handleFilterToggle = (audience) => {
if (activeFilters.includes(audience)) {
setActiveFilters(activeFilters.filter((filter) => filter !== audience));
} else {
setActiveFilters([...new Set([...activeFilters, audience])]);
}
};

return (
<Layout>
<header className={"hero"}>
<div className="container">
<div className="hero__content">
<div>
<h1 className="hero__title">DataHub Learn</h1>
<p className="hero__subtitle">Learn about the hot topics in the data ecosystem and how DataHub can help you with your data journey.</p>
</div>
</div>
<div className={styles.filterBar}>
<strong>For: </strong>
{audiences.map((audience) => (
<button
className={`button button--secondary ${activeFilters.includes(audience) && "button--active"}`}
onClick={() => handleFilterToggle(audience)}
>
{audience}
</button>
))}
</div>
</div>
</header>
<div className="container">
<div className="row">
{(filteredItems || []).map(({ content: BlogPostContent }) => (
<BlogPostProvider key={BlogPostContent.metadata.permalink} content={BlogPostContent}>
<LearnItemCard />
</BlogPostProvider>
))}
</div>
<BlogListPaginator metadata={metadata} />
</div>
</Layout>
);
}

export default function BlogListPage(props) {
return (
<HtmlClassNameProvider className={clsx(ThemeClassNames.wrapper.blogPages, ThemeClassNames.page.blogListPage)}>
<BlogListPageMetadata {...props} />
<BlogListPageContent {...props} />
</HtmlClassNameProvider>
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
.filterBar {
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
flex-wrap: wrap;
}
Loading

0 comments on commit 049776b

Please sign in to comment.