Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Jan 29, 2024
2 parents 492560c + 2bb4b73 commit 65cc070
Show file tree
Hide file tree
Showing 133 changed files with 4,152 additions and 841 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/airflow-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
directory: .
fail_ci_if_error: false
flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
name: pytest-airflow
flags: airflow,airflow-${{ matrix.extra_pip_extras }}
name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
verbose: true

event-file:
Expand Down
18 changes: 17 additions & 1 deletion .github/workflows/spark-smoke-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,12 @@ jobs:
cache: "pip"
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Disk Check
run: df -h . && docker images
- name: Remove images
run: docker image prune -a -f || true
- name: Disk Check
run: df -h . && docker images
- name: Smoke test
run: |
./gradlew :metadata-integration:java:spark-lineage:integrationTest \
Expand All @@ -54,12 +58,24 @@ jobs:
-x :datahub-web-react:yarnBuild \
-x :datahub-web-react:distZip \
-x :datahub-web-react:jar
- name: store logs
if: failure()
run: |
docker ps -a
docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: docker logs
path: "docker/build/container-logs/*.log"
path: |
"**/build/container-logs/*.log"
"*.log"
- uses: actions/upload-artifact@v3
if: always()
with:
Expand Down
3 changes: 3 additions & 0 deletions datahub-frontend/app/auth/AuthUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ public class AuthUtils {
public static final String USE_NONCE = "useNonce";
public static final String READ_TIMEOUT = "readTimeout";
public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
// Retained for backwards compatibility
public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";
public static final String PREFERRED_JWS_ALGORITHM_2 = "preferredJwsAlgorithm2";

/**
* Determines whether the inbound request should be forward to downstream Metadata Service. Today,
Expand Down
4 changes: 2 additions & 2 deletions datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,8 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
extractJwtAccessTokenClaims =
Optional.of(jsonNode.get(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).asBoolean());
}
if (jsonNode.has(OIDC_PREFERRED_JWS_ALGORITHM)) {
preferredJwsAlgorithm = Optional.of(jsonNode.get(OIDC_PREFERRED_JWS_ALGORITHM).asText());
if (jsonNode.has(PREFERRED_JWS_ALGORITHM_2)) {
preferredJwsAlgorithm = Optional.of(jsonNode.get(PREFERRED_JWS_ALGORITHM_2).asText());
} else {
preferredJwsAlgorithm =
Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
Expand Down
3 changes: 3 additions & 0 deletions datahub-frontend/play.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ play {
test {
useJUnitPlatform()

testLogging.showStandardStreams = true
testLogging.exceptionFormat = 'full'

def playJava17CompatibleJvmArgs = [
"--add-opens=java.base/java.lang=ALL-UNNAMED",
//"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
Expand Down
24 changes: 24 additions & 0 deletions datahub-frontend/test/security/OidcConfigurationTest.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package security;

import static auth.AuthUtils.*;
import static auth.sso.oidc.OidcConfigs.*;
import static org.junit.jupiter.api.Assertions.assertEquals;

Expand All @@ -24,6 +25,7 @@
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import org.pac4j.oidc.client.OidcClient;
import org.json.JSONObject;

public class OidcConfigurationTest {

Expand Down Expand Up @@ -317,4 +319,26 @@ public void readTimeoutPropagation() {
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
assertEquals(10000, ((OidcClient) oidcProvider.client()).getConfiguration().getReadTimeout());
}

@Test
public void readPreferredJwsAlgorithmPropagationFromConfig() {
final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString();
CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
assertEquals("RS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
}

@Test
public void readPreferredJwsAlgorithmPropagationFromJSON() {
final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "Unused").put(PREFERRED_JWS_ALGORITHM_2, "HS256").toString();
CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
assertEquals("HS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ private static com.linkedin.datahub.graphql.generated.AssertionInfo mapAssertion
mapDatasetAssertionInfo(gmsAssertionInfo.getDatasetAssertion());
assertionInfo.setDatasetAssertion(datasetAssertion);
}
assertionInfo.setDescription(gmsAssertionInfo.getDescription());
return assertionInfo;
}

Expand Down
5 changes: 5 additions & 0 deletions datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -6803,6 +6803,11 @@ type AssertionInfo {
Dataset-specific assertion information
"""
datasetAssertion: DatasetAssertionInfo

"""
An optional human-readable description of the assertion
"""
description: String
}

"""
Expand Down
34 changes: 8 additions & 26 deletions datahub-web-react/src/App.tsx
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
import React, { useEffect, useState } from 'react';
import React from 'react';
import Cookies from 'js-cookie';
import { message } from 'antd';
import { BrowserRouter as Router } from 'react-router-dom';
import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client';
import { onError } from '@apollo/client/link/error';
import { ThemeProvider } from 'styled-components';
import { Helmet, HelmetProvider } from 'react-helmet-async';
import './App.less';
import { Routes } from './app/Routes';
import { Theme } from './conf/theme/types';
import defaultThemeConfig from './conf/theme/theme_light.config.json';
import { PageRoutes } from './conf/Global';
import { isLoggedInVar } from './app/auth/checkAuthStatus';
import { GlobalCfg } from './conf';
import possibleTypesResult from './possibleTypes.generated';
import { ErrorCodes } from './app/shared/constants';
import CustomThemeProvider from './CustomThemeProvider';
import { useCustomTheme } from './customThemeContext';

/*
Construct Apollo Client
Expand Down Expand Up @@ -71,33 +70,16 @@ const client = new ApolloClient({
});

export const InnerApp: React.VFC = () => {
const [dynamicThemeConfig, setDynamicThemeConfig] = useState<Theme>(defaultThemeConfig);

useEffect(() => {
if (import.meta.env.DEV) {
import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
setDynamicThemeConfig(theme);
});
} else {
// Send a request to the server to get the theme config.
fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
.then((response) => response.json())
.then((theme) => {
setDynamicThemeConfig(theme);
});
}
}, []);

return (
<HelmetProvider>
<Helmet>
<title>{dynamicThemeConfig.content.title}</title>
</Helmet>
<ThemeProvider theme={dynamicThemeConfig}>
<CustomThemeProvider>
<Helmet>
<title>{useCustomTheme().theme?.content.title}</title>
</Helmet>
<Router>
<Routes />
</Router>
</ThemeProvider>
</CustomThemeProvider>
</HelmetProvider>
);
};
Expand Down
32 changes: 32 additions & 0 deletions datahub-web-react/src/CustomThemeProvider.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import React, { useEffect, useState } from 'react';
import { ThemeProvider } from 'styled-components';
import { Theme } from './conf/theme/types';
import defaultThemeConfig from './conf/theme/theme_light.config.json';
import { CustomThemeContext } from './customThemeContext';

const CustomThemeProvider = ({ children }: { children: React.ReactNode }) => {
const [currentTheme, setTheme] = useState<Theme>(defaultThemeConfig);

useEffect(() => {
if (import.meta.env.DEV) {
import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
setTheme(theme);
});
} else {
// Send a request to the server to get the theme config.
fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
.then((response) => response.json())
.then((theme) => {
setTheme(theme);
});
}
}, []);

return (
<CustomThemeContext.Provider value={{ theme: currentTheme, updateTheme: setTheme }}>
<ThemeProvider theme={currentTheme}>{children}</ThemeProvider>
</CustomThemeContext.Provider>
);
};

export default CustomThemeProvider;
5 changes: 5 additions & 0 deletions datahub-web-react/src/Mocks.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ export const dataset1 = {
embed: null,
browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' },
autoRenderAspects: [],
structuredProperties: null,
};

export const dataset2 = {
Expand Down Expand Up @@ -393,6 +394,7 @@ export const dataset2 = {
embed: null,
browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' },
autoRenderAspects: [],
structuredProperties: null,
};

export const dataset3 = {
Expand Down Expand Up @@ -626,6 +628,7 @@ export const dataset3 = {
dataProduct: null,
lastProfile: null,
lastOperation: null,
structuredProperties: null,
} as Dataset;

export const dataset3WithSchema = {
Expand All @@ -650,6 +653,7 @@ export const dataset3WithSchema = {
globalTags: null,
glossaryTerms: null,
label: 'hi',
schemaFieldEntity: null,
},
{
__typename: 'SchemaField',
Expand All @@ -665,6 +669,7 @@ export const dataset3WithSchema = {
globalTags: null,
glossaryTerms: null,
label: 'hi',
schemaFieldEntity: null,
},
],
hash: '',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ type Props = {
description: string,
) => Promise<FetchResult<UpdateDatasetMutation, Record<string, any>, Record<string, any>> | void>;
isEdited?: boolean;
isReadOnly?: boolean;
};

const ABBREVIATED_LIMIT = 80;
Expand All @@ -97,10 +98,11 @@ export default function DescriptionField({
onUpdate,
isEdited = false,
original,
isReadOnly,
}: Props) {
const [showAddModal, setShowAddModal] = useState(false);
const overLimit = removeMarkdown(description).length > 80;
const isSchemaEditable = React.useContext(SchemaEditableContext);
const isSchemaEditable = React.useContext(SchemaEditableContext) && !isReadOnly;
const onCloseModal = () => setShowAddModal(false);
const { urn, entityType } = useEntityData();

Expand Down Expand Up @@ -140,11 +142,12 @@ export default function DescriptionField({
{expanded || !overLimit ? (
<>
{!!description && <StyledViewer content={description} readOnly />}
{!!description && (
{!!description && (EditButton || overLimit) && (
<ExpandedActions>
{overLimit && (
<ReadLessText
onClick={() => {
onClick={(e) => {
e.stopPropagation();
handleExpanded(false);
}}
>
Expand All @@ -162,7 +165,8 @@ export default function DescriptionField({
readMore={
<>
<Typography.Link
onClick={() => {
onClick={(e) => {
e.stopPropagation();
handleExpanded(true);
}}
>
Expand All @@ -177,7 +181,7 @@ export default function DescriptionField({
</StripMarkdownText>
</>
)}
{isSchemaEditable && isEdited && <EditedLabel>(edited)</EditedLabel>}
{isEdited && <EditedLabel>(edited)</EditedLabel>}
{showAddModal && (
<div>
<UpdateDescriptionModal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export default function UpdateDescriptionModal({ title, description, original, o
>
<Form layout="vertical">
<Form.Item>
<StyledEditor content={updatedDesc} onChange={setDesc} />
<StyledEditor content={updatedDesc} onChange={setDesc} dataTestId="description-editor" />
</Form.Item>
{!isAddDesc && description && original && (
<Form.Item label={<FormLabel>Original:</FormLabel>}>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import React from 'react';
import { useEntityRegistry } from '../../../../useEntityRegistry';
import { PlatformIcon } from '../../../../search/filters/utils';
import { Entity } from '../../../../../types.generated';
import { IconStyleType } from '../../../Entity';
import { ANTD_GRAY } from '../../constants';

interface Props {
entity: Entity;
size?: number;
}

export default function EntityIcon({ entity, size = 14 }: Props) {
const entityRegistry = useEntityRegistry();
const genericEntityProps = entityRegistry.getGenericEntityProperties(entity.type, entity);
const logoUrl = genericEntityProps?.platform?.properties?.logoUrl;
const icon = logoUrl ? (
<PlatformIcon src={logoUrl} size={size} />
) : (
entityRegistry.getIcon(entity.type, size, IconStyleType.ACCENT, ANTD_GRAY[9])
);

return <>{icon}</>;
}
Loading

0 comments on commit 65cc070

Please sign in to comment.