Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
anshbansal authored Feb 26, 2024
2 parents e8cda87 + 5921a33 commit 8846e33
Show file tree
Hide file tree
Showing 241 changed files with 12,657 additions and 3,357 deletions.
8 changes: 0 additions & 8 deletions .github/workflows/docker-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,9 @@ on:
push:
branches:
- master
paths-ignore:
- "docs-website/**"
- "docs/**"
- "**.md"
pull_request:
branches:
- "**"
paths-ignore:
- "docs-website/**"
- "docs/**"
- "**.md"
release:
types: [published]

Expand Down
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ plugins {
id 'com.gorylenko.gradle-git-properties' version '2.4.1'
id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
id 'com.palantir.docker' version '0.35.0' apply false
id 'com.avast.gradle.docker-compose' version '0.17.5'
id 'com.avast.gradle.docker-compose' version '0.17.6'
id "com.diffplug.spotless" version "6.23.3"
// https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
// TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
import com.linkedin.datahub.graphql.resolvers.entity.EntityExistsResolver;
import com.linkedin.datahub.graphql.resolvers.entity.EntityPrivilegesResolver;
import com.linkedin.datahub.graphql.resolvers.form.BatchAssignFormResolver;
import com.linkedin.datahub.graphql.resolvers.form.BatchRemoveFormResolver;
import com.linkedin.datahub.graphql.resolvers.form.CreateDynamicFormAssignmentResolver;
import com.linkedin.datahub.graphql.resolvers.form.IsFormAssignedToMeResolver;
import com.linkedin.datahub.graphql.resolvers.form.SubmitFormPromptResolver;
Expand Down Expand Up @@ -1214,6 +1215,7 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
new CreateDynamicFormAssignmentResolver(this.formService))
.dataFetcher(
"verifyForm", new VerifyFormResolver(this.formService, this.groupService))
.dataFetcher("batchRemoveForm", new BatchRemoveFormResolver(this.formService))
.dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
.dataFetcher(
"updateIncidentStatus",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package com.linkedin.datahub.graphql.resolvers.form;

import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;

import com.datahub.authentication.Authentication;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.BatchAssignFormInput;
import com.linkedin.metadata.service.FormService;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;

public class BatchRemoveFormResolver implements DataFetcher<CompletableFuture<Boolean>> {

private final FormService _formService;

public BatchRemoveFormResolver(@Nonnull final FormService formService) {
_formService = Objects.requireNonNull(formService, "formService must not be null");
}

@Override
public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
throws Exception {
final QueryContext context = environment.getContext();

final BatchAssignFormInput input =
bindArgument(environment.getArgument("input"), BatchAssignFormInput.class);
final Urn formUrn = UrnUtils.getUrn(input.getFormUrn());
final List<String> entityUrns = input.getEntityUrns();
final Authentication authentication = context.getAuthentication();

// TODO: (PRD-1062) Add permission check once permission exists

return CompletableFuture.supplyAsync(
() -> {
try {
_formService.batchUnassignFormForEntities(
entityUrns.stream().map(UrnUtils::getUrn).collect(Collectors.toList()),
formUrn,
authentication);
return true;
} catch (Exception e) {
throw new RuntimeException(
String.format("Failed to perform update against input %s", input), e);
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import com.google.common.collect.ImmutableSet;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.generated.AndFilterInput;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.FacetFilterInput;
import com.linkedin.datahub.graphql.generated.LineageDirection;
Expand Down Expand Up @@ -92,9 +93,14 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment

final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
final List<FacetFilterInput> filters =
input.getFilters() != null ? input.getFilters() : new ArrayList<>();
final Integer maxHops = getMaxHops(filters);
final List<AndFilterInput> filters =
input.getOrFilters() != null ? input.getOrFilters() : new ArrayList<>();
final List<FacetFilterInput> facetFilters =
filters.stream()
.map(AndFilterInput::getAnd)
.flatMap(List::stream)
.collect(Collectors.toList());
final Integer maxHops = getMaxHops(facetFilters);

@Nullable
final Long startTimeMillis =
Expand All @@ -117,7 +123,8 @@ public CompletableFuture<SearchAcrossLineageResults> get(DataFetchingEnvironment
start,
count);

final Filter filter = ResolverUtils.buildFilter(filters, input.getOrFilters());
final Filter filter =
ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters());
SearchFlags searchFlags = null;
com.linkedin.datahub.graphql.generated.SearchFlags inputFlags = input.getSearchFlags();
if (inputFlags != null) {
Expand Down
22 changes: 22 additions & 0 deletions datahub-graphql-core/src/main/resources/forms.graphql
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
extend type Mutation {
"""
Remove a form from a given list of entities.
"""
batchRemoveForm(input: BatchRemoveFormInput!): Boolean!
}

"""
Requirements forms that are assigned to an entity.
"""
Expand Down Expand Up @@ -376,3 +383,18 @@ input VerifyFormInput {
"""
entityUrn: String!
}

"""
Input for batch removing a form from different entities
"""
input BatchRemoveFormInput {
"""
The urn of the form being removed from entities
"""
formUrn: String!

"""
The entities that this form is being removed from
"""
entityUrns: [String!]!
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
import com.linkedin.metadata.models.registry.ConfigEntityRegistry;
import com.linkedin.metadata.models.registry.EntityRegistry;
import com.linkedin.mxe.MetadataChangeProposal;
Expand All @@ -22,14 +22,14 @@

public class TestUtils {

public static EntityService<MCPUpsertBatchItem> getMockEntityService() {
public static EntityService<ChangeItemImpl> getMockEntityService() {
PathSpecBasedSchemaAnnotationVisitor.class
.getClassLoader()
.setClassAssertionStatus(PathSpecBasedSchemaAnnotationVisitor.class.getName(), false);
EntityRegistry registry =
new ConfigEntityRegistry(TestUtils.class.getResourceAsStream("/test-entity-registry.yaml"));
EntityService<MCPUpsertBatchItem> mockEntityService =
(EntityService<MCPUpsertBatchItem>) Mockito.mock(EntityService.class);
EntityService<ChangeItemImpl> mockEntityService =
(EntityService<ChangeItemImpl>) Mockito.mock(EntityService.class);
Mockito.when(mockEntityService.getEntityRegistry()).thenReturn(registry);
return mockEntityService;
}
Expand Down Expand Up @@ -111,14 +111,14 @@ public static QueryContext getMockDenyContext(String actorUrn, AuthorizationRequ
}

public static void verifyIngestProposal(
EntityService<MCPUpsertBatchItem> mockService,
EntityService<ChangeItemImpl> mockService,
int numberOfInvocations,
MetadataChangeProposal proposal) {
verifyIngestProposal(mockService, numberOfInvocations, List.of(proposal));
}

public static void verifyIngestProposal(
EntityService<MCPUpsertBatchItem> mockService,
EntityService<ChangeItemImpl> mockService,
int numberOfInvocations,
List<MetadataChangeProposal> proposals) {
AspectsBatchImpl batch =
Expand All @@ -128,29 +128,29 @@ public static void verifyIngestProposal(
}

public static void verifySingleIngestProposal(
EntityService<MCPUpsertBatchItem> mockService,
EntityService<ChangeItemImpl> mockService,
int numberOfInvocations,
MetadataChangeProposal proposal) {
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
.ingestProposal(Mockito.eq(proposal), Mockito.any(AuditStamp.class), Mockito.eq(false));
}

public static void verifyIngestProposal(
EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
.ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.eq(false));
}

public static void verifySingleIngestProposal(
EntityService<MCPUpsertBatchItem> mockService, int numberOfInvocations) {
EntityService<ChangeItemImpl> mockService, int numberOfInvocations) {
Mockito.verify(mockService, Mockito.times(numberOfInvocations))
.ingestProposal(
Mockito.any(MetadataChangeProposal.class),
Mockito.any(AuditStamp.class),
Mockito.eq(false));
}

public static void verifyNoIngestProposal(EntityService<MCPUpsertBatchItem> mockService) {
public static void verifyNoIngestProposal(EntityService<ChangeItemImpl> mockService) {
Mockito.verify(mockService, Mockito.times(0))
.ingestProposal(Mockito.any(AspectsBatchImpl.class), Mockito.anyBoolean());
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package com.linkedin.datahub.graphql.resolvers.form;

import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext;
import static org.testng.Assert.assertThrows;
import static org.testng.Assert.assertTrue;

import com.datahub.authentication.Authentication;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.generated.BatchAssignFormInput;
import com.linkedin.metadata.service.FormService;
import graphql.com.google.common.collect.ImmutableList;
import graphql.schema.DataFetchingEnvironment;
import java.util.concurrent.CompletionException;
import org.mockito.Mockito;
import org.testng.annotations.Test;

public class BatchRemoveFormResolverTest {

private static final String TEST_DATASET_URN =
"urn:li:dataset:(urn:li:dataPlatform:hive,name,PROD)";
private static final String TEST_FORM_URN = "urn:li:form:1";

private static final BatchAssignFormInput TEST_INPUT =
new BatchAssignFormInput(TEST_FORM_URN, ImmutableList.of(TEST_DATASET_URN));

@Test
public void testGetSuccess() throws Exception {
FormService mockFormService = initMockFormService(true);
BatchRemoveFormResolver resolver = new BatchRemoveFormResolver(mockFormService);

// Execute resolver
QueryContext mockContext = getMockAllowContext();
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);

boolean success = resolver.get(mockEnv).get();

assertTrue(success);

// Validate that we called unassign on the service
Mockito.verify(mockFormService, Mockito.times(1))
.batchUnassignFormForEntities(
Mockito.eq(ImmutableList.of(UrnUtils.getUrn(TEST_DATASET_URN))),
Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)),
Mockito.any(Authentication.class));
}

@Test
public void testThrowsError() throws Exception {
FormService mockFormService = initMockFormService(false);
BatchRemoveFormResolver resolver = new BatchRemoveFormResolver(mockFormService);

// Execute resolver
QueryContext mockContext = getMockAllowContext();
DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class);
Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT);
Mockito.when(mockEnv.getContext()).thenReturn(mockContext);

assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());

// Validate that we called unassign on the service - but it throws an error
Mockito.verify(mockFormService, Mockito.times(1))
.batchUnassignFormForEntities(
Mockito.eq(ImmutableList.of(UrnUtils.getUrn(TEST_DATASET_URN))),
Mockito.eq(UrnUtils.getUrn(TEST_FORM_URN)),
Mockito.any(Authentication.class));
}

private FormService initMockFormService(final boolean shouldSucceed) throws Exception {
FormService service = Mockito.mock(FormService.class);

if (!shouldSucceed) {
Mockito.doThrow(new RuntimeException())
.when(service)
.batchUnassignFormForEntities(
Mockito.any(), Mockito.any(), Mockito.any(Authentication.class));
}

return service;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl;
import com.linkedin.metadata.entity.ebean.batch.MCPUpsertBatchItem;
import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl;
import com.linkedin.mxe.MetadataChangeProposal;
import graphql.schema.DataFetchingEnvironment;
import java.util.concurrent.CompletionException;
Expand Down Expand Up @@ -221,7 +221,7 @@ public void testGetUnauthorized() throws Exception {

@Test
public void testGetEntityClientException() throws Exception {
EntityService<MCPUpsertBatchItem> mockService = getMockEntityService();
EntityService<ChangeItemImpl> mockService = getMockEntityService();

Mockito.doThrow(RuntimeException.class)
.when(mockService)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,9 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
final RecordTemplate aspectRecord;
try {
aspectRecord =
EntityUtils.toAspectRecord(
entityName, aspectName, aspect.getMetadata(), _entityRegistry);
EntityUtils.toSystemAspect(aspect.toEntityAspect(), _entityService)
.get()
.getRecordTemplate();
} catch (Exception e) {
context
.report()
Expand Down
Loading

0 comments on commit 8846e33

Please sign in to comment.