From ae48bd14de8182704cbaf1fba86abb4e1a16aba4 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 23 Apr 2024 13:44:02 -0500 Subject: [PATCH 1/7] fix(mae): fix mae standalone platform consumer (#10352) --- .../BusinessAttributeUpdateHookService.java | 146 ++++++----- metadata-jobs/pe-consumer/build.gradle | 1 + .../hook/BusinessAttributeUpdateHook.java | 20 +- .../hook/BusinessAttributeUpdateHookTest.java | 236 ++++++++++-------- .../src/main/resources/application.yaml | 3 +- 5 files changed, 235 insertions(+), 171 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java index 63dd2fede8680..32fbff73a1d20 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/BusinessAttributeUpdateHookService.java @@ -6,27 +6,31 @@ import com.google.common.collect.ImmutableSet; import com.linkedin.businessattribute.BusinessAttributes; -import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; -import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.Aspect; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.models.graph.Edge; +import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.aspect.models.graph.RelatedEntity; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.graph.RelatedEntitiesResult; -import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.PlatformEvent; import com.linkedin.platform.event.v1.EntityChangeEvent; import io.datahubproject.metadata.context.OperationContext; import java.util.Arrays; +import java.util.Map; import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; @Slf4j @@ -34,25 +38,21 @@ public class BusinessAttributeUpdateHookService { private static final String BUSINESS_ATTRIBUTE_OF = "BusinessAttributeOf"; - private final GraphService graphService; - private final EntityService entityService; - private final EntityRegistry entityRegistry; - + private final UpdateIndicesService updateIndicesService; private final int relatedEntitiesCount; + private final int getRelatedEntitiesBatchSize; public static final String TAG = "TAG"; public static final String GLOSSARY_TERM = "GLOSSARY_TERM"; public static final String DOCUMENTATION = "DOCUMENTATION"; public BusinessAttributeUpdateHookService( - GraphService graphService, - EntityService entityService, - EntityRegistry entityRegistry, - @NonNull @Value("${businessAttribute.fetchRelatedEntitiesCount}") int relatedEntitiesCount) { - this.graphService = graphService; - this.entityService = entityService; - this.entityRegistry = entityRegistry; + @NonNull UpdateIndicesService updateIndicesService, + @NonNull @Value("${businessAttribute.fetchRelatedEntitiesCount}") int relatedEntitiesCount, + @NonNull @Value("${businessAttribute.fetchRelatedEntitiesBatchSize}") int relatedBatchSize) { + this.updateIndicesService = updateIndicesService; this.relatedEntitiesCount = relatedEntitiesCount; + this.getRelatedEntitiesBatchSize = relatedBatchSize; } public void handleChangeEvent( @@ -76,58 +76,76 @@ public void handleChangeEvent( Urn urn = entityChangeEvent.getEntityUrn(); log.info("Business Attribute update hook invoked for urn :" + urn); - RelatedEntitiesResult entityAssociatedWithBusinessAttribute = - graphService.findRelatedEntities( + fetchRelatedEntities(opContext, urn, batch -> processBatch(opContext, batch), null, 0); + } + + private void fetchRelatedEntities( + @NonNull final OperationContext opContext, + @NonNull final Urn urn, + @NonNull final Consumer resultConsumer, + @Nullable String scrollId, + int consumedEntityCount) { + GraphRetriever graph = opContext.getRetrieverContext().get().getGraphRetriever(); + + RelatedEntitiesScrollResult result = + graph.scrollRelatedEntities( null, newFilter("urn", urn.toString()), null, EMPTY_FILTER, Arrays.asList(BUSINESS_ATTRIBUTE_OF), newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), - 0, - relatedEntitiesCount); - - for (RelatedEntity relatedEntity : entityAssociatedWithBusinessAttribute.getEntities()) { - String entityUrnStr = relatedEntity.getUrn(); - try { - Urn entityUrn = new Urn(entityUrnStr); - final AspectSpec aspectSpec = - entityRegistry - .getEntitySpec(Constants.SCHEMA_FIELD_ENTITY_NAME) - .getAspectSpec(Constants.BUSINESS_ATTRIBUTE_ASPECT); - - EnvelopedAspect envelopedAspect = - entityService.getLatestEnvelopedAspect( - opContext, - Constants.SCHEMA_FIELD_ENTITY_NAME, - entityUrn, - Constants.BUSINESS_ATTRIBUTE_ASPECT); - BusinessAttributes businessAttributes = - new BusinessAttributes(envelopedAspect.getValue().data()); - - final AuditStamp auditStamp = - new AuditStamp() - .setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)) - .setTime(System.currentTimeMillis()); - - entityService - .alwaysProduceMCLAsync( - opContext, - entityUrn, - Constants.SCHEMA_FIELD_ENTITY_NAME, - Constants.BUSINESS_ATTRIBUTE_ASPECT, - aspectSpec, - null, - businessAttributes, - null, - null, - auditStamp, - ChangeType.UPSERT) - .getFirst(); - - } catch (Exception e) { - throw new RuntimeException(e); - } + Edge.EDGE_SORT_CRITERION, + scrollId, + getRelatedEntitiesBatchSize, + null, + null); + resultConsumer.accept(result); + + if (result.getScrollId() != null && consumedEntityCount < relatedEntitiesCount) { + fetchRelatedEntities( + opContext, + urn, + resultConsumer, + result.getScrollId(), + consumedEntityCount + result.getEntities().size()); } } + + private void processBatch( + @NonNull OperationContext opContext, @NonNull RelatedEntitiesScrollResult batch) { + AspectRetriever aspectRetriever = opContext.getRetrieverContext().get().getAspectRetriever(); + + Set entityUrns = + batch.getEntities().stream() + .map(RelatedEntity::getUrn) + .map(UrnUtils::getUrn) + .collect(Collectors.toSet()); + + Map> entityAspectMap = + aspectRetriever.getLatestAspectObjects( + entityUrns, Set.of(Constants.BUSINESS_ATTRIBUTE_ASPECT)); + + entityAspectMap.entrySet().stream() + .filter(entry -> entry.getValue().containsKey(Constants.BUSINESS_ATTRIBUTE_ASPECT)) + .forEach( + entry -> { + final Urn entityUrn = entry.getKey(); + final Aspect aspect = entry.getValue().get(Constants.BUSINESS_ATTRIBUTE_ASPECT); + + updateIndicesService.handleChangeEvent( + opContext, + PegasusUtils.constructMCL( + null, + Constants.SCHEMA_FIELD_ENTITY_NAME, + entityUrn, + ChangeType.UPSERT, + Constants.BUSINESS_ATTRIBUTE_ASPECT, + opContext.getAuditStamp(), + new BusinessAttributes(aspect.data()), + null, + null, + null)); + }); + } } diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 5c031dafba4da..35fdf0231c406 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -25,6 +25,7 @@ dependencies { testRuntimeOnly externalDependency.logbackClassic testImplementation externalDependency.springBootTest testImplementation externalDependency.testng + testImplementation project(':metadata-operation-context') } task avroSchemaSources(type: Copy) { diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java index 7f6fc76e9ebb3..c98de632c9118 100644 --- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/hook/BusinessAttributeUpdateHook.java @@ -1,26 +1,38 @@ package com.datahub.event.hook; import com.linkedin.gms.factory.common.GraphServiceFactory; -import com.linkedin.gms.factory.entity.EntityServiceFactory; -import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.metadata.service.BusinessAttributeUpdateHookService; import com.linkedin.mxe.PlatformEvent; import io.datahubproject.metadata.context.OperationContext; import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Import; import org.springframework.stereotype.Component; @Slf4j @Component -@Import({EntityServiceFactory.class, EntityRegistryFactory.class, GraphServiceFactory.class}) +@Import(GraphServiceFactory.class) public class BusinessAttributeUpdateHook implements PlatformEventHook { protected final BusinessAttributeUpdateHookService businessAttributeUpdateHookService; + protected final boolean enabled; public BusinessAttributeUpdateHook( - BusinessAttributeUpdateHookService businessAttributeUpdateHookService) { + BusinessAttributeUpdateHookService businessAttributeUpdateHookService, + @Value("${featureFlags.businessAttributeEntityEnabled}") boolean enabled) { this.businessAttributeUpdateHookService = businessAttributeUpdateHookService; + this.enabled = enabled; + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public void init() { + log.info("Initialized PlatformEventHook: BusinessAttributeUpdateHook"); } /** diff --git a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java index 818903414dccd..9db3a77e710a3 100644 --- a/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java +++ b/metadata-jobs/pe-consumer/src/test/java/com/datahub/event/hook/BusinessAttributeUpdateHookTest.java @@ -1,15 +1,14 @@ package com.datahub.event.hook; -import static com.datahub.event.hook.EntityRegistryTestUtil.ENTITY_REGISTRY; import static com.linkedin.metadata.Constants.BUSINESS_ATTRIBUTE_ASPECT; -import static com.linkedin.metadata.Constants.SCHEMA_FIELD_ENTITY_NAME; -import static com.linkedin.metadata.search.utils.QueryUtils.EMPTY_FILTER; -import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; -import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; -import static org.testng.Assert.assertEquals; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -23,31 +22,36 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.DataMap; import com.linkedin.entity.Aspect; -import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.models.graph.Edge; +import com.linkedin.metadata.aspect.models.graph.RelatedEntities; +import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.aspect.models.graph.RelatedEntity; -import com.linkedin.metadata.entity.EntityService; -import com.linkedin.metadata.graph.GraphService; -import com.linkedin.metadata.graph.RelatedEntitiesResult; -import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.query.filter.RelationshipFilter; import com.linkedin.metadata.service.BusinessAttributeUpdateHookService; +import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.timeline.data.ChangeCategory; import com.linkedin.metadata.timeline.data.ChangeOperation; import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.PlatformEvent; import com.linkedin.mxe.PlatformEventHeader; -import com.linkedin.mxe.SystemMetadata; import com.linkedin.platform.event.v1.EntityChangeEvent; import com.linkedin.platform.event.v1.Parameters; -import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.metadata.context.RetrieverContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.List; import java.util.Map; -import java.util.concurrent.Future; +import java.util.Set; import org.mockito.Mockito; +import org.mockito.stubbing.OngoingStubbing; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -63,118 +67,94 @@ public class BusinessAttributeUpdateHookTest { private static final long EVENT_TIME = 123L; private static final String TEST_ACTOR_URN = "urn:li:corpuser:test"; private static Urn actorUrn; - private GraphService mockGraphService; - private EntityService mockEntityService; + private UpdateIndicesService mockUpdateIndicesService; private BusinessAttributeUpdateHook businessAttributeUpdateHook; private BusinessAttributeUpdateHookService businessAttributeServiceHook; @BeforeMethod public void setupTest() throws URISyntaxException { - mockGraphService = mock(GraphService.class); - mockEntityService = mock(EntityService.class); + mockUpdateIndicesService = mock(UpdateIndicesService.class); actorUrn = Urn.createFromString(TEST_ACTOR_URN); businessAttributeServiceHook = - new BusinessAttributeUpdateHookService( - mockGraphService, mockEntityService, ENTITY_REGISTRY, 100); - businessAttributeUpdateHook = new BusinessAttributeUpdateHook(businessAttributeServiceHook); + new BusinessAttributeUpdateHookService(mockUpdateIndicesService, 100, 1); + businessAttributeUpdateHook = + new BusinessAttributeUpdateHook(businessAttributeServiceHook, true); } @Test public void testMCLOnBusinessAttributeUpdate() throws Exception { PlatformEvent platformEvent = createPlatformEventBusinessAttribute(); - final RelatedEntitiesResult mockRelatedEntities = - new RelatedEntitiesResult( - 0, - 1, - 1, + + // mock response + OperationContext opContext = + mockOperationContextWithGraph( ImmutableList.of( + new RelatedEntity(BUSINESS_ATTRIBUTE_OF, SCHEMA_FIELD_URN.toString()), new RelatedEntity(BUSINESS_ATTRIBUTE_OF, SCHEMA_FIELD_URN.toString()))); - // mock response - Mockito.when( - mockGraphService.findRelatedEntities( - null, - newFilter("urn", TEST_BUSINESS_ATTRIBUTE_URN), - null, - EMPTY_FILTER, - Arrays.asList(BUSINESS_ATTRIBUTE_OF), - newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), - 0, - 100)) - .thenReturn(mockRelatedEntities); - assertEquals(mockRelatedEntities.getTotal(), 1); - - Mockito.when( - mockEntityService.getLatestEnvelopedAspect( - any(OperationContext.class), - eq(SCHEMA_FIELD_ENTITY_NAME), - eq(SCHEMA_FIELD_URN), - eq(BUSINESS_ATTRIBUTE_ASPECT))) - .thenReturn(envelopedAspect()); - // mock response - Mockito.when( - mockEntityService.alwaysProduceMCLAsync( - any(OperationContext.class), - any(Urn.class), - Mockito.anyString(), - Mockito.anyString(), - any(AspectSpec.class), - eq(null), - any(), - any(), - any(), - any(), - any(ChangeType.class))) - .thenReturn(Pair.of(mock(Future.class), false)); + when(opContext + .getRetrieverContext() + .get() + .getAspectRetriever() + .getLatestAspectObjects( + eq(Set.of(SCHEMA_FIELD_URN)), eq(Set.of(BUSINESS_ATTRIBUTE_ASPECT)))) + .thenReturn( + Map.of( + SCHEMA_FIELD_URN, + Map.of(BUSINESS_ATTRIBUTE_ASPECT, new Aspect(new BusinessAttributes().data())))); // invoke - businessAttributeServiceHook.handleChangeEvent(mock(OperationContext.class), platformEvent); + businessAttributeServiceHook.handleChangeEvent(opContext, platformEvent); // verify - Mockito.verify(mockGraphService, Mockito.times(1)) - .findRelatedEntities( - any(), any(), any(), any(), any(), any(), Mockito.anyInt(), Mockito.anyInt()); - - Mockito.verify(mockEntityService, Mockito.times(1)) - .alwaysProduceMCLAsync( - any(OperationContext.class), - any(Urn.class), - Mockito.anyString(), - Mockito.anyString(), - any(AspectSpec.class), - eq(null), - any(), - any(), + // page 1 + Mockito.verify(opContext.getRetrieverContext().get().getGraphRetriever(), Mockito.times(1)) + .scrollRelatedEntities( + isNull(), + any(Filter.class), + isNull(), + any(Filter.class), any(), + any(RelationshipFilter.class), + eq(Edge.EDGE_SORT_CRITERION), + isNull(), + anyInt(), + isNull(), + isNull()); + // page 2 + Mockito.verify(opContext.getRetrieverContext().get().getGraphRetriever(), Mockito.times(1)) + .scrollRelatedEntities( + isNull(), + any(Filter.class), + isNull(), + any(Filter.class), any(), - any(ChangeType.class)); + any(RelationshipFilter.class), + eq(Edge.EDGE_SORT_CRITERION), + eq("1"), + anyInt(), + isNull(), + isNull()); + + Mockito.verifyNoMoreInteractions(opContext.getRetrieverContext().get().getGraphRetriever()); + + // 2 pages = 2 ingest proposals + Mockito.verify(mockUpdateIndicesService, Mockito.times(2)) + .handleChangeEvent(any(OperationContext.class), any(MetadataChangeLog.class)); } @Test private void testMCLOnInvalidCategory() throws Exception { PlatformEvent platformEvent = createPlatformEventInvalidCategory(); + OperationContext opContext = mockOperationContextWithGraph(ImmutableList.of()); // invoke - businessAttributeServiceHook.handleChangeEvent(mock(OperationContext.class), platformEvent); + businessAttributeServiceHook.handleChangeEvent(opContext, platformEvent); // verify - Mockito.verify(mockGraphService, Mockito.times(0)) - .findRelatedEntities( - any(), any(), any(), any(), any(), any(), Mockito.anyInt(), Mockito.anyInt()); - - Mockito.verify(mockEntityService, Mockito.times(0)) - .alwaysProduceMCLAsync( - any(OperationContext.class), - any(Urn.class), - Mockito.anyString(), - Mockito.anyString(), - any(AspectSpec.class), - eq(null), - any(), - any(), - any(), - any(), - any(ChangeType.class)); + Mockito.verifyNoInteractions(opContext.getRetrieverContext().get().getGraphRetriever()); + Mockito.verifyNoInteractions(opContext.getRetrieverContext().get().getAspectRetriever()); + Mockito.verifyNoInteractions(mockUpdateIndicesService); } public static PlatformEvent createPlatformEventBusinessAttribute() throws Exception { @@ -241,10 +221,62 @@ private static PlatformEvent createChangeEvent( return platformEvent; } - private EnvelopedAspect envelopedAspect() { - EnvelopedAspect envelopedAspect = new EnvelopedAspect(); - envelopedAspect.setValue(new Aspect(new BusinessAttributes().data())); - envelopedAspect.setSystemMetadata(new SystemMetadata()); - return envelopedAspect; + private OperationContext mockOperationContextWithGraph(List graphEdges) { + GraphRetriever graphRetriever = mock(GraphRetriever.class); + + RetrieverContext mockRetrieverContext = mock(RetrieverContext.class); + when(mockRetrieverContext.getAspectRetriever()).thenReturn(mock(AspectRetriever.class)); + when(mockRetrieverContext.getGraphRetriever()).thenReturn(graphRetriever); + + OperationContext opContext = + TestOperationContexts.systemContextNoSearchAuthorization(mockRetrieverContext); + + // reset mock for test + reset(opContext.getRetrieverContext().get().getAspectRetriever()); + + if (!graphEdges.isEmpty()) { + + int idx = 0; + OngoingStubbing multiStub = + when( + graphRetriever.scrollRelatedEntities( + isNull(), + any(Filter.class), + isNull(), + any(Filter.class), + eq(Arrays.asList(BUSINESS_ATTRIBUTE_OF)), + any(RelationshipFilter.class), + eq(Edge.EDGE_SORT_CRITERION), + nullable(String.class), + eq(1), + isNull(), + isNull())); + + for (RelatedEntity relatedEntity : graphEdges) { + final String scrollId; + if (idx < graphEdges.size() - 1) { + idx += 1; + scrollId = String.valueOf(idx); + } else { + scrollId = null; + } + + multiStub = + multiStub.thenReturn( + new RelatedEntitiesScrollResult( + graphEdges.size(), + 1, + scrollId, + List.of( + new RelatedEntities( + relatedEntity.getRelationshipType(), + relatedEntity.getUrn(), + TEST_BUSINESS_ATTRIBUTE_URN, + RelationshipDirection.INCOMING, + null)))); + } + } + + return opContext; } } diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 414ade52a71d6..c6397c3ce5abb 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -445,4 +445,5 @@ forms: enabled: { $FORMS_HOOK_ENABLED:true } businessAttribute: - fetchRelatedEntitiesCount: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_COUNT:100000} + fetchRelatedEntitiesCount: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_COUNT:20000} + fetchRelatedEntitiesBatchSize: ${BUSINESS_ATTRIBUTE_RELATED_ENTITIES_BATCH_SIZE:1000} From ca2a10e36f9a9b82055be19889fe0c2dc459a152 Mon Sep 17 00:00:00 2001 From: Shubham Jagtap <132359390+shubhamjagtap639@users.noreply.github.com> Date: Wed, 24 Apr 2024 04:34:20 +0530 Subject: [PATCH 2/7] fix(ingestion/qlik): Unable to ingest more than ten spaces (#10228) Co-authored-by: Harshal Sheth --- .../ingestion/source/qlik_sense/qlik_api.py | 19 +++++++++++++------ .../integration/qlik_sense/test_qlik_sense.py | 7 ++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py index abcd0c0ce8353..66a18873d86df 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/qlik_api.py @@ -50,12 +50,19 @@ def _log_http_error(self, message: str) -> Any: def get_spaces(self) -> List[Space]: spaces: List[Space] = [] try: - response = self.session.get(f"{self.rest_api_url}/spaces") - response.raise_for_status() - for space_dict in response.json()[Constant.DATA]: - space = Space.parse_obj(space_dict) - spaces.append(space) - self.spaces[space.id] = space.name + url = f"{self.rest_api_url}/spaces" + while True: + response = self.session.get(url) + response.raise_for_status() + response_dict = response.json() + for space_dict in response_dict[Constant.DATA]: + space = Space.parse_obj(space_dict) + spaces.append(space) + self.spaces[space.id] = space.name + if Constant.NEXT in response_dict[Constant.LINKS]: + url = response_dict[Constant.LINKS][Constant.NEXT][Constant.HREF] + else: + break # Add personal space entity spaces.append(Space.parse_obj(PERSONAL_SPACE_DICT)) self.spaces[PERSONAL_SPACE_DICT[Constant.ID]] = PERSONAL_SPACE_DICT[ diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index 00a50b7dbe54c..818bc5198a5b1 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -49,7 +49,12 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "createdBy": "657b5abe656297cec3d8b205", "updatedAt": "2024-01-09T09:13:38.002Z", } - ] + ], + "links": { + "self": { + "href": "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/spaces" + } + }, }, }, "https://iq37k6byr9lgam8.us.qlikcloud.com/api/v1/items": { From dc3ca789ff35df41474ab583171fd3d9679e62f3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:54:49 -0500 Subject: [PATCH 3/7] docker(ingestion-base): set certificate location for python (#10364) --- docker/datahub-ingestion-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index bfd4ee1143f5e..b7d5a11fdf6d8 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -67,6 +67,7 @@ RUN addgroup --gid 1000 datahub && \ chmod +x /entrypoint.sh USER datahub +ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV VIRTUAL_ENV=/datahub-ingestion/.venv ENV PATH="${VIRTUAL_ENV}/bin:$PATH" RUN python3 -m venv $VIRTUAL_ENV && \ From a5d60b45614777325479816a802e39f2dee31e58 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:55:15 -0500 Subject: [PATCH 4/7] build(ci): unified workflow update 1 (#10353) --- .github/workflows/docker-unified.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 59d5f246e6aee..3d1d4090e4fbd 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -453,7 +453,7 @@ jobs: name: Build and Push DataHub Kafka Setup Docker Image runs-on: ubuntu-latest needs: setup - if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} + if: ${{ needs.setup.outputs.kafka_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 @@ -474,7 +474,7 @@ jobs: name: Build and Push DataHub MySQL Setup Docker Image runs-on: ubuntu-latest needs: setup - if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} + if: ${{ needs.setup.outputs.mysql_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 @@ -495,7 +495,7 @@ jobs: name: Build and Push DataHub Elasticsearch Setup Docker Image runs-on: ubuntu-latest needs: setup - if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || needs.setup.outputs.publish == 'true' }} + if: ${{ needs.setup.outputs.elasticsearch_setup_change == 'true' || (needs.setup.outputs.publish == 'true' || needs.setup.outputs.pr-publish == 'true') }} steps: - name: Check out the repo uses: acryldata/sane-checkout-action@v3 @@ -645,7 +645,7 @@ jobs: runs-on: ubuntu-latest outputs: tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' || needs.setup.outputs.pr-publish != 'true') }} + needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true') }} needs: [setup, datahub_ingestion_base_slim_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: @@ -740,7 +740,7 @@ jobs: runs-on: ubuntu-latest outputs: tag: ${{ steps.tag.outputs.tag }} - needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' || needs.setup.outputs.pr-publish != 'true' ) }} + needs_artifact_download: ${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && ( needs.setup.outputs.publish != 'true' && needs.setup.outputs.pr-publish != 'true' ) }} needs: [setup, datahub_ingestion_base_full_build] if: ${{ needs.setup.outputs.ingestion_change == 'true' || needs.setup.outputs.publish == 'true' }} steps: @@ -870,6 +870,11 @@ jobs: test_strategy: ${{ fromJson(needs.smoke_test_matrix.outputs.matrix) }} if: ${{ always() && !failure() && !cancelled() && needs.smoke_test_matrix.outputs.matrix != '[]' }} steps: + - name: Free up disk space + run: | + sudo apt-get remove 'dotnet-*' azure-cli || true + sudo rm -rf /usr/local/lib/android/ || true + sudo docker image prune -a -f || true - name: Disk Check run: df -h . && docker images - name: Check out the repo @@ -990,6 +995,8 @@ jobs: docker tag '${{ env.DATAHUB_INGESTION_IMAGE }}:head-slim' '${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }}' fi fi + - name: Disk Check + run: df -h . && docker images - name: run quickstart env: DATAHUB_TELEMETRY_ENABLED: false From ec21b018f1304c2f8352941240a93a607247f052 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Tue, 23 Apr 2024 17:43:30 -0700 Subject: [PATCH 5/7] feat(ui): Adding new analytics charts for new users, top users past month (#10344) Co-authored-by: John Joyce Co-authored-by: John Joyce --- .../analytics/resolver/GetChartsResolver.java | 246 ++++++++++++++---- .../GetMetadataAnalyticsResolver.java | 10 +- .../analytics/service/AnalyticsUtil.java | 118 ++++++++- .../datahub/graphql/util/DateUtil.java | 7 + .../java/com/linkedin/metadata/Constants.java | 1 + .../com/linkedin/identity/CorpUserStatus.pdl | 6 + .../cypress/e2e/analytics/analytics.js | 4 +- 7 files changed, 332 insertions(+), 60 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java index 394a305e3f4d0..767c9b4d4e71b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java @@ -1,5 +1,8 @@ package com.linkedin.datahub.graphql.analytics.resolver; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -9,17 +12,31 @@ import com.linkedin.datahub.graphql.generated.AnalyticsChart; import com.linkedin.datahub.graphql.generated.AnalyticsChartGroup; import com.linkedin.datahub.graphql.generated.BarChart; +import com.linkedin.datahub.graphql.generated.Cell; import com.linkedin.datahub.graphql.generated.DateInterval; import com.linkedin.datahub.graphql.generated.DateRange; +import com.linkedin.datahub.graphql.generated.EntityProfileParams; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.LinkParams; import com.linkedin.datahub.graphql.generated.NamedBar; import com.linkedin.datahub.graphql.generated.NamedLine; import com.linkedin.datahub.graphql.generated.Row; import com.linkedin.datahub.graphql.generated.TableChart; import com.linkedin.datahub.graphql.generated.TimeSeriesChart; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.util.DateUtil; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import io.datahubproject.metadata.context.OperationContext; @@ -28,6 +45,7 @@ import java.util.List; import java.util.Optional; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.joda.time.DateTime; @@ -41,14 +59,13 @@ public final class GetChartsResolver implements DataFetcher get(DataFetchingEnvironment environment) throws Exception { + public List get(DataFetchingEnvironment environment) throws Exception { final QueryContext context = environment.getContext(); - try { return ImmutableList.of( AnalyticsChartGroup.builder() .setGroupId("DataHubUsageAnalytics") - .setTitle("DataHub Usage Analytics") + .setTitle("Usage Analytics") .setCharts(getProductAnalyticsCharts(context.getOperationContext())) .build(), AnalyticsChartGroup.builder() @@ -87,8 +104,106 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart( .build(); } + @Nullable + private AnalyticsChart getTopUsersChart(OperationContext opContext) { + try { + final DateUtil dateUtil = new DateUtil(); + final DateRange trailingMonthDateRange = dateUtil.getTrailingMonthDateRange(); + final List columns = ImmutableList.of("Name", "Title", "Email"); + + final String topUsersTitle = "Top Users"; + final List topUserRows = + _analyticsService.getTopNTableChart( + _analyticsService.getUsageIndexName(), + Optional.of(trailingMonthDateRange), + "actorUrn.keyword", + Collections.emptyMap(), + ImmutableMap.of( + "actorUrn.keyword", + ImmutableList.of("urn:li:corpuser:admin", "urn:li:corpuser:datahub")), + Optional.empty(), + 30, + AnalyticsUtil::buildCellWithEntityLandingPage); + AnalyticsUtil.convertToUserInfoRows(opContext, _entityClient, topUserRows); + return TableChart.builder() + .setTitle(topUsersTitle) + .setColumns(columns) + .setRows(topUserRows) + .build(); + } catch (Exception e) { + log.error("Failed to retrieve top users chart!", e); + return null; + } + } + + private SearchResult searchForNewUsers(@Nonnull final OperationContext opContext) + throws Exception { + // Search for new users in the past month. + final DateUtil dateUtil = new DateUtil(); + final DateRange trailingMonthDateRange = dateUtil.getTrailingMonthDateRange(); + return _entityClient.search( + opContext, + CORP_USER_ENTITY_NAME, + "*", + new Filter() + .setOr( + new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion() + .setAnd( + new CriterionArray( + ImmutableList.of( + new Criterion() + .setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME) + .setCondition(Condition.GREATER_THAN) + .setValue( + String.valueOf( + trailingMonthDateRange.getStart())))))))), + new SortCriterion() + .setField(CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME) + .setOrder(SortOrder.DESCENDING), + 0, + 100); + } + + @Nonnull + private Row buildNewUsersRow(@Nonnull final SearchEntity entity) { + final Row row = new Row(); + row.setValues(ImmutableList.of(entity.getEntity().toString())); + final Cell cell = new Cell(); + cell.setValue(entity.getEntity().toString()); + cell.setEntity(UrnToEntityMapper.map(null, entity.getEntity())); + cell.setLinkParams( + new LinkParams( + null, new EntityProfileParams(entity.getEntity().toString(), EntityType.CORP_USER))); + row.setCells(ImmutableList.of(cell)); + return row; + } + + @Nullable + private AnalyticsChart getNewUsersChart(OperationContext opContext) { + try { + final List columns = ImmutableList.of("Name", "Title", "Email"); + final String newUsersTitle = "New Users"; + final SearchResult result = searchForNewUsers(opContext); + final List newUserRows = new ArrayList<>(); + for (SearchEntity entity : result.getEntities()) { + newUserRows.add(buildNewUsersRow(entity)); + } + AnalyticsUtil.convertToUserInfoRows(opContext, _entityClient, newUserRows); + return TableChart.builder() + .setTitle(newUsersTitle) + .setColumns(columns) + .setRows(newUserRows) + .build(); + } catch (Exception e) { + log.error("Failed to retrieve new users chart!", e); + return null; + } + } + /** TODO: Config Driven Charts Instead of Hardcoded. */ - private List getProductAnalyticsCharts(@Nonnull OperationContext opContext) + private List getProductAnalyticsCharts(OperationContext opContext) throws Exception { final List charts = new ArrayList<>(); DateUtil dateUtil = new DateUtil(); @@ -97,12 +212,15 @@ private List getProductAnalyticsCharts(@Nonnull OperationContext final DateTime startOfNextMonth = dateUtil.getStartOfNextMonth(); final DateRange trailingWeekDateRange = dateUtil.getTrailingWeekDateRange(); + // WAU charts.add( getActiveUsersTimeSeriesChart( startOfNextWeek.minusWeeks(10), startOfNextWeek.minusMillis(1), "Weekly Active Users", DateInterval.WEEK)); + + // MAU charts.add( getActiveUsersTimeSeriesChart( startOfNextMonth.minusMonths(12), @@ -110,7 +228,19 @@ private List getProductAnalyticsCharts(@Nonnull OperationContext "Monthly Active Users", DateInterval.MONTH)); - String searchesTitle = "Searches Last Week"; + // New users chart - past month + final AnalyticsChart newUsersChart = getNewUsersChart(opContext); + if (newUsersChart != null) { + charts.add(newUsersChart); + } + + // Top users chart - past month + final AnalyticsChart topUsersChart = getTopUsersChart(opContext); + if (topUsersChart != null) { + charts.add(topUsersChart); + } + + String searchesTitle = "Number of Searches"; DateInterval dailyInterval = DateInterval.DAY; String searchEventType = "SearchEvent"; @@ -131,7 +261,7 @@ private List getProductAnalyticsCharts(@Nonnull OperationContext .setLines(searchesTimeseries) .build()); - final String topSearchTitle = "Top Search Queries"; + final String topSearchTitle = "Top Searches (Past Week)"; final List columns = ImmutableList.of("Query", "Count"); final List topSearchQueries = @@ -151,34 +281,8 @@ private List getProductAnalyticsCharts(@Nonnull OperationContext .setRows(topSearchQueries) .build()); - final String sectionViewsTitle = "Section Views across Entity Types"; - final List sectionViewsPerEntityType = - _analyticsService.getBarChart( - _analyticsService.getUsageIndexName(), - Optional.of(trailingWeekDateRange), - ImmutableList.of("entityType.keyword", "section.keyword"), - ImmutableMap.of("type", ImmutableList.of("EntitySectionViewEvent")), - Collections.emptyMap(), - Optional.empty(), - true); - charts.add( - BarChart.builder().setTitle(sectionViewsTitle).setBars(sectionViewsPerEntityType).build()); - - final String actionsByTypeTitle = "Actions by Entity Type"; - final List eventsByEventType = - _analyticsService.getBarChart( - _analyticsService.getUsageIndexName(), - Optional.of(trailingWeekDateRange), - ImmutableList.of("entityType.keyword", "actionType.keyword"), - ImmutableMap.of("type", ImmutableList.of("EntityActionEvent")), - Collections.emptyMap(), - Optional.empty(), - true); - charts.add(BarChart.builder().setTitle(actionsByTypeTitle).setBars(eventsByEventType).build()); - - final String topViewedTitle = "Top Viewed Dataset"; - final List columns5 = ImmutableList.of("Dataset", "#Views"); - + final String topViewedDatasetsTitle = "Top Viewed Datasets (Past Week)"; + final List columns5 = ImmutableList.of("Name", "View Count"); final List topViewedDatasets = _analyticsService.getTopNTableChart( _analyticsService.getUsageIndexName(), @@ -198,19 +302,75 @@ private List getProductAnalyticsCharts(@Nonnull OperationContext _entityClient, topViewedDatasets, Constants.DATASET_ENTITY_NAME, - ImmutableSet.of(Constants.DATASET_KEY_ASPECT_NAME), + ImmutableSet.of( + Constants.DATASET_KEY_ASPECT_NAME, Constants.DATASET_PROPERTIES_ASPECT_NAME), AnalyticsUtil::getDatasetName); charts.add( TableChart.builder() - .setTitle(topViewedTitle) + .setTitle(topViewedDatasetsTitle) .setColumns(columns5) .setRows(topViewedDatasets) .build()); + final String topViewedDashboardsTitle = "Top Viewed Dashboards (Past Week)"; + final List columns6 = ImmutableList.of("Name", "View Count"); + final List topViewedDashboards = + _analyticsService.getTopNTableChart( + _analyticsService.getUsageIndexName(), + Optional.of(trailingWeekDateRange), + "entityUrn.keyword", + ImmutableMap.of( + "type", + ImmutableList.of("EntityViewEvent"), + "entityType.keyword", + ImmutableList.of(EntityType.DASHBOARD.name())), + Collections.emptyMap(), + Optional.empty(), + 10, + AnalyticsUtil::buildCellWithEntityLandingPage); + AnalyticsUtil.hydrateDisplayNameForTable( + opContext, + _entityClient, + topViewedDashboards, + Constants.DASHBOARD_ENTITY_NAME, + ImmutableSet.of(Constants.DASHBOARD_INFO_ASPECT_NAME), + AnalyticsUtil::getDashboardName); + charts.add( + TableChart.builder() + .setTitle(topViewedDashboardsTitle) + .setColumns(columns6) + .setRows(topViewedDashboards) + .build()); + + final String sectionViewsTitle = "Tab Views By Entity Type (Past Week)"; + final List sectionViewsPerEntityType = + _analyticsService.getBarChart( + _analyticsService.getUsageIndexName(), + Optional.of(trailingWeekDateRange), + ImmutableList.of("entityType.keyword", "section.keyword"), + ImmutableMap.of("type", ImmutableList.of("EntitySectionViewEvent")), + Collections.emptyMap(), + Optional.empty(), + true); + charts.add( + BarChart.builder().setTitle(sectionViewsTitle).setBars(sectionViewsPerEntityType).build()); + + final String actionsByTypeTitle = "Actions By Entity Type (Past Week)"; + final List eventsByEventType = + _analyticsService.getBarChart( + _analyticsService.getUsageIndexName(), + Optional.of(trailingWeekDateRange), + ImmutableList.of("entityType.keyword", "actionType.keyword"), + ImmutableMap.of("type", ImmutableList.of("EntityActionEvent")), + Collections.emptyMap(), + Optional.empty(), + true); + charts.add(BarChart.builder().setTitle(actionsByTypeTitle).setBars(eventsByEventType).build()); + return charts; } - private List getGlobalMetadataAnalyticsCharts(@Nonnull OperationContext opContext) + private List getGlobalMetadataAnalyticsCharts(OperationContext opContext) throws Exception { final List charts = new ArrayList<>(); // Chart 1: Entities per domain @@ -239,7 +399,7 @@ private List getGlobalMetadataAnalyticsCharts(@Nonnull Operation AnalyticsUtil::getPlatformName); if (!entitiesPerDomain.isEmpty()) { charts.add( - BarChart.builder().setTitle("Entities per Domain").setBars(entitiesPerDomain).build()); + BarChart.builder().setTitle("Entities By Domain").setBars(entitiesPerDomain).build()); } // Chart 2: Entities per platform @@ -261,10 +421,7 @@ private List getGlobalMetadataAnalyticsCharts(@Nonnull Operation AnalyticsUtil::getPlatformName); if (!entitiesPerPlatform.isEmpty()) { charts.add( - BarChart.builder() - .setTitle("Entities per Platform") - .setBars(entitiesPerPlatform) - .build()); + BarChart.builder().setTitle("Assets By Platform").setBars(entitiesPerPlatform).build()); } // Chart 3: Entities per term @@ -286,7 +443,8 @@ private List getGlobalMetadataAnalyticsCharts(@Nonnull Operation Constants.GLOSSARY_TERM_KEY_ASPECT_NAME, Constants.GLOSSARY_TERM_INFO_ASPECT_NAME), AnalyticsUtil::getTermName); if (!entitiesPerTerm.isEmpty()) { - charts.add(BarChart.builder().setTitle("Entities per Term").setBars(entitiesPerTerm).build()); + charts.add( + BarChart.builder().setTitle("Entities With Term").setBars(entitiesPerTerm).build()); } // Chart 4: Entities per fabric type @@ -301,7 +459,7 @@ private List getGlobalMetadataAnalyticsCharts(@Nonnull Operation false); if (entitiesPerEnv.size() > 1) { charts.add( - BarChart.builder().setTitle("Entities per Environment").setBars(entitiesPerEnv).build()); + BarChart.builder().setTitle("Entities By Environment").setBars(entitiesPerEnv).build()); } return charts; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java index b21bd270a9dc1..01f2e6c8462e3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java @@ -96,7 +96,7 @@ private List getCharts(MetadataAnalyticsInput input, OperationCo Constants.DOMAIN_ENTITY_NAME, ImmutableSet.of(Constants.DOMAIN_PROPERTIES_ASPECT_NAME), AnalyticsUtil::getDomainName); - charts.add(BarChart.builder().setTitle("Entities by Domain").setBars(domainChart).build()); + charts.add(BarChart.builder().setTitle("Data Assets by Domain").setBars(domainChart).build()); } Optional platformAggregation = @@ -114,7 +114,7 @@ private List getCharts(MetadataAnalyticsInput input, OperationCo ImmutableSet.of(Constants.DATA_PLATFORM_INFO_ASPECT_NAME), AnalyticsUtil::getPlatformName); charts.add( - BarChart.builder().setTitle("Entities by Platform").setBars(platformChart).build()); + BarChart.builder().setTitle("Data Assets by Platform").setBars(platformChart).build()); } Optional termAggregation = @@ -132,7 +132,7 @@ private List getCharts(MetadataAnalyticsInput input, OperationCo ImmutableSet.of( Constants.GLOSSARY_TERM_KEY_ASPECT_NAME, Constants.GLOSSARY_TERM_INFO_ASPECT_NAME), AnalyticsUtil::getTermName); - charts.add(BarChart.builder().setTitle("Entities by Term").setBars(termChart).build()); + charts.add(BarChart.builder().setTitle("Data Assets by Term").setBars(termChart).build()); } Optional envAggregation = @@ -144,7 +144,7 @@ private List getCharts(MetadataAnalyticsInput input, OperationCo List termChart = buildBarChart(envAggregation.get()); if (termChart.size() > 1) { charts.add( - BarChart.builder().setTitle("Entities by Environment").setBars(termChart).build()); + BarChart.builder().setTitle("Data Assets by Environment").setBars(termChart).build()); } } @@ -162,7 +162,7 @@ private List buildBarChart(AggregationMetadata aggregation) { .setSegments( ImmutableList.of( BarSegment.builder() - .setLabel("#Entities") + .setLabel("Count") .setValue(entry.getValue().intValue()) .build())) .build()) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java index ab60a6d8116b1..a17745948eb82 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java @@ -1,6 +1,11 @@ package com.linkedin.datahub.graphql.analytics.service; +import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME; + +import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.dashboard.DashboardInfo; import com.linkedin.datahub.graphql.generated.BarSegment; import com.linkedin.datahub.graphql.generated.Cell; import com.linkedin.datahub.graphql.generated.Entity; @@ -11,16 +16,18 @@ import com.linkedin.datahub.graphql.generated.SearchParams; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.dataplatform.DataPlatformInfo; +import com.linkedin.dataset.DatasetProperties; import com.linkedin.domain.DomainProperties; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryTermInfo; +import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.Constants; -import com.linkedin.metadata.key.DatasetKey; import com.linkedin.metadata.key.GlossaryTermKey; import io.datahubproject.metadata.context.OperationContext; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -29,6 +36,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; @@ -151,6 +159,60 @@ public static void hydrateDisplayNameForTable( })); } + public static void convertToUserInfoRows( + OperationContext opContext, EntityClient entityClient, List rows) throws Exception { + final Set userUrns = + rows.stream() + .filter(row -> !row.getCells().isEmpty()) + .map(row -> UrnUtils.getUrn(row.getCells().get(0).getValue())) + .collect(Collectors.toSet()); + final Map gmsResponseByUser = + entityClient.batchGetV2( + opContext, + CORP_USER_INFO_ASPECT_NAME, + userUrns, + ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME)); + final Map urnToCorpUserInfo = + gmsResponseByUser.entrySet().stream() + .filter( + entry -> + entry.getValue() != null + && entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME)) + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + new CorpUserInfo( + entry + .getValue() + .getAspects() + .get(CORP_USER_INFO_ASPECT_NAME) + .getValue() + .data()))); + // Populate a row with the user link, title, and email. + rows.forEach( + row -> { + Urn urn = UrnUtils.getUrn(row.getCells().get(0).getValue()); + EntityResponse response = gmsResponseByUser.get(urn); + String maybeDisplayName = response != null ? getUserName(response).orElse(null) : null; + String maybeEmail = + urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getEmail() : null; + String maybeTitle = + urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getTitle() : null; + if (maybeDisplayName != null) { + row.getCells().get(0).setValue(maybeDisplayName); + } + final List newCells = new ArrayList<>(); + // First add the user cell + newCells.add(row.getCells().get(0)); + // Then, add the title row. + newCells.add(new Cell(maybeTitle != null ? maybeTitle : "None", null, null)); + // Finally, add the email row. + newCells.add(new Cell(maybeEmail != null ? maybeEmail : "None", null, null)); + row.setCells(newCells); + }); + } + public static Map getUrnToDisplayName( @Nonnull OperationContext opContext, EntityClient entityClient, @@ -194,24 +256,62 @@ public static Optional getPlatformName(EntityResponse entityResponse) { EnvelopedAspect envelopedDataPlatformInfo = entityResponse.getAspects().get(Constants.DATA_PLATFORM_INFO_ASPECT_NAME); if (envelopedDataPlatformInfo == null) { - return Optional.empty(); + return Optional.of(entityResponse.getUrn().getId()); } DataPlatformInfo dataPlatformInfo = new DataPlatformInfo(envelopedDataPlatformInfo.getValue().data()); - return Optional.of( + final String infoDisplayName = dataPlatformInfo.getDisplayName() == null ? dataPlatformInfo.getName() - : dataPlatformInfo.getDisplayName()); + : dataPlatformInfo.getDisplayName(); + return Optional.of(infoDisplayName != null ? infoDisplayName : entityResponse.getUrn().getId()); } public static Optional getDatasetName(EntityResponse entityResponse) { - EnvelopedAspect envelopedDatasetKey = - entityResponse.getAspects().get(Constants.DATASET_KEY_ASPECT_NAME); - if (envelopedDatasetKey == null) { + EnvelopedAspect envelopedDatasetProperties = + entityResponse.getAspects().get(Constants.DATASET_PROPERTIES_ASPECT_NAME); + if (envelopedDatasetProperties == null) { return Optional.empty(); } - DatasetKey datasetKey = new DatasetKey(envelopedDatasetKey.getValue().data()); - return Optional.of(datasetKey.getName()); + DatasetProperties datasetProperties = + new DatasetProperties(envelopedDatasetProperties.getValue().data()); + return Optional.of( + datasetProperties.hasName() + ? datasetProperties.getName() + : entityResponse.getUrn().getEntityKey().get(1)); + } + + public static Optional getDashboardName(EntityResponse entityResponse) { + EnvelopedAspect envelopedDashboardName = + entityResponse.getAspects().get(Constants.DASHBOARD_INFO_ASPECT_NAME); + if (envelopedDashboardName == null) { + return Optional.empty(); + } + DashboardInfo dashboardInfo = new DashboardInfo(envelopedDashboardName.getValue().data()); + return Optional.of(dashboardInfo.getTitle()); + } + + public static Optional getUserName(EntityResponse entityResponse) { + EnvelopedAspect envelopedCorpUserInfo = + entityResponse.getAspects().get(CORP_USER_INFO_ASPECT_NAME); + if (envelopedCorpUserInfo == null) { + return Optional.of(entityResponse.getUrn().getId()); + } + CorpUserInfo corpUserInfo = new CorpUserInfo(envelopedCorpUserInfo.getValue().data()); + final String userInfoName = + corpUserInfo.hasDisplayName() + ? corpUserInfo.getDisplayName() + : getUserFullName(corpUserInfo.getFirstName(), corpUserInfo.getLastName()); + return Optional.of(userInfoName != null ? userInfoName : entityResponse.getUrn().getId()); + } + + @Nullable + private static String getUserFullName( + @Nullable final String firstName, @Nullable final String lastName) { + if (firstName != null && lastName != null) { + return firstName + " " + lastName; + } + return null; } public static Optional getTermName(EntityResponse entityResponse) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java index 677ad8afbaca3..600db4ac04fc5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java @@ -35,4 +35,11 @@ public DateRange getTrailingWeekDateRange() { return new DateRange( String.valueOf(aWeekAgoStart.getMillis()), String.valueOf(todayEnd.getMillis())); } + + public DateRange getTrailingMonthDateRange() { + final DateTime todayEnd = getTomorrowStart().minusMillis(1); + final DateTime aMonthAgoStart = todayEnd.minusMonths(1).plusMillis(1); + return new DateRange( + String.valueOf(aMonthAgoStart.getMillis()), String.valueOf(todayEnd.getMillis())); + } } diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 34fe5493a24be..c200a4bc30d19 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -125,6 +125,7 @@ public class Constants { public static final String ROLE_MEMBERSHIP_ASPECT_NAME = "roleMembership"; public static final String CORP_USER_SETTINGS_ASPECT_NAME = "corpUserSettings"; + public static final String CORP_USER_STATUS_LAST_MODIFIED_FIELD_NAME = "statusLastModifiedAt"; // Group public static final String CORP_GROUP_KEY_ASPECT_NAME = "corpGroupKey"; diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserStatus.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserStatus.pdl index fb4aba49976c4..7d4bb2dbd04cf 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserStatus.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserStatus.pdl @@ -20,5 +20,11 @@ record CorpUserStatus { /** * Audit stamp containing who last modified the status and when. */ + @Searchable = { + "/time": { + "fieldName": "statusLastModifiedAt", + "fieldType": "COUNT" + } + } lastModified: AuditStamp } diff --git a/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js b/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js index 0b8d1c443975d..0e5105717d2be 100644 --- a/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js +++ b/smoke-test/tests/cypress/cypress/e2e/analytics/analytics.js @@ -1,5 +1,5 @@ describe('analytics', () => { - it('can go to a chart and see analytics in Section Views', () => { + it('can go to a chart and see analytics in tab views', () => { cy.login(); cy.goToChart("urn:li:chart:(looker,cypress_baz1)"); @@ -8,7 +8,7 @@ describe('analytics', () => { cy.wait(1000); cy.goToAnalytics(); - cy.contains("Section Views across Entity Types").scrollIntoView({ + cy.contains("Tab Views By Entity Type (Past Week)").scrollIntoView({ ensureScrollable: false }) cy.waitTextPresent("dashboards"); From cb6d744ae719f072b15d35c1ebaaf741c66abc35 Mon Sep 17 00:00:00 2001 From: Camilo Gutierrez <60492494+camilogutierrez@users.noreply.github.com> Date: Wed, 24 Apr 2024 07:34:02 -0500 Subject: [PATCH 6/7] feat(ingestion/bigquery): support for table clones (#10274) Co-authored-by: Equipo DataOps --- .../src/datahub/ingestion/source/bigquery_v2/queries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py index 86971fce36a53..3545cc7743838 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py @@ -74,7 +74,7 @@ class BigqueryQuery: table_name) as p on t.table_name = p.table_name WHERE - table_type in ('{BigqueryTableType.BASE_TABLE}', '{BigqueryTableType.EXTERNAL}') + table_type in ('{BigqueryTableType.BASE_TABLE}', '{BigqueryTableType.EXTERNAL}', '{BigqueryTableType.CLONE}') {{table_filter}} order by table_schema ASC, @@ -101,7 +101,7 @@ class BigqueryQuery: and t.TABLE_NAME = tos.TABLE_NAME and tos.OPTION_NAME = "description" WHERE - table_type in ('{BigqueryTableType.BASE_TABLE}', '{BigqueryTableType.EXTERNAL}') + table_type in ('{BigqueryTableType.BASE_TABLE}', '{BigqueryTableType.EXTERNAL}', '{BigqueryTableType.CLONE}') {{table_filter}} order by table_schema ASC, From 6af0e8fd614ddb588c6a6235ebf7753e56ca47e7 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 24 Apr 2024 20:07:15 +0530 Subject: [PATCH 7/7] build(ingest): update base requirements file (#10368) --- .../base-requirements.txt | 175 +++++++++--------- 1 file changed, 90 insertions(+), 85 deletions(-) diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index f2d7675d32ae3..82be7ce1a6e66 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,24 +1,25 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. -acryl-datahub-classify==0.0.9 +acryl-datahub-classify==0.0.10 acryl-PyHive==0.6.16 -acryl-sqlglot==22.4.1.dev4 +acryl-sqlglot==23.11.2.dev2 aenum==3.1.15 -aiohttp==3.9.3 +aiohttp==3.9.5 aiosignal==1.3.1 alembic==1.13.1 altair==4.2.0 anyio==4.3.0 -apache-airflow==2.8.4 -apache-airflow-providers-common-io==1.3.0 -apache-airflow-providers-common-sql==1.11.1 -apache-airflow-providers-ftp==3.7.0 -apache-airflow-providers-http==4.10.0 +apache-airflow==2.9.0 +apache-airflow-providers-common-io==1.3.1 +apache-airflow-providers-common-sql==1.12.0 +apache-airflow-providers-fab==1.0.4 +apache-airflow-providers-ftp==3.8.0 +apache-airflow-providers-http==4.10.1 apache-airflow-providers-imap==3.5.0 apache-airflow-providers-smtp==1.6.1 apache-airflow-providers-sqlite==3.7.1 -apispec==6.6.0 +apispec==6.6.1 appnope==0.1.4 -argcomplete==3.2.3 +argcomplete==3.3.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 asgiref==3.8.1 @@ -35,8 +36,8 @@ beautifulsoup4==4.12.3 bleach==6.1.0 blinker==1.7.0 blis==0.7.11 -boto3==1.34.71 -botocore==1.34.71 +boto3==1.34.90 +botocore==1.34.90 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 @@ -61,50 +62,50 @@ comm==0.2.2 confection==0.1.4 ConfigUpdater==3.2 confluent-kafka==2.3.0 -connexion==2.14.1 +connexion==2.14.2 cron-descriptor==1.4.3 -croniter==2.0.3 +croniter==2.0.5 cryptography==42.0.5 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.23.0 -databricks-sql-connector==2.9.5 +databricks-sdk==0.25.1 +databricks-sql-connector==2.9.6 dataflows-tabulator==1.54.3 db-dtypes==1.2.0 debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.16.3 +deltalake==0.17.1 Deprecated==1.2.14 dill==0.3.8 dnspython==2.6.1 docker==7.0.0 -docutils==0.20.1 -ecdsa==0.18.0 +docutils==0.21.2 +ecdsa==0.19.0 elasticsearch==7.13.4 -email-validator==1.3.1 +email_validator==2.1.1 entrypoints==0.4 et-xmlfile==1.1.0 -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 executing==2.0.1 expandvars==0.12.0 fastavro==1.9.4 fastjsonschema==2.19.1 -filelock==3.13.3 +filelock==3.13.4 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.14.6 +GeoAlchemy2==0.15.0 gitdb==4.0.11 -GitPython==3.1.42 +GitPython==3.1.43 google-api-core==2.18.0 google-auth==2.29.0 google-cloud-appengine-logging==1.4.3 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.19.0 +google-cloud-bigquery==3.21.0 google-cloud-core==2.4.1 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 @@ -117,18 +118,18 @@ graphql-core==3.2.3 great-expectations==0.15.50 greenlet==3.0.3 grpc-google-iam-v1==0.13.0 -grpcio==1.62.1 -grpcio-status==1.62.1 -grpcio-tools==1.62.1 +grpcio==1.62.2 +grpcio-status==1.62.2 +grpcio-tools==1.62.2 gssapi==1.8.3 -gunicorn==21.2.0 +gunicorn==22.0.0 h11==0.14.0 -httpcore==1.0.4 +httpcore==1.0.5 httpx==0.27.0 humanfriendly==10.0 -idna==3.6 +idna==3.7 ijson==3.2.3 -importlib_metadata==7.1.0 +importlib-metadata==7.0.0 importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 @@ -138,7 +139,7 @@ ipython-genutils==0.2.0 ipywidgets==8.1.2 iso3166==2.1.1 isodate==0.6.1 -itsdangerous==2.1.2 +itsdangerous==2.2.0 jedi==0.19.1 Jinja2==3.1.3 jmespath==1.0.1 @@ -151,34 +152,36 @@ jsonschema==4.21.1 jsonschema-specifications==2023.12.1 jupyter-server==1.16.0 jupyter_client==7.4.9 -jupyter_core==5.0.0 +jupyter_core==4.12.0 jupyterlab_pygments==0.3.0 jupyterlab_widgets==3.0.10 -langcodes==3.3.0 +langcodes==3.4.0 +language_data==1.2.0 lark==1.1.4 lazy-object-proxy==1.10.0 leb128==1.0.7 -limits==3.10.1 +limits==3.11.0 linear-tsv==1.1.0 linkify-it-py==2.0.3 lkml==1.3.4 lockfile==0.12.2 looker-sdk==23.0.0 -lxml==5.1.0 +lxml==5.2.1 lz4==4.3.3 makefun==1.15.2 -Mako==1.3.2 +Mako==1.3.3 +marisa-trie==1.1.0 markdown-it-py==3.0.0 MarkupSafe==2.1.5 marshmallow==3.21.1 marshmallow-oneofschema==3.1.1 -marshmallow-sqlalchemy==0.26.1 -matplotlib-inline==0.1.6 +marshmallow-sqlalchemy==0.28.2 +matplotlib-inline==0.1.7 mdit-py-plugins==0.4.0 mdurl==0.1.2 mistune==3.0.2 mixpanel==4.10.1 -mlflow-skinny==2.11.3 +mlflow-skinny==2.12.1 mmhash3==3.0.1 more-itertools==10.2.0 moto==4.2.14 @@ -189,40 +192,41 @@ mypy-extensions==1.0.0 nbclassic==1.0.0 nbclient==0.6.3 nbconvert==7.16.3 -nbformat==5.10.3 +nbformat==5.10.4 nest-asyncio==1.6.0 -networkx==3.2.1 +networkx==3.3 notebook==6.5.6 notebook_shim==0.2.4 numpy==1.26.4 oauthlib==3.2.2 okta==1.7.0 -openlineage-airflow==1.2.0 -openlineage-integration-common==1.2.0 -openlineage-python==1.2.0 -openlineage_sql==1.2.0 +openlineage-airflow==1.7.0 +openlineage-integration-common==1.7.0 +openlineage-python==1.7.0 +openlineage_sql==1.7.0 openpyxl==3.1.2 -opentelemetry-api==1.16.0 -opentelemetry-exporter-otlp==1.16.0 -opentelemetry-exporter-otlp-proto-grpc==1.16.0 -opentelemetry-exporter-otlp-proto-http==1.16.0 -opentelemetry-proto==1.16.0 -opentelemetry-sdk==1.16.0 -opentelemetry-semantic-conventions==0.37b0 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-proto==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 ordered-set==4.1.0 -packaging==23.2 -pandas==2.2.1 +packaging==24.0 +pandas==2.1.4 pandocfilters==1.5.1 parse==1.20.1 -parso==0.8.3 +parso==0.8.4 pathlib_abc==0.1.1 pathspec==0.12.1 pathy==0.11.0 pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 -platformdirs==3.11.0 -pluggy==1.4.0 +platformdirs==4.2.1 +pluggy==1.5.0 preshed==3.0.9 prison==0.2.1 progressbar2==4.4.2 @@ -236,20 +240,20 @@ ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 py-partiql-parser==0.5.0 -pyarrow==15.0.2 +pyarrow==16.0.0 pyarrow-hotfix==0.6 pyasn1==0.6.0 pyasn1_modules==0.4.0 pyathena==2.25.2 pycountry==23.12.11 -pycparser==2.21 +pycparser==2.22 pycryptodome==3.20.0 -pydantic==1.10.14 -pydash==7.0.7 +pydantic==1.10.15 +pydash==8.0.0 pydruid==0.6.6 Pygments==2.17.2 pyiceberg==0.4.0 -pymongo==4.6.2 +pymongo==4.6.3 PyMySQL==1.1.0 pyOpenSSL==24.1.0 pyparsing==3.0.9 @@ -258,7 +262,7 @@ python-daemon==3.0.1 python-dateutil==2.9.0.post0 python-jose==3.3.0 python-ldap==3.4.4 -python-nvd3==0.15.0 +python-nvd3==0.16.0 python-slugify==8.0.4 python-stdnum==1.20 python-tds==1.15.0 @@ -267,9 +271,9 @@ pytz==2024.1 PyYAML==6.0.1 pyzmq==24.0.1 redash-toolbelt==0.1.9 -redshift-connector==2.1.0 -referencing==0.34.0 -regex==2023.12.25 +redshift-connector==2.1.1 +referencing==0.35.0 +regex==2024.4.16 requests==2.31.0 requests-file==2.0.0 requests-gssapi==1.3.0 @@ -284,20 +288,20 @@ rpds-py==0.18.0 rsa==4.9 ruamel.yaml==0.17.17 s3transfer==0.10.1 -schwifty==2024.1.1.post0 -scipy==1.12.0 -scramp==1.4.4 -Send2Trash==1.8.2 -sentry-sdk==1.43.0 +schwifty==2024.4.0 +scipy==1.13.0 +scramp==1.4.5 +Send2Trash==1.8.3 +sentry-sdk==1.45.0 setproctitle==1.3.3 -simple-salesforce==1.12.5 +simple-salesforce==1.12.6 six==1.16.0 slack-sdk==3.18.1 smart-open==6.4.0 smmap==5.0.1 sniffio==1.3.1 -snowflake-connector-python==3.7.1 -snowflake-sqlalchemy==1.5.1 +snowflake-connector-python==3.9.1 +snowflake-sqlalchemy==1.5.3 sortedcontainers==2.4.0 soupsieve==2.5 spacy==3.5.0 @@ -305,7 +309,8 @@ spacy-legacy==3.0.12 spacy-loggers==1.0.5 sql-metadata==2.2.2 SQLAlchemy==1.4.44 -sqlalchemy-bigquery==1.10.0 +sqlalchemy-bigquery==1.11.0 +sqlalchemy-cockroachdb==1.4.4 SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 @@ -316,11 +321,11 @@ srsly==2.4.8 stack-data==0.6.3 strictyaml==1.7.3 tableauserverclient==0.25 -tableschema==1.20.10 +tableschema==1.20.11 tabulate==0.9.0 tenacity==8.2.3 -teradatasql==20.0.0.8 -teradatasqlalchemy==17.20.0.0 +teradatasql==20.0.0.10 +teradatasqlalchemy==20.0.0.0 termcolor==2.4.0 terminado==0.18.1 text-unidecode==1.3 @@ -328,7 +333,7 @@ thinc==8.1.12 thrift==0.16.0 thrift-sasl==0.4.3 time-machine==2.14.1 -tinycss2==1.2.1 +tinycss2==1.3.0 toml==0.10.2 tomlkit==0.12.4 toolz==0.12.1 @@ -338,13 +343,13 @@ traitlets==5.2.1.post0 trino==0.328.0 typer==0.7.0 typing-inspect==0.9.0 -typing_extensions==4.10.0 +typing_extensions==4.11.0 tzdata==2024.1 tzlocal==5.2 uc-micro-py==1.0.3 ujson==5.9.0 unicodecsv==0.14.1 -universal-pathlib==0.1.4 +universal_pathlib==0.2.2 urllib3==1.26.18 vertica-python==1.3.8 vertica-sqlalchemy-dialect==0.0.8.1 @@ -353,8 +358,8 @@ wasabi==1.1.2 wcmatch==8.5.1 wcwidth==0.2.13 webencodings==0.5.1 -websocket-client==1.7.0 -Werkzeug==2.3.8 +websocket-client==1.8.0 +Werkzeug==2.2.3 widgetsnbextension==4.0.10 wrapt==1.16.0 WTForms==3.1.2