Skip to content

Commit

Permalink
Merge branch 'datahub-project:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
jayacryl authored Sep 19, 2024
2 parents 2263859 + 7a6f194 commit 70553e4
Show file tree
Hide file tree
Showing 13 changed files with 170 additions and 40 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@
import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver;
import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver;
import com.linkedin.datahub.graphql.resolvers.structuredproperties.CreateStructuredPropertyResolver;
import com.linkedin.datahub.graphql.resolvers.structuredproperties.DeleteStructuredPropertyResolver;
import com.linkedin.datahub.graphql.resolvers.structuredproperties.RemoveStructuredPropertiesResolver;
import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpdateStructuredPropertyResolver;
import com.linkedin.datahub.graphql.resolvers.structuredproperties.UpsertStructuredPropertiesResolver;
Expand Down Expand Up @@ -1344,6 +1345,9 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher(
"updateStructuredProperty",
new UpdateStructuredPropertyResolver(this.entityClient))
.dataFetcher(
"deleteStructuredProperty",
new DeleteStructuredPropertyResolver(this.entityClient))
.dataFetcher("raiseIncident", new RaiseIncidentResolver(this.entityClient))
.dataFetcher(
"updateIncidentStatus",
Expand Down Expand Up @@ -2117,6 +2121,9 @@ private void configureStructuredPropertyResolvers(final RuntimeWiring.Builder bu
.getAllowedTypes().stream()
.map(entityTypeType.getKeyProvider())
.collect(Collectors.toList()))));
builder.type(
"StructuredPropertyEntity",
typeWiring -> typeWiring.dataFetcher("exists", new EntityExistsResolver(entityService)));
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package com.linkedin.datahub.graphql.resolvers.structuredproperties;

import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;

import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.QueryContext;
import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
import com.linkedin.datahub.graphql.exception.AuthorizationException;
import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput;
import com.linkedin.entity.client.EntityClient;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class DeleteStructuredPropertyResolver implements DataFetcher<CompletableFuture<Boolean>> {

private final EntityClient _entityClient;

public DeleteStructuredPropertyResolver(@Nonnull final EntityClient entityClient) {
_entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null");
}

@Override
public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
throws Exception {
final QueryContext context = environment.getContext();

final DeleteStructuredPropertyInput input =
bindArgument(environment.getArgument("input"), DeleteStructuredPropertyInput.class);
final Urn propertyUrn = UrnUtils.getUrn(input.getUrn());

return CompletableFuture.supplyAsync(
() -> {
try {
if (!AuthorizationUtils.canManageStructuredProperties(context)) {
throw new AuthorizationException(
"Unable to delete structured property. Please contact your admin.");
}
_entityClient.deleteEntity(context.getOperationContext(), propertyUrn);
return true;
} catch (Exception e) {
throw new RuntimeException(
String.format("Failed to perform update against input %s", input), e);
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import static com.linkedin.metadata.Constants.*;

import com.google.common.collect.ImmutableList;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.data.DataMap;
import com.linkedin.data.template.StringArrayMap;
import com.linkedin.datahub.graphql.QueryContext;
Expand Down Expand Up @@ -46,6 +48,9 @@ public StructuredPropertyEntity apply(
final StructuredPropertyEntity result = new StructuredPropertyEntity();
result.setUrn(entityResponse.getUrn().toString());
result.setType(EntityType.STRUCTURED_PROPERTY);
// set the default required values for a structured property in case references are still being
// cleaned up
setDefaultProperty(result);
EnvelopedAspectMap aspectMap = entityResponse.getAspects();
MappingHelper<StructuredPropertyEntity> mappingHelper = new MappingHelper<>(aspectMap, result);
mappingHelper.mapToResult(
Expand Down Expand Up @@ -134,4 +139,22 @@ private EntityTypeEntity createEntityTypeEntity(final String entityTypeUrnStr) {
entityType.setType(EntityType.ENTITY_TYPE);
return entityType;
}

/*
* In the case that a property is deleted and the references haven't been cleaned up yet (this process is async)
* set a default property to prevent APIs breaking. The UI queries for whether the entity exists and it will
* be filtered out.
*/
private void setDefaultProperty(final StructuredPropertyEntity result) {
StructuredPropertyDefinition definition = new StructuredPropertyDefinition();
definition.setQualifiedName("");
definition.setCardinality(PropertyCardinality.SINGLE);
definition.setImmutable(true);
definition.setValueType(
createDataTypeEntity(UrnUtils.getUrn("urn:li:dataType:datahub.string")));
definition.setEntityTypes(
ImmutableList.of(
createEntityTypeEntity(UrnUtils.getUrn("urn:li:entityType:datahub.dataset"))));
result.setDefinition(definition);
}
}
20 changes: 20 additions & 0 deletions datahub-graphql-core/src/main/resources/properties.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ extend type Mutation {
Update an existing structured property
"""
updateStructuredProperty(input: UpdateStructuredPropertyInput!): StructuredPropertyEntity!

"""
Delete an existing structured property
"""
deleteStructuredProperty(input: DeleteStructuredPropertyInput!): Boolean!
}

"""
Expand All @@ -34,6 +39,11 @@ type StructuredPropertyEntity implements Entity {
"""
type: EntityType!

"""
Whether or not this entity exists on DataHub
"""
exists: Boolean

"""
Definition of this structured property including its name
"""
Expand Down Expand Up @@ -457,3 +467,13 @@ input UpdateTypeQualifierInput {
"""
newAllowedTypes: [String!]
}

"""
Input for deleting a form
"""
input DeleteStructuredPropertyInput {
"""
The urn of the structured properties that is being deleted
"""
urn: String!
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,10 @@ export const SchemaTab = ({ properties }: { properties?: any }) => {
const hasProperties = useMemo(
() =>
entityWithSchema?.schemaMetadata?.fields?.some(
(schemaField) => !!schemaField.schemaFieldEntity?.structuredProperties?.properties?.length,
(schemaField) =>
!!schemaField.schemaFieldEntity?.structuredProperties?.properties?.filter(
(prop) => prop.structuredProperty.exists,
)?.length,
),
[entityWithSchema],
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,16 @@ interface Props {
export default function FieldProperties({ expandedField }: Props) {
const { schemaFieldEntity } = expandedField;
const { refetch } = useGetEntityWithSchema(true);
const properties =
schemaFieldEntity?.structuredProperties?.properties?.filter((prop) => prop.structuredProperty.exists) || [];

if (!schemaFieldEntity?.structuredProperties?.properties?.length) return null;
if (!schemaFieldEntity || !properties.length) return null;

return (
<>
<SectionHeader>Properties</SectionHeader>
<PropertiesWrapper>
{schemaFieldEntity.structuredProperties.properties.map((structuredProp) => {
{properties.map((structuredProp) => {
const isRichText =
structuredProp.structuredProperty.definition.valueType?.info.type === StdDataType.RichText;
const valuesData = mapStructuredPropertyValues(structuredProp);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,26 @@ export function mapStructuredPropertyValues(structuredPropertiesEntry: Structure
function getStructuredPropertyRows(entityData?: GenericEntityProperties | null) {
const structuredPropertyRows: PropertyRow[] = [];

entityData?.structuredProperties?.properties?.forEach((structuredPropertiesEntry) => {
const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition;
structuredPropertyRows.push({
displayName: displayName || qualifiedName,
qualifiedName,
values: mapStructuredPropertyValues(structuredPropertiesEntry),
dataType: structuredPropertiesEntry.structuredProperty.definition.valueType,
structuredProperty: structuredPropertiesEntry.structuredProperty,
type:
structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename
? {
type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type,
nativeDataType: typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType,
}
: undefined,
entityData?.structuredProperties?.properties
?.filter((prop) => prop.structuredProperty.exists)
.forEach((structuredPropertiesEntry) => {
const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition;
structuredPropertyRows.push({
displayName: displayName || qualifiedName,
qualifiedName,
values: mapStructuredPropertyValues(structuredPropertiesEntry),
dataType: structuredPropertiesEntry.structuredProperty.definition.valueType,
structuredProperty: structuredPropertiesEntry.structuredProperty,
type:
structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename
? {
type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type,
nativeDataType:
typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType,
}
: undefined,
});
});
});

return structuredPropertyRows;
}
Expand Down
1 change: 1 addition & 0 deletions datahub-web-react/src/graphql/fragments.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -1314,6 +1314,7 @@ fragment structuredPropertyFields on StructuredPropertyEntity {

fragment structuredPropertiesFields on StructuredPropertiesEntry {
structuredProperty {
exists
...structuredPropertyFields
}
values {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@
min-width: 0;
width: 100%;
padding-left: 20px;
padding-right: 0;

div {
padding-left: 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
},
"externalUrl": "http://airflow.example.com/tree?dag_id=datahub_emitter_operator_jinja_template_dag",
"name": "datahub_emitter_operator_jinja_template_dag",
"description": "An example dag with jinja template"
"description": "An example dag with jinja template",
"env": "PROD"
}
}
},
Expand Down Expand Up @@ -140,13 +141,14 @@
"downstream_task_ids": "[]",
"inlets": "[]",
"outlets": "[]",
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<<non-serializable: timedelta>>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<<non-serializable: timedelta>>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"multiple_outputs\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"<<non-serializable: _DownstreamPriorityWeightStrategy>>\"}, \"type\": \"operator\"}]}"
},
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task",
"name": "datahub_emitter_operator_jinja_template_dag_task",
"type": {
"string": "COMMAND"
}
},
"env": "PROD"
}
}
},
Expand Down Expand Up @@ -233,16 +235,16 @@
"state": "running",
"operator": "DatahubEmitterOperator",
"priority_weight": "1",
"log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1",
"log_url": "http://airflow.example.com/dags/datahub_emitter_operator_jinja_template_dag/grid?dag_run_id=manual_run_test&task_id=datahub_emitter_operator_jinja_template_dag_task&map_index=-1&tab=logs",
"orchestrator": "airflow",
"dag_id": "datahub_emitter_operator_jinja_template_dag",
"task_id": "datahub_emitter_operator_jinja_template_dag_task"
},
"externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=datahub_emitter_operator_jinja_template_dag_task&dag_id=datahub_emitter_operator_jinja_template_dag&map_index=-1",
"externalUrl": "http://airflow.example.com/dags/datahub_emitter_operator_jinja_template_dag/grid?dag_run_id=manual_run_test&task_id=datahub_emitter_operator_jinja_template_dag_task&map_index=-1&tab=logs",
"name": "datahub_emitter_operator_jinja_template_dag_datahub_emitter_operator_jinja_template_dag_task_manual_run_test",
"type": "BATCH_AD_HOC",
"created": {
"time": 1726570937652,
"time": 1726699852909,
"actor": "urn:li:corpuser:datahub"
}
}
Expand All @@ -267,7 +269,7 @@
"aspectName": "dataProcessInstanceRunEvent",
"aspect": {
"json": {
"timestampMillis": 1726570937652,
"timestampMillis": 1726699852909,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
Expand Down Expand Up @@ -360,13 +362,14 @@
"downstream_task_ids": "[]",
"inlets": "[]",
"outlets": "[]",
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<<non-serializable: timedelta>>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"downstream\"}, \"type\": \"operator\"}]}"
"openlineage_run_facet_unknownSourceAttribute": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.18.0/integration/airflow\", \"_schemaURL\": \"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet\", \"unknownItems\": [{\"name\": \"DatahubEmitterOperator\", \"properties\": {\"depends_on_past\": false, \"downstream_task_ids\": \"[]\", \"execution_timeout\": \"<<non-serializable: timedelta>>\", \"executor_config\": {}, \"ignore_first_depends_on_past\": true, \"is_setup\": false, \"is_teardown\": false, \"mapped\": false, \"multiple_outputs\": false, \"operator_class\": \"datahub_airflow_plugin.operators.datahub.DatahubEmitterOperator\", \"owner\": \"airflow\", \"priority_weight\": 1, \"queue\": \"default\", \"retries\": 0, \"retry_exponential_backoff\": false, \"task_id\": \"datahub_emitter_operator_jinja_template_dag_task\", \"trigger_rule\": \"all_success\", \"upstream_task_ids\": \"[]\", \"wait_for_downstream\": false, \"wait_for_past_depends_before_skipping\": false, \"weight_rule\": \"<<non-serializable: _DownstreamPriorityWeightStrategy>>\"}, \"type\": \"operator\"}]}"
},
"externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=datahub_emitter_operator_jinja_template_dag&_flt_3_task_id=datahub_emitter_operator_jinja_template_dag_task",
"name": "datahub_emitter_operator_jinja_template_dag_task",
"type": {
"string": "COMMAND"
}
},
"env": "PROD"
}
}
},
Expand Down Expand Up @@ -441,7 +444,7 @@
"aspectName": "dataProcessInstanceRunEvent",
"aspect": {
"json": {
"timestampMillis": 1726570937811,
"timestampMillis": 1726699853392,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
Expand Down
Loading

0 comments on commit 70553e4

Please sign in to comment.