Skip to content

Commit

Permalink
Simple export function
Browse files Browse the repository at this point in the history
  • Loading branch information
@RoelantVos
RoelantVos committed Oct 30, 2023
1 parent 015460d commit ce04332
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 19 deletions.
6 changes: 6 additions & 0 deletions DataWarehouseAutomation/DataWarehouseAutomation.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RunDwhAutomation", "RunDwhA
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchemaJsonConverter", "SchemaJsonConverter\SchemaJsonConverter.csproj", "{07214B80-37A9-4137-B0F3-109FA0D0926A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Export", "Export\Export.csproj", "{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -78,6 +80,10 @@ Global
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{07214B80-37A9-4137-B0F3-109FA0D0926A}.Release|Any CPU.Build.0 = Release|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{555AD192-6E8B-49DD-B60C-2E2CC41C0FA2}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
14 changes: 14 additions & 0 deletions DataWarehouseAutomation/Export/Export.csproj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="DataWarehouseAutomation" Version="1.3.4" />
</ItemGroup>

</Project>
98 changes: 98 additions & 0 deletions DataWarehouseAutomation/Export/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
using System.Dynamic;
using DataWarehouseAutomation;
using HandlebarsDotNet;
using Newtonsoft.Json;

var inputMetadataDirectory = @"D:\Git_Repos\jarvis-TEAM-metadata\Development\TEAM\Metadata";
var outputMetadataDirectory = @"D:\Git_Repos\";

var exceptionList = new List<string>
{
"VDW_Samples_TEAM_Attribute_Mapping.json",
"Development_TEAM_Attribute_Mapping.json",
"sample_TEAM_Attribute_Mapping.json"
};

List<DataItemMappingTuple> exportOutput = new List<DataItemMappingTuple>();

foreach (string file in Directory.EnumerateFiles(inputMetadataDirectory, "*.json", SearchOption.TopDirectoryOnly))
{
if (!exceptionList.Contains(Path.GetFileName(file)))
{
try
{
Console.WriteLine(file);
var json = File.ReadAllText(file);
var deserializedMapping = JsonConvert.DeserializeObject<DataWarehouseAutomation.DataObjectMappingList>(json);

foreach (var dataObjectMapping in deserializedMapping.DataObjectMappings)
{
var sourceDataObjectRaw = dataObjectMapping.SourceDataObjects.FirstOrDefault();

if (dataObjectMapping.DataItemMappings != null)
{
foreach (var dataItemMapping in dataObjectMapping.DataItemMappings)
{
var sourceDataItemRaw = dataItemMapping.SourceDataItems.FirstOrDefault();

var localDataItemMappingTuple = new DataItemMappingTuple
{
SourceDataObject = sourceDataObjectRaw.name,
SourceDataItem = sourceDataItemRaw.name,
TargetDataObject = dataObjectMapping.TargetDataObject.Name,
TargetDataItem = dataItemMapping.TargetDataItem.Name
};

if (!exportOutput.Contains(localDataItemMappingTuple))
{
exportOutput.Add(localDataItemMappingTuple);
}
}
}
}
}
catch (Exception exception)
{
Console.WriteLine($"Issue: "+exception.Message);
}
}
}

// Export to file
using (StreamWriter writer = new StreamWriter(outputMetadataDirectory+"TEAM-export.csv"))
{
foreach (var exportRow in exportOutput)
{
writer.WriteLine($"{exportRow.SourceDataObject},{exportRow.SourceDataItem},{exportRow.TargetDataObject},{exportRow.TargetDataItem}");
}
}

// Finish the application.
Console.ReadKey();

internal class DataItemMappingTuple
{
internal string SourceDataObject
{
get;
set;
}

internal string SourceDataItem
{
get;
set;
}

internal string TargetDataObject
{
get;
set;
}

internal string TargetDataItem
{
get;
set;
}
}
71 changes: 52 additions & 19 deletions DataWarehouseAutomation/SchemaJsonConverter/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,14 +126,6 @@
jsonObjectDataObjectMapping.Remove("mappingName");
jsonObjectDataObjectMapping.Add("name", mappingNameNode);

var getName = jsonObjectDataObjectMapping.TryGetPropertyValue("name", out var mappingNameJsonNode).ToString();

// Add the mapping name as a 'name' to the list of mappings, only once.
if (jsonObject["dataObjectMappings"].AsArray().IndexOf(dataObjectMapping) == 0)
{
jsonObject["name"] = mappingNameJsonNode.ToString();
}

// Rename the mapping classifications.
var mappingClassificationsNode = jsonObjectDataObjectMapping["mappingClassifications"];
jsonObjectDataObjectMapping.Remove("mappingClassifications");
Expand All @@ -142,6 +134,18 @@
// Update the classifications
UpdateClassifications(jsonObjectDataObjectMapping, classificationGuids);

var getName = jsonObjectDataObjectMapping.TryGetPropertyValue("name", out var mappingNameJsonNode).ToString();
var getClassification = jsonObjectDataObjectMapping.TryGetPropertyValue("classifications", out var mappingClassificationJsonNode).ToString();

var copyOfMappingClassificationNode = JsonNode.Parse(mappingClassificationJsonNode.ToJsonString());

// Add the mapping name as a 'name', and the classification, to the list of mappings, only once.
if (jsonObject["dataObjectMappings"].AsArray().IndexOf(dataObjectMapping) == 0)
{
jsonObject["name"] = mappingNameJsonNode.ToString();
jsonObject["classifications"] = copyOfMappingClassificationNode;
}

// Rename the business key definitions.
var businessKeyDefinitionsNode = jsonObjectDataObjectMapping["businessKeys"];
jsonObjectDataObjectMapping.Remove("businessKeys");
Expand Down Expand Up @@ -183,23 +187,26 @@

// Data Items.
var dataItems = new List<JsonObject>();
foreach (var dataItem in dataObjectJsonObject["dataItems"].AsArray())
if (dataObjectJsonObject["dataItems"] != null)
{
var dataItemJsonObject = JsonNode.Parse(dataItem.ToJsonString()).AsObject();
foreach (var dataItem in dataObjectJsonObject["dataItems"].AsArray())
{
var dataItemJsonObject = JsonNode.Parse(dataItem.ToJsonString()).AsObject();

// Type must be first.
dataItemJsonObject.Add("dataItemType", "dataItem");
// Type must be first.
dataItemJsonObject.Add("dataItemType", "dataItem");

// Ensure each item has a GUID
AddGuid(dataItemJsonObject, dataItemGuids, "name");
// Ensure each item has a GUID
AddGuid(dataItemJsonObject, dataItemGuids, "name");

// Re-add unchanged properties to manage order.
AddUnchangedDataItemProperties(dataItemJsonObject);
// Re-add unchanged properties to manage order.
AddUnchangedDataItemProperties(dataItemJsonObject);

// Replace properties with newer names (upgrade).
ReplaceDataObjectProperties(dataItemJsonObject);
// Replace properties with newer names (upgrade).
ReplaceDataObjectProperties(dataItemJsonObject);

dataItems.Add(dataItemJsonObject);
dataItems.Add(dataItemJsonObject);
}
}

foreach (var dataItem in dataItems)
Expand Down Expand Up @@ -773,6 +780,32 @@ void UpdateClassifications(JsonObject jsonObject, Dictionary<string, Guid> objec
var classificationJsonObject = JsonNode.Parse(classification.ToJsonString()).AsObject();
var getClassification = classificationJsonObject.TryGetPropertyValue("classification", out var classificationValue).ToString();

// Rename to phyiscal model object classifications.
string newClassificationValue = classificationValue.ToString() switch
{
"Source" => "Source",
"Core Business Concept" => "Hub",
"CoreBusinessConcept" => "Hub",
"Context" => "Satellite",
"Natural Business Relationship" => "Link",
"NaturalBusinessRelationship" => "Link",
"Natural Business Relationship Context" => "Link-Satellite",
"NaturalBusinessRelationshipContext" => "Link-Satellite",
"Natural Business Relationship Context Driving Key" => "Link-Satellite Driving Key",
"NaturalBusinessRelationshipContextDrivingKey" => "Link-Satellite Driving Key",
"Person" => "Conceptual",
"Place" => "Conceptual",
"Event" => "Physical",
_ => "Unknown"
};

if (newClassificationValue != "Unknown")
{
classificationJsonObject["classification"] = newClassificationValue;
classificationValue = newClassificationValue;
}

// Add a group.
string groupValue = classificationValue.ToString() switch
{
"Source" => "Solution Layer",
Expand Down

0 comments on commit ce04332

Please sign in to comment.