Docs

data-solution-automation-engine · Aug 27, 2023 · b8bc3f5 · b8bc3f5
1 parent 16d15a3
commit b8bc3f5
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 199 deletions.
diff --git a/DataWarehouseAutomation/DataWarehouseAutomation.sln b/DataWarehouseAutomation/DataWarehouseAutomation.sln
@@ -24,19 +24,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Metadata", "Sample M
 		Sample_Metadata\sampleBasicWithExtensions.json = Sample_Metadata\sampleBasicWithExtensions.json
 		Sample_Metadata\sampleCalculation.json = Sample_Metadata\sampleCalculation.json
 		Sample_Metadata\sampleCustomFunctions.json = Sample_Metadata\sampleCustomFunctions.json
+		sampleDataVaultHub.json = sampleDataVaultHub.json
 		Sample_Metadata\sampleFreeForm.json = Sample_Metadata\sampleFreeForm.json
 		Sample_Metadata\sampleJsonStagingWithPsaDetails.json = Sample_Metadata\sampleJsonStagingWithPsaDetails.json
 		Sample_Metadata\sampleMultipleDataItemMappings.json = Sample_Metadata\sampleMultipleDataItemMappings.json
 		Sample_Metadata\sampleSimpleDDL.json = Sample_Metadata\sampleSimpleDDL.json
 		Sample_Metadata\sampleSourceQuery.json = Sample_Metadata\sampleSourceQuery.json
-		Sample_Metadata\sampleTEAMv16.json = Sample_Metadata\sampleTEAMv16.json
-		Sample_Metadata\sampleTEAM_Hub.json = Sample_Metadata\sampleTEAM_Hub.json
-		Sample_Metadata\sampleTEAM_Hub_v161.json = Sample_Metadata\sampleTEAM_Hub_v161.json
-		Sample_Metadata\sampleTEAM_LSAT.json = Sample_Metadata\sampleTEAM_LSAT.json
-		Sample_Metadata\sampleVDW_Hub.json = Sample_Metadata\sampleVDW_Hub.json
-		Sample_Metadata\sampleVDW_HubWithLookup.json = Sample_Metadata\sampleVDW_HubWithLookup.json
-		Sample_Metadata\sampleVDW_Sat_Customer_v161.json = Sample_Metadata\sampleVDW_Sat_Customer_v161.json
-		Sample_Metadata\sampleVDW_StagingArea.json = Sample_Metadata\sampleVDW_StagingArea.json
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample Templates", "Sample Templates", "{968E4A48-A2A3-4F7E-B8D9-243D1F4CCE17}"

diff --git a/DataWarehouseAutomation/DataWarehouseAutomation/DataObjectMapping.cs b/DataWarehouseAutomation/DataWarehouseAutomation/DataObjectMapping.cs
@@ -2,6 +2,11 @@
 
 /// <summary>
 /// The mapping between a source and target data set / table / file.
+/// The DataObjectMapping is the element that defines an individual source-to-target mapping / ETL process. It is a mapping between a source and target object - referred to as DataObjects. The DataObject is in fact a reusable definition in the Json schema.
+/// This definition is used twice in the DataObjectMapping: as the* SourceDataObject* and as the* TargetDataObject* - both instances of the DataObject class / type.
+///
+/// The other key component of a DataObjectMapping is the* DataItemMapping*, which describes the column-to-column(or transformation-to-column).
+/// The SourceDataObject, TargetDataObject and DataItemMapping are the mandatory components of a DataObjectMapping.There are many other attributes that can be set, and there are mandatory items within the DataObjects and DataItems.These are all described in the Json schema.
 /// </summary>
 public class DataObjectMapping : IMetadata
 {

diff --git a/DataWarehouseAutomation/DataWarehouseAutomation/DataObjectMappingList.cs b/DataWarehouseAutomation/DataWarehouseAutomation/DataObjectMappingList.cs
@@ -1,5 +1,20 @@
-namespace DataWarehouseAutomation;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics.X86;
+using System;
 
+namespace DataWarehouseAutomation;
+
+/// <summary>
+/// The schema's top-level object is a 'DataObjectMappingList'. It is an array of individual source-to-target mappings called 'DataObjectMappings', commonly referred to as 'mappings'.
+/// In code, this means a DataObjectMappingList is defined as a List<DataObjectMapping>.
+/// 
+/// A DataObjectMapping is a unique ETL mapping / transformation that moves, or interprets, data from a given source to a given destination.
+/// The decision to start the format with an array / list that contains potentially multiple DataObjectMappings relates to the Data Warehouse virtualisation use-case. In these implementations, multiple individual mappings together create a single view object. 
+/// As an example, consider the loading of a Core Business Concept / Hub type entity from various different sources.If you would use these different mappings to generate ETL processes you would create one physical ETL object for each mapping. However, if you are seeking to generate a view that represents the target table the result you would use the collection (list) of mappings to generate separate statements that are unioned in the single view object.
+/// </summary>
 public class DataObjectMappingList
 {
     /// <summary>

diff --git a/DataWarehouseAutomation/sampleDataVaultHub.json b/DataWarehouseAutomation/sampleDataVaultHub.json
@@ -0,0 +1,36 @@
+{
+  "sourceToTargetMappingList": [
+    {
+      "sourceObject": {
+        "mappingObjectId": 2,
+        "mappingObjectName": "STG_PROFILER_OFFER",
+        "businessKey": {
+          "businessKeyComponentBehaviour": "None",
+          "businessKeyComponents": [
+            {
+              "columnName": "OfferID"
+            }
+          ]
+        }
+      },
+      "targetObject": {
+        "mappingObjectId": 2,
+        "mappingObjectName": "HUB_INCENTIVE_OFFER",
+        "businessKey": {
+          "businessKeyComponents": [
+            {
+              "columnName": "INCENTIVE_OFFER_HSH"
+            }
+          ]
+        }
+      },
+      "filterCriterion": "3=3",
+      "columnMapping": [
+        {
+          "sourceColumn": { "columnName": "OfferId" },
+          "targetColumn": { "columnName": "OFFER_ID" }
+        }
+      ]
+    }
+  ]
+}
diff --git a/docs/Todo/Checklist for Data Vault conformance.md b/docs/Todo/Checklist for Data Vault conformance.md
diff --git a/docs/Todo/interfaceDataWarehouseAutomationMetadata.md b/docs/Todo/interfaceDataWarehouseAutomationMetadata.md
diff --git a/docs/api/index.md b/docs/api/index.md
@@ -1,15 +1,19 @@
-# An interface schema definition Data Warehouse Automation
+# Interface specification - Data Solution Automation Metadata
 
-This schema definition, aims to facilitate greater interoperability between various DWH automation and ETL generation approaches, tools and ecosystems. The intent is to define a sufficiently generic format that can be used to record and share information about Data Warehouse (DWH) automation metadata. 
+The **interface for data solution automation metadata** provides an agreed (canonical) format for the exchange of relevant metadata for data solution/warehouse automation. The intent is to define a *sufficiently generic* format, that can be used to record and share information about data solution automation metadata, so that more time can be spent on concepts, patterns, and solution ideas - instead of reinventing the wheel on what exactly is required to automate a data solution.
 
-The schema definition can be directly viewed [here](https://github.com/RoelantVos/Data_Warehouse_Automation_Metadata_Interface/blob/master/GenericInterface/interfaceDataWarehouseAutomationMetadata.json), and is part of [this Github repository](https://github.com/RoelantVos/Data_Warehouse_Automation_Metadata_Interface). The repository contains various supporting components such as:
+This in itself aims to facilitate greater interoperability between various data solution / data warehouse automation and data logistics generations approaches and ecosystems.
 
-- A simple Class Library (DLL) that has implemented the schema structure, as well as a validation function to test Json files / messages against the schema.
+The schema definition can be directly viewed [here](https://github.com/RoelantVos/Data_Warehouse_Automation_Metadata_Interface/blob/master/GenericInterface/interfaceDataWarehouseAutomationMetadata.json), and is part of [this GitHub repository](https://github.com/RoelantVos/Data_Warehouse_Automation_Metadata_Interface). The repository contains various supporting components such as:
+
+- A simple Class Library (DLL) that has implemented the schema structure, as well as a validation function to test JSON files / messages against the schema
 - Starter documentation.
-- A sample implementation that generates code using [Handlebars.Net](http://roelantvos.com/blog/using-handlebars-to-generate-data-vault-hub-load-processes/). The example that uses the Handlebars generates code using a sample Json file that conforms to the interface schema.
+- A sample implementation that generates code using [Handlebars.Net](http://roelantvos.com/blog/using-handlebars-to-generate-data-vault-hub-load-processes/). The example that uses the Handlebars generates code using a sample JSON file that conforms to the interface schema.
 - A simple regression test application that demonstrates different usages of the schema.
 
-In principle, the schema can be used to generate an entire Data Warehouse.
+The schema is and examples are validated / extended using <https://www.jsonschemavalidator.net/>. Standards are followed from http://json-schema.org/.  Also see http://json-schema.org/learn/miscellaneous-examples.html.
+
+In principle, the schema can be used to generate an entire Data Warehouse, Data Lake and equivalent and/or similar.
 
 ## Why is this relevant?