json_input: Add support for flattening an array of arrays

#398
OpenDataServices · Sep 26, 2022 · da1973f · da1973f
1 parent b5619d6
commit da1973f
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+
+- Add support for flattening an array of arrays https://github.com/OpenDataServices/flatten-tool/issues/398
+
 ## [0.17.2] - 2022-06-15
 
 ### Fixed

diff --git a/flattentool/json_input.py b/flattentool/json_input.py
@@ -419,10 +419,20 @@ def parse_json_dict(
                     # Check for an array of BASIC types
                     # TODO Make this check the schema
                     # TODO Error if the any of the values contain the separator
-                    # TODO Support doubly nested arrays
                     flattened_dict[sheet_key(sheet, parent_name + key)] = ";".join(
                         map(str, value)
                     )
+                # Arrays of arrays
+                elif all(
+                    l not in BASIC_TYPES
+                    and not hasattr(l, "items")
+                    and hasattr(l, "__iter__")
+                    and all(type(x) in BASIC_TYPES for x in l)
+                    for l in value
+                ):
+                    flattened_dict[sheet_key(sheet, parent_name + key)] = ";".join(
+                        map(lambda l: ",".join(map(str, l)), value)
+                    )
                 else:
                     if (
                         self.rollup and parent_name == ""

diff --git a/flattentool/tests/fixtures/360-giving-schema.json b/flattentool/tests/fixtures/360-giving-schema.json
@@ -623,6 +623,13 @@
       "description": "A web link pointing to the source of this data. This may be an original 360Giving data file, a file from which the data was converted, or an organisation website.",
       "weight": 25,
       "title": "Data Source"
+    },
+    "test_array_of_arrays": {
+        "type": "array",
+        "items": {
+            "type": "array",
+            "items": {"type": "string"}
+        }
     }
   }
 }
diff --git a/flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json b/flattentool/tests/fixtures/fundingproviders-grants_fixed_2_grants.json
@@ -35,7 +35,8 @@
             ],
             "Grant type": "Large Awards",
             "Full name of applicant": "Miss Jane Roe",
-            "awardDate": "24/07/2014"
+            "awardDate": "24/07/2014",
+            "test_array_of_arrays": [["a", "r"], ["s", "t", "n"], ["e"]]
         },
         {
             "recipientOrganization": [

diff --git a/flattentool/tests/test_json_input.py b/flattentool/tests/test_json_input.py
@@ -152,6 +152,17 @@ def test_parse_array():
     assert parser.sub_sheets == {}
 
 
+def test_parse_array_of_arrays():
+    parser = JSONParser(
+        root_json_dict=[
+            OrderedDict([("testarray", [["item", "anotheritem", 42], ["a", "b", 1]])])
+        ]
+    )
+    assert list(parser.main_sheet) == ["testarray"]
+    assert list(parser.main_sheet.lines) == [{"testarray": "item,anotheritem,42;a,b,1"}]
+    assert parser.sub_sheets == {}
+
+
 def test_root_list_path():
     parser = JSONParser(
         root_json_dict={