csvs: Make CSV upload work again

#100
Open-Telecoms-Data · Jun 29, 2023 · 808ac63 · 808ac63
1 parent 87e6883
commit 808ac63
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 15 deletions.
diff --git a/cove_ofds/process.py b/cove_ofds/process.py
@@ -1,8 +1,11 @@
 import json
 import os.path
+import shutil
+import tempfile
 import zipfile
 
 import flattentool
+from django.core.files.storage import default_storage
 from libcoveofds.additionalfields import AdditionalFields
 from libcoveofds.geojson import (
     GeoJSONAssumeFeatureType,
@@ -158,23 +161,51 @@ def process(self, process_data: dict) -> dict:
         if os.path.exists(self.data_filename):
             return process_data
 
-        output_dir = os.path.join(
-            self.supplied_data.data_dir(), CONVERT_CSVS_INTO_JSON_DIR_NAME
-        )
-
-        os.makedirs(output_dir, exist_ok=True)
-
-        schema = OFDSSchema
+        with tempfile.TemporaryDirectory() as tmp_dir_name:
+            print(tmp_dir_name)
+            # Copy all files from storage to temp directory for processing
+            os.makedirs(os.path.join(tmp_dir_name, "input"), exist_ok=True)
+            for supplied_data_file in self.supplied_data_files:
+                with default_storage.open(supplied_data_file.storage_name()) as fp_read:
+                    with open(
+                        os.path.join(
+                            tmp_dir_name, "input", supplied_data_file.filename
+                        ),
+                        "wb",
+                    ) as fp_write:
+                        shutil.copyfileobj(fp_read, fp_write)
+                        print(supplied_data_file.filename)
+
+            # process
+            tmp_output_dir = os.path.join(tmp_dir_name, "output")
+            os.makedirs(tmp_output_dir, exist_ok=True)
+
+            schema = OFDSSchema
+
+            unflatten_kwargs = {
+                "output_name": os.path.join(tmp_output_dir, "unflattened.json"),
+                "root_list_path": "networks",
+                "input_format": "csv",
+                "schema": schema.network_schema_url,
+                "convert_wkt": True,
+            }
 
-        unflatten_kwargs = {
-            "output_name": os.path.join(output_dir, "unflattened.json"),
-            "root_list_path": "networks",
-            "input_format": "csv",
-            "schema": schema.package_schema_url,
-            "convert_wkt": True,
-        }
+            flattentool.unflatten(
+                os.path.join(tmp_dir_name, "input"), **unflatten_kwargs
+            )
 
-        flattentool.unflatten(self.supplied_data.upload_dir(), **unflatten_kwargs)
+            # Copy result back to storage
+            with open(
+                os.path.join(tmp_dir_name, "output", "unflattened.json"), "rb"
+            ) as fp_read:
+                default_storage.save(
+                    os.path.join(
+                        self.supplied_data.storage_dir(),
+                        CONVERT_CSVS_INTO_JSON_DIR_NAME,
+                        "unflattened.json",
+                    ),
+                    fp_read,
+                )
 
         return process_data
 

diff --git a/libcoveweb2/models.py b/libcoveweb2/models.py
@@ -135,6 +135,19 @@ def upload_url(self):
             self.filename,
         )
 
+    def storage_name(self):
+        """For use with Django storage classes.
+        Returns full name in storage
+        Example use:
+        default_storage.open(os.path.join(supplied_data_file.storage_name())
+        """
+        return os.path.join(
+            str(self.supplied_data.id),
+            "supplied_data",
+            str(self.id),
+            self.filename,
+        )
+
     def does_exist_in_storage(self):
         return os.path.exists(self.upload_dir_and_filename())