Merge pull request #93 from ghrcdaac/Lucy-GHRCCLOUD-5793-sbuskylerimp…

…acts-CloudOnly

Lucy ghrccloud 5793 sbuskylerimpacts cloud only

mdx/granule_metadata_extractor/processing/process_sbuskylerimpacts.py

@@ -1,6 +1,10 @@
 from ..src.extract_netcdf_metadata import ExtractNetCDFMetadata
 import os, re
 from datetime import datetime, timedelta
+from math import radians, degrees, sin, cos, asin, acos, sqrt
+from netCDF4 import Dataset
+import numpy as np
+
 try:
     import pyart
 except ImportError:
@@ -14,15 +18,40 @@ class ExtractSbuskylerimpactsMetadata(ExtractNetCDFMetadata):
     def __init__(self, file_path):
         #super().__init__(file_path)
         self.file_path = file_path
-        #these are needed to metadata extractor
-        self.fileformat = 'netCDF-4'
 
         # extracting time and space metadata from .nc file
+        nc = Dataset(self.file_path)
+        if nc.file_format == 'NETCDF3_CLASSIC':
+           self.fileformat = 'netCDF-3'
+           [self.minTime, self.maxTime, self.SLat, self.NLat, self.WLon, self.ELon] = \
+                           self.get_variables_min_max_nc3(nc)
+        else:#netCDF-4, read data via pyart
+           self.fileformat = 'netCDF-4'
+           [self.minTime, self.maxTime, self.SLat, self.NLat, self.WLon, self.ELon] = \
+                           self.get_variables_min_max_pyart()
+        nc.close()
+
+
+    def get_variables_min_max_nc3(self,data):
+        #SBU SKYLER radar range ~ 50 km
+        #Earth radius ~ 6371 km
+        mlat = radians(data.Latitude)
+        mlon = radians(data.Longitude)
+        plat = mlat
+        dlon = degrees(acos((cos(50./6371)-sin(mlat)*sin(plat))/cos(mlat)/cos(plat)))
+        dlat = degrees(50./6371.)
+        maxlat, minlat, maxlon, minlon = [data.Latitude+dlat, data.Latitude-dlat, 
+                                          data.Longitude+dlon, data.Longitude-dlon]
+
+        #Time: unit: seconds since 1970-01-01 00:00:00
+        sec = np.array(data['Time'][:])
+        minTime = datetime(1970,1,1) + timedelta(seconds=int(min(sec)))
+        maxTime = datetime(1970,1,1) + timedelta(seconds=int(max(sec)))
+
+        return minTime, maxTime, minlat, maxlat, minlon, maxlon
 
-        [self.minTime, self.maxTime, self.SLat, self.NLat, self.WLon, self.ELon] = \
-                        self.get_variables_min_max()
 
-    def get_variables_min_max(self):
+    def get_variables_min_max_pyart(self):
         """
         :param radar: Dataset opened
         :return:

mdx/granule_metadata_extractor/src/helpers/creators/sbuskylerimpacts.py

@@ -0,0 +1,85 @@
+# create lookup zip for sbuskylerimpacts 2023 data 
+# for all future collections
+from datetime import datetime, timedelta
+from utils.mdx import MDX
+import cProfile
+import time
+import math
+import re
+
+from netCDF4 import Dataset
+import numpy as np
+from math import radians, degrees, sin, cos, asin, acos, sqrt
+
+#try:
+#    import pyart
+#except ImportError:
+#    pyart = None
+
+short_name = "sbuskylerimpacts"
+provider_path = "sbuskylerimpacts/fieldCampaigns/impacts/SBU_SKYLER/data/2023/"
+file_type = "netCDF-3"
+
+class MDXProcessing(MDX):
+
+    def __init__(self):
+        super().__init__()
+
+    def process(self, filename, file_obj_stream) -> dict:
+        """
+        Individual collection processing logic for spatial and temporal 
+        metadata extraction
+        :param filename: name of file to process
+        :type filename: str
+        :param file_obj_stream: file object stream to be processed
+        :type file_obj_stream: botocore.response.StreamingBody
+        """
+        return self.get_nc_metadata(filename, file_obj_stream)
+
+
+    def get_nc_metadata(self, filename, file_obj_stream):
+        """
+        Extract temporal and spatial metadata from netCDF-3 files
+        """
+        data = Dataset("in-mem-file", mode='r', memory=file_obj_stream.read())
+
+        #SBU SKYLER radar range ~ 50 km
+        #Earth radius ~ 6371 km
+        mlat = radians(data.Latitude)
+        mlon = radians(data.Longitude)
+        plat = mlat
+        dlon = degrees(acos((cos(50./6371)-sin(mlat)*sin(plat))/cos(mlat)/cos(plat)))
+        dlat = degrees(50./6371.)
+        north, south, east, west = [data.Latitude+dlat, data.Latitude-dlat,
+                                    data.Longitude+dlon, data.Longitude-dlon]
+
+        #Time: unit: seconds since 1970-01-01 00:00:00
+        sec = np.array(data['Time'][:])
+        start_time = datetime(1970,1,1) + timedelta(seconds=int(min(sec)))
+        end_time = datetime(1970,1,1) + timedelta(seconds=int(max(sec)))
+
+        data.close()
+        return {
+            "start": start_time,
+            "end": end_time,
+            "north": north,
+            "south": south,
+            "east": east,
+            "west": west,
+            "format": file_type
+        }
+
+
+    def main(self):
+        # start_time = time.time()
+        self.process_collection(short_name, provider_path)
+        # elapsed_time = time.time() - start_time
+        # print(f"Elapsed time in seconds: {elapsed_time}")
+        self.shutdown_ec2()
+
+
+if __name__ == '__main__':
+    MDXProcessing().main()
+    # The below can be use to run a profiler and see which functions are
+    # taking the most time to process
+    # cProfile.run('MDXProcessing().main()', sort='tottime')