From db2dfe0b6bcac1629d6a3a6e3480a0a0066a9941 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Fri, 29 Mar 2024 15:44:58 -0500 Subject: [PATCH 1/3] Added lookup creator code exradimpacts.py --- .../src/helpers/creators/exradimpacts.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 mdx/granule_metadata_extractor/src/helpers/creators/exradimpacts.py diff --git a/mdx/granule_metadata_extractor/src/helpers/creators/exradimpacts.py b/mdx/granule_metadata_extractor/src/helpers/creators/exradimpacts.py new file mode 100644 index 0000000..056247b --- /dev/null +++ b/mdx/granule_metadata_extractor/src/helpers/creators/exradimpacts.py @@ -0,0 +1,75 @@ +# create lookup zip for parprbimpacts +# for all future collections +from datetime import datetime, timedelta +from utils.mdx import MDX +import cProfile +import time +import math +import re + +from netCDF4 import Dataset +import numpy as np + +short_name = "exradimpacts" +provider_path = "exradimpacts/fieldCampaigns/impacts/EXRAD/data/" +file_type = "HDF-5" + + +class MDXProcessing(MDX): + + def __init__(self): + super().__init__() + + def process(self, filename, file_obj_stream) -> dict: + """ + Individual collection processing logic for spatial and temporal + metadata extraction + :param filename: name of file to process + :type filename: str + :param file_obj_stream: file object stream to be processed + :type file_obj_stream: botocore.response.StreamingBody + """ + return self.get_nc_metadata(filename, file_obj_stream) + + + def get_nc_metadata(self, filename, file_obj_stream): + """ + Extract temporal and spatial metadata from HDF-5 files + """ + datafile = Dataset("in-mem-file", mode='r', memory=file_obj_stream.read()) + lat = h5.groups['Navigation'].groups['Data'].variables['Latitude'][:] + lon = h5.groups['Navigation'].groups['Data'].variables['Longitude'][:] + tm = h5.groups['Time'].groups['Data'].variables['TimeUTC'][:] + + north, south, east, west = [np.max(lat), np.min(lat), + np.max(lon), np.min(lon)] + + start_time = datetime(1970,1,1) + timedelta(seconds=float(np.min(tm))) + end_time = datetime(1970,1,1) + timedelta(seconds=float(np.max(tm))) + + + datafile.close() + return { + "start": start_time, + "end": end_time, + "north": north, + "south": south, + "east": east, + "west": west, + "format": file_type + } + + + def main(self): + # start_time = time.time() + self.process_collection(short_name, provider_path) + # elapsed_time = time.time() - start_time + # print(f"Elapsed time in seconds: {elapsed_time}") + self.shutdown_ec2() + + +if __name__ == '__main__': + MDXProcessing().main() + # The below can be use to run a profiler and see which functions are + # taking the most time to process + # cProfile.run('MDXProcessing().main()', sort='tottime') From c99ea4c934e70f882145f48e4acc38fccc824698 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 3 Apr 2024 15:01:45 -0500 Subject: [PATCH 2/3] Created exradimpacts.zip locally due to large file size --- .../src/helpers/exradimpacts.zip | Bin 1182 -> 5687 bytes .../src/helpers/exradimpacts.zip_2020_2022 | Bin 0 -> 1182 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 diff --git a/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip b/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip index 7ce05c99c3eb7d8ddfedb195a47df7232f347a8e..c95058d17f9c0bf14b4467e246e45387f7f16454 100644 GIT binary patch literal 5687 zcma)=OK%)S6oo?qY+xDok=IO7_tv|1d4PyQkQGKKVq;mtQ3A0C`H={sEF%613R{-^ z4E_pBww$V(?snHycaIaxaaYfzzd84wd#Vl}-TLtMjr8^JtKXkK{o=pXpZGO>e}Zc- zH=CcYei1)iY|h>sJbZk3_x}33qc5L6x%=Sg8@hM&8@v+^V8#e-`5YmTABP_FMd0Dd=Kw$v{NFy#a(k? zOl|^c@WRO%x(;(A=To!03Mi$=K5g8sYC{C9KGAY>SyXG`gft0Xn zl5E>%jJw)xhH_J>&G6_fD>8IpS+-H4UR70%EFAmv%mbN$X8F=d*Jd^d<1$81=Dg4@ z%Q#DB*$bl=$(Sd}nAgmaBr2{&mX^_H6jkw;z8c{cT=y9XSv;jDb51zQ1=^E&++{NV z92sdv!cA+|^%=Ucp;@j*Nl7!MBKeA}U(TAD${H1{vH0MinF4H8jd(nreXu8U0sM<( zY*-4ptl6aLqugNnoGd~Y^Bb9dS+ltt2BGi{+p#u&();MyBua8;i{#!xv$m9+h$XKY5F-By5Ool;V)FPQ;gW8_f ztS6hEXA+`>q}c=+<{1)-i>9>UK!$KZ`I)bi;gcS@J}|f<3sN|jKa%-sOao(jQg`8s zr^6M8Mm1Srqqv0s)bfrZZR3phsk{XJUgbiabs7PfF&Prwp(%RiN`}XdG zhfhYS7%{9pu2oLOYAtmNk6UGiX6u%dhrs=d%M2yyx;!=TX%VXIsD<%Wc2vU2T*|X} zuzu*fFJ80b%JV}#hDCPax5mp2X?$m*10GZ;FasW9dR3N~2qjbXW$8i8%Q4Falw~9I zx7BO5UB+YdRSJn+=BV(}avJI=-(0%`9&)~}qCcLznBY*IC}FMwvc@)NF>I05)HIYP zU%UAVU)BfCqUuplma}1E6sWp70aK3xjkG@wlh}B%+9}Vl$0(2v-Pf>{Hs6eXIi6@V zU>&>)AC`UI*;T;{@q`P^OnEvZRJmzXo;vsFZ1+h>(ijIVI#fd|R^stt{JHs5K-{ma zvs=dQ4(&;G_)5F5EY%zxBrZFl5#1iv8Bm<2k(Zlz)Z7o`=O^BJ%Qo?$~zbFc>hG%2r^6S1i9D z4B*oxkM{EIIrzNAX~lkcN=~(2B24OO+TE~Xtt{h&0fa@|9#2^u$yM9Bo!$eAGUKs8 zIDWHgdG2Yv*1vswI6BR|{m13^f8!K0eSeDYi>uSqv$50|S_7o#vY@;waGe{$JOI^zAG9*R6ZE@2PX3fkGkwqg2ivLi}nI zTv(m|?Y_D>^6Zy0OY*(18{9s>Cx34!|LX70<@^23=Wolmir%e9K650eKE_TopQCicm6H3l-Xxpz5Ck3*!%zA zob&(V(aib8-M)&m=cL$y3g2xDBU-rIHE#r@X=b&qWE4KIB6$PH3f_n&nPUkj6;fX| zx;i#Z5IgCoE%{5ew2$ZY)kyQ6*5ZkGCcHge*c`;d{j9A=p-@?}*-K)gYKw!Fnzz8z zs+37FZ&%cAiah%*d^>LxXH-Y}$(Tz4C2cwkGqiF z*y(%eAuh*L`VQvK7kpqB{4YB#LFCx1pTV=LQbhjbZd-HnVDG(N?>v`PvpPk*iu2Yy zbu`Xh6lfzA{MfDV#G^awQ@movt3wbl_sxCQDNO!m|SA| zXvUd_+c{qvvVWaPJ@ecD(;{ij9of@`pUjs|-#l^triC>*_v&SBAPdTd_-c%Z7ZLxLb`@A0CCKvg_RXhNmDRk3K z!%h7}%({SCWIGV^05e5#X>M*}Q6(Z*d@JQi{RPBmxk9)1G}j>m0oU(!t|xD`%sjdJ zFz3$Jq?%kAp{7}(t z)4~PIwz_}lnicnG)_c_jYYf7~COo><;KIn;VtAGP&X*=0u@7AJ4;&&^XQkG)e_q*V z@jSqS`FG;h)8AwsnfQEGUNXt?r|lnx0B=SnIc8khTmtAX1_nkT-qHwSp=5eiNTx^g n8nS_yL4<6eD6ot`4KSdIpa8=#k(CW(ITH{r2GUP~`WP4h#m)3S diff --git a/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 b/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 new file mode 100644 index 0000000000000000000000000000000000000000..7ce05c99c3eb7d8ddfedb195a47df7232f347a8e GIT binary patch literal 1182 zcmWIWW@Zs#U|`^2c$8uqe*I)$`UNHi20?BH25tr!hMfHT?9u|gtm6E<&=5`r=H-Pv zscV6_w1S&~k>v$50|S_7o#vY@;waGe{$JOI^zAG9*R6ZE@2PX3fkGkwqg2ivLi}nI zTv(m|?Y_D>^6Zy0OY*(18{9s>Cx34!|LX70<@^23=Wolmir%e9K650eKE_TopQCicm6H3l-Xxpz5Ck3*!%zA zob&(V(aib8-M)&m=cL$y3g2xDBU-rIHE#r@X=b&qWE4KIB6$PH3f_n&nPUkj6;fX| zx;i#Z5IgCoE%{5ew2$ZY)kyQ6*5ZkGCcHge*c`;d{j9A=p-@?}*-K)gYKw!Fnzz8z zs+37FZ&%cAiah%*d^>LxXH-Y}$(Tz4C2cwkGqiF z*y(%eAuh*L`VQvK7kpqB{4YB#LFCx1pTV=LQbhjbZd-HnVDG(N?>v`PvpPk*iu2Yy zbu`Xh6lfzA{MfDV#G^awQ@movt3wbl_sxCQDNO!m|SA| zXvUd_+c{qvvVWaPJ@ecD(;{ij9of@`pUjs|-#l^triC>*_v&SBAPdTd_-c%Z7ZLxLb`@A0CCKvg_RXhNmDRk3K z!%h7}%({SCWIGV^05e5#X>M*}Q6(Z*d@JQi{RPBmxk9)1G}j>m0oU(!t|xD`%sjdJ zFz3$Jq?%kAp{7}(t z)4~PIwz_}lnicnG)_c_jYYf7~COo><;KIn;VtAGP&X*=0u@7AJ4;&&^XQkG)e_q*V z@jSqS`FG;h)8AwsnfQEGUNXt?r|lnx0B=SnIc8khTmtAX1_nkT-qHwSp=5eiNTx^g n8nS_yL4<6eD6ot`4KSdIpa8=#k(CW(ITH{r2GUP~`WP4h#m)3S literal 0 HcmV?d00001 From 2c21d696ed1c06f1148f9f5a48b7655bcbfb8c2e Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 15 Apr 2024 14:05:50 -0500 Subject: [PATCH 3/3] Deleted exradimpacts.zip_2020_2022 --- .../src/helpers/exradimpacts.zip_2020_2022 | Bin 1182 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 diff --git a/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 b/mdx/granule_metadata_extractor/src/helpers/exradimpacts.zip_2020_2022 deleted file mode 100644 index 7ce05c99c3eb7d8ddfedb195a47df7232f347a8e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1182 zcmWIWW@Zs#U|`^2c$8uqe*I)$`UNHi20?BH25tr!hMfHT?9u|gtm6E<&=5`r=H-Pv zscV6_w1S&~k>v$50|S_7o#vY@;waGe{$JOI^zAG9*R6ZE@2PX3fkGkwqg2ivLi}nI zTv(m|?Y_D>^6Zy0OY*(18{9s>Cx34!|LX70<@^23=Wolmir%e9K650eKE_TopQCicm6H3l-Xxpz5Ck3*!%zA zob&(V(aib8-M)&m=cL$y3g2xDBU-rIHE#r@X=b&qWE4KIB6$PH3f_n&nPUkj6;fX| zx;i#Z5IgCoE%{5ew2$ZY)kyQ6*5ZkGCcHge*c`;d{j9A=p-@?}*-K)gYKw!Fnzz8z zs+37FZ&%cAiah%*d^>LxXH-Y}$(Tz4C2cwkGqiF z*y(%eAuh*L`VQvK7kpqB{4YB#LFCx1pTV=LQbhjbZd-HnVDG(N?>v`PvpPk*iu2Yy zbu`Xh6lfzA{MfDV#G^awQ@movt3wbl_sxCQDNO!m|SA| zXvUd_+c{qvvVWaPJ@ecD(;{ij9of@`pUjs|-#l^triC>*_v&SBAPdTd_-c%Z7ZLxLb`@A0CCKvg_RXhNmDRk3K z!%h7}%({SCWIGV^05e5#X>M*}Q6(Z*d@JQi{RPBmxk9)1G}j>m0oU(!t|xD`%sjdJ zFz3$Jq?%kAp{7}(t z)4~PIwz_}lnicnG)_c_jYYf7~COo><;KIn;VtAGP&X*=0u@7AJ4;&&^XQkG)e_q*V z@jSqS`FG;h)8AwsnfQEGUNXt?r|lnx0B=SnIc8khTmtAX1_nkT-qHwSp=5eiNTx^g n8nS_yL4<6eD6ot`4KSdIpa8=#k(CW(ITH{r2GUP~`WP4h#m)3S