Load STAC collections and items & filter & regroup items

Closes #2069 #2070 #2071

package.json

@@ -7,7 +7,7 @@
     "@fortawesome/fontawesome-svg-core": "^6.2.0",
     "@fortawesome/free-solid-svg-icons": "^6.2.0",
     "@fortawesome/react-fontawesome": "^0.2.0",
-    "@gulfofmaine/tsstac": "0.1.2",
+    "@gulfofmaine/tsstac": "0.1.3",
     "@reduxjs/toolkit": "^1.9.0",
     "@sentry/react": "^7.18.0",
     "@sentry/tracing": "^7.18.0",

src/Pages/Modeling/stac.tsx

@@ -20,7 +20,7 @@ import {
 } from "reactstrap"
 import { useQuery, useQueries, UseQueryResult } from "react-query"
 
-import STAC, { IAsset, ICatalog, ICollection, IItem, IFetchData } from "@gulfofmaine/tsstac"
+import STAC, { IAsset, ICatalog, ICollection, IItem, IFetchData, ISTAC } from "@gulfofmaine/tsstac"
 
 import { queryClient } from "queryClient"
 
@@ -63,6 +63,22 @@ async function getItem(catalog: ICatalog, id: string, depth: number = -1): Promi
   return item
 }
 
+async function getChild(
+  url: string,
+  parent?: ICatalog | ICollection,
+  root_catalog?: ICatalog
+): Promise<ICatalog | ICollection> {
+  // const response = await fetch(url)
+  // const json = await response.json()
+
+  // return json as ICatalog | ICollection
+  return (await stac.get(url, { parent, root: root_catalog })) as ICatalog | ICollection
+}
+
+async function getItemByUrl(url: string, parent?: ICatalog | ICollection, root_catalog?: ICatalog): Promise<IItem> {
+  return (await stac.get(url, { parent, root: root_catalog })) as IItem
+}
+
 export function useItemQuery(id: string, enabled: boolean = true) {
   const catalogQuery = useRootCatalogQuery()
   return useQuery<IItem>(
@@ -93,20 +109,175 @@ export const StacCatalogRoot = () => {
   const catalogQuery = useRootCatalogQuery()
 
   if (catalogQuery.isLoading) {
-    return <div>Loading root catalog</div>
+    return <div>Loading model catalog</div>
   }
 
   if (catalogQuery.data) {
     return (
       <div style={{ height: "60vh", overflow: "scroll" }}>
-        <StacCatalogCollection cat={catalogQuery.data} />
+        {/* <StacCatalogCollection cat={catalogQuery.data} /> */}
+        {/* Loaded root catalog */}
+        {/* <StacTraverse catalog={catalogQuery.data} /> */}
+        {/* Loaded catalog */}
+        <STACTraverseBase catalog={catalogQuery.data} />
       </div>
     )
   }
 
   return <div>Error loading root catalog</div>
 }
 
+const STACTraverseBase = ({ catalog }: { catalog: ICatalog }) => {
+  const root_children_urls = catalog
+    .get_child_links()
+    .map((link) => ({ parent: catalog, url: catalog.url_for_link(link) }))
+
+  // return <React.Fragment>Loaded root catalog</React.Fragment>
+  return <STACCollectionsLoader catalog={catalog} initial_children_urls={new Set(root_children_urls)} />
+}
+
+interface StacURL {
+  parent: ICatalog | ICollection
+  url: string
+}
+
+const STACCollectionsLoader = ({
+  catalog,
+  initial_children_urls,
+}: {
+  catalog: ICatalog
+  initial_children_urls: Set<StacURL>
+}) => {
+  const [childrenUrls, setChildrenUrls] = React.useState<Set<StacURL>>(initial_children_urls)
+
+  const addChildrenUrls = (urls: StacURL[]) => setChildrenUrls((prev) => new Set([...prev, ...urls]))
+
+  const childrenQueries = useQueries(
+    Array.from(childrenUrls).map(({ url, parent }) => ({
+      queryKey: ["get-stac-child", { url }],
+      queryFn: () => getChild(url as string, parent, catalog),
+      refetchOnWindowFocus: false,
+    }))
+  )
+
+  childrenQueries.forEach((query) => {
+    if (query.data) {
+      const children = query.data.get_child_links().map((link) => query.data.url_for_link(link))
+      const new_children = children
+        .map((child) => ({ parent: query.data, url: child }))
+        .filter((child) => !childrenUrls.has(child))
+
+      // Need to add new_children to state if need be
+      if (0 < new_children.length) {
+        addChildrenUrls(new_children)
+      }
+    }
+  })
+
+  const itemsUrls = childrenQueries
+    .filter((query) => query.data)
+    .map((query) => query.data)
+    .map((collection) =>
+      collection!.get_item_links().map((link) => ({ url: collection!.url_for_link(link), parent: collection! }))
+    )
+    .flat()
+
+  if (0 < itemsUrls.length) {
+    return <STACItemsLoader catalog={catalog} itemUrls={itemsUrls} />
+  }
+
+  return (
+    <ul>
+      <li>Loaded root catalog</li>
+      <li>Loading {Array.from(childrenUrls).length} sub-catalogs/collections</li>
+      <li>Loading {itemsUrls.length} items</li>
+    </ul>
+  )
+}
+
+const STACItemsLoader = ({ catalog, itemUrls }: { catalog: ICatalog; itemUrls: StacURL[] }) => {
+  const itemsQuery = useQueries(
+    Array.from(itemUrls).map(({ url, parent }) => ({
+      queryKey: ["get-stac-item", { url }],
+      queryFn: () => getItemByUrl(url as string, parent, catalog),
+      refetchOnWindowFocus: false,
+    }))
+  )
+
+  // First filter by which items are loaded
+  const loadedItems = itemsQuery.filter((query) => query.isSuccess).map((query) => query.data)
+
+  // Then group by collection
+  const itemsByCollection = loadedItems.reduce((result, item) => {
+    if (item && item.collection) {
+      const collection = item.collection
+      if (!result[collection]) {
+        result[collection] = []
+      }
+
+      result[collection].push(item)
+    }
+    return result
+  }, {})
+
+  // Find the item for each collection with the latest forecast_reference_time
+  // let latestItems = {}
+  // Object.keys(itemsByCollection).forEach((key) => {
+  //   const items = itemsByCollection[key]
+
+  //   const item = items.reduce((a, b) => {
+  //     const a_date = new Date(a.properties["cube:dimensions"].forecast_reference_time.values[0])
+  //     const b_date = new Date(b.properties["cube:dimensions"].forecast_reference_time.values[0])
+
+  //     if (a_date.valueOf() < b_date.valueOf()) {
+  //       return b
+  //     }
+  //     return a
+  //   })
+  //   latestItems[key] = item
+  // })
+
+  const latestItems = Object.keys(itemsByCollection).map((key) => {
+    const items = itemsByCollection[key]
+
+    const item = items.reduce((a, b) => {
+      const a_date = new Date(a.properties["cube:dimensions"].forecast_reference_time.values[0])
+      const b_date = new Date(b.properties["cube:dimensions"].forecast_reference_time.values[0])
+
+      if (a_date.valueOf() < b_date.valueOf()) {
+        return b
+      }
+      return a
+    })
+    return item
+  })
+
+  // Regroup by {ioos_category: {standard_name: Items[]}}
+  let ioosCategoryItems = {}
+  latestItems.forEach((item) => {
+    Object.keys(item.properties["cube:variables"]).forEach((itemVar) => {
+      const attrs = item.properties["cube:variables"][itemVar]["gmri-cube:attrs"]
+
+      if (attrs.hasOwnProperty("ioos_category") && attrs.hasOwnProperty("standard_name")) {
+        const ioos_category = attrs.ioos_category
+        const standard_name = attrs.standard_name
+
+        if (!ioosCategoryItems[ioos_category]) {
+          ioosCategoryItems[ioos_category] = {}
+        }
+
+        if (!ioosCategoryItems[ioos_category][standard_name]) {
+          ioosCategoryItems[ioos_category][standard_name] = []
+        }
+
+        ioosCategoryItems[ioos_category][standard_name].push(item)
+      }
+    })
+  })
+
+  return <React.Fragment>Loading items</React.Fragment>
+}
+
 export const StacCatalogCollection = ({ cat }: { cat: ICatalog | ICollection }) => {
   const childrenQuery = useQuery(["stac-children", cat.id], async () => {
     const children = await cat.get_children()

tsconfig.json

@@ -20,7 +20,8 @@
     "noEmit": true,
     "jsx": "react-jsx",
     "baseUrl": "src",
-    "noFallthroughCasesInSwitch": true
+    "noFallthroughCasesInSwitch": true,
+    "downlevelIteration": true
   },
   "include": [
     "src"

yarn.lock

@@ -3259,10 +3259,10 @@
   dependencies:
     prop-types "^15.8.1"
 
-"@gulfofmaine/tsstac@0.1.2":
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/@gulfofmaine/tsstac/-/tsstac-0.1.2.tgz#9bff425e6a8ddcbd83bab53d15d32b0102f4df1a"
-  integrity sha512-OEzw5wVc1WBGRJBGGyjYxZ7uc8X5rBHyyHAsQJW80DBJdnuK6apHZqA1k3XvfsCTelZluoaL85nOOPcdLoBBfw==
+"@gulfofmaine/tsstac@0.1.3":
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/@gulfofmaine/tsstac/-/tsstac-0.1.3.tgz#8e31a6a06147eaa76a9a75b641f960eb9a0c2865"
+  integrity sha512-JQzYu2d9QYESDOwlaTXlEr7Dj/sOXlhkUXEuQmA7wTfosqPbDie/4TQQxZvcGSjuBQMXytBvMSaOAU/eckK3mg==
 
 "@hapi/hoek@^9.0.0":
   version "9.0.4"