From 2410b9ae0a5b2a2778e5e274a8ef05f786faf6de Mon Sep 17 00:00:00 2001 From: CI Date: Tue, 25 Jun 2024 14:20:04 +0000 Subject: [PATCH] Deployed cf07d13 to 0.5.1 with MkDocs 1.6.0 and mike 2.1.2 --- 0.5.1/api/arrow/index.html | 101 ++++++++++++++++++++++++++------ 0.5.1/search/search_index.json | 2 +- 0.5.1/sitemap.xml | 16 ++--- 0.5.1/sitemap.xml.gz | Bin 278 -> 278 bytes 0.5.1/usage/index.html | 20 ++++++- latest/api/arrow/index.html | 101 ++++++++++++++++++++++++++------ latest/search/search_index.json | 2 +- latest/sitemap.xml | 16 ++--- latest/sitemap.xml.gz | Bin 278 -> 278 bytes latest/usage/index.html | 20 ++++++- 10 files changed, 220 insertions(+), 58 deletions(-) diff --git a/0.5.1/api/arrow/index.html b/0.5.1/api/arrow/index.html index 17bb1c8..304788a 100644 --- a/0.5.1/api/arrow/index.html +++ b/0.5.1/api/arrow/index.html @@ -916,7 +916,7 @@

*, chunk_size: int = 8192, schema: Schema | InferredSchema | None = None -) -> Iterable[RecordBatch] +) -> RecordBatchReader
@@ -968,10 +968,10 @@

Returns:

  • - Iterable[RecordBatch] + RecordBatchReader –
    -

    an iterable of pyarrow RecordBatches with the STAC-GeoParquet representation of items.

    +

    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.

@@ -994,7 +994,7 @@

chunk_size: int = DEFAULT_JSON_CHUNK_SIZE, schema: Schema | None = None, limit: int | None = None -) -> Iterator[RecordBatch] +) -> RecordBatchReader

@@ -1054,13 +1054,13 @@

Yields:

+

Returns:

  • - RecordBatch + RecordBatchReader –
    -

    Arrow RecordBatch with a single chunk of Item data.

    +

    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.

@@ -1255,6 +1255,8 @@

pyarrow.parquet.ParquetWriter.

@@ -1268,12 +1270,43 @@

-
stac_table_to_items(table: Table) -> Iterable[dict]
+
stac_table_to_items(
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+) -> Iterable[dict]
 

 
     

 
-      
Convert a STAC Table to a generator of STAC Item dicts

+      
Convert STAC Arrow to a generator of STAC Item dicts.

+
+
+
Parameters:

+    
    
+        
  • 
+          table
+              (Table | RecordBatchReader | ArrowStreamExportable)
+          –
+          
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
+          
    
+        
    • 
+    

+
+
+
Yields:

+    
    
+        
  • 
+              Iterable[dict]
+          –
+          
    
+            
    A STAC dict for each input row.
    
+          
    
+        
    • 
+    

 
     

 
@@ -1287,12 +1320,40 @@ 

 
 
 

-
stac_table_to_ndjson(table: Table, dest: str | Path | PathLike[bytes]) -> None
+
stac_table_to_ndjson(
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+    dest: str | Path | PathLike[bytes],
+) -> None
 

 
     

 
-      
Write a STAC Table to a newline-delimited JSON file.

+      
Write STAC Arrow to a newline-delimited JSON file.

+
+
+
Parameters:

+    
    
+        
  • 
+          table
+              (Table | RecordBatchReader | ArrowStreamExportable)
+          –
+          
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
+          
    
+        
    • 
+        
  • 
+          dest
+              (str | Path | PathLike[bytes])
+          –
+          
    
+            
    The destination where newline-delimited JSON should be written.
    
+          
    
+        
    • 
+    

 
     

 
@@ -1307,8 +1368,8 @@ 

 
 

 
to_parquet(
-    table: Table,
-    where: Any,
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+    output_path: str | Path,
     *,
     schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,
     **kwargs: Any
@@ -1325,15 +1386,19 @@ 

     
    
         
  • 
           table
-              (Table)
+              (Table | RecordBatchReader | ArrowStreamExportable)
           –
           
    
-            
    The table to write to Parquet
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
           
    
         
    • 
         
  • 
-          where
-              (Any)
+          output_path
+              (str | Path)
           –
           
    
             
    The destination for saving.
    
@@ -1354,6 +1419,8 @@ 
    
           
    
         
    • 
     

+      
All other keyword args are passed on to
+pyarrow.parquet.ParquetWriter.

 
     

 
diff --git a/0.5.1/search/search_index.json b/0.5.1/search/search_index.json
index 582ab72..e02f440 100644
--- a/0.5.1/search/search_index.json
+++ b/0.5.1/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"STAC-geoparquet","text":"
Convert STAC items between JSON, GeoParquet, pgstac, and Delta Lake.
"},{"location":"#purpose","title":"Purpose","text":"
The STAC spec defines a JSON-based schema. But it can be hard to manage and search through many millions of STAC items in JSON format. For one, JSON is very large on disk. And you need to parse the entire JSON data into memory to extract just a small piece of information, say the datetime and one asset of an Item.
 
GeoParquet can be a good complement to JSON for many bulk-access and analytic use cases. While STAC Items are commonly distributed as individual JSON files on object storage or through a STAC API, STAC GeoParquet allows users to access a large number of STAC items in bulk without making repeated HTTP requests.
 
For analytic questions like \"find the items in the Sentinel-2 collection in June 2024 over New York City with cloud cover of less than 20%\" it can be much, much faster to find the relevant data from a GeoParquet source than from JSON, because GeoParquet needs to load only the relevant columns for that query, not the full data.
 
See the STAC-GeoParquet specification for details on the exact schema of the written Parquet files.
"},{"location":"#documentation","title":"Documentation","text":"
Documentation website
"},{"location":"drawbacks/","title":"Drawbacks","text":"
Trying to represent STAC data in GeoParquet has some drawbacks.
"},{"location":"drawbacks/#unable-to-represent-undefined-values","title":"Unable to represent undefined values","text":"
Parquet is unable to represent the difference between undefined and null, and so is unable to perfectly round-trip STAC data with undefined values.
 
In JSON a value can have one of three states: defined, undefined, or null. The \"b\" key in the next three examples illustrates this:
 
Defined:
 
{\n  \"a\": 1,\n  \"b\": \"foo\"\n}\n
 
Undefined:
 
{\n  \"a\": 2\n}\n
 
Null:
 
{\n  \"a\": 3,\n  \"b\": null\n}\n
 
Because Parquet is a columnar format, it is only able to represent undefined at the column level. So if those three JSON items above were converted to Parquet, the column \"b\" would exist because it exists in the first and third item, and the second item would have \"b\" inferred as null:
 a b 1 \"foo\" 2 null 3 null 
Then when the second item is converted back to JSON, it will be returned as
 
{\n  \"a\": 2\n  \"b\": null\n}\n
 
which is not strictly equal to the input.
"},{"location":"drawbacks/#schema-difficulties","title":"Schema difficulties","text":"
JSON is schemaless while Parquet requires a strict schema, and it can be very difficult to unite these two systems. This is such an important consideration that we have a documentation page just to discuss this point.
"},{"location":"schema/","title":"Schema considerations","text":"
A STAC Item is a JSON object to describe an external geospatial dataset. The STAC specification defines a common core, plus a variety of extensions. Additionally, STAC Items may include custom extensions outside the common ones. Crucially, the majority of the specified fields in the core spec and extensions define optional keys. Those keys often differ across STAC collections and may even differ within a single collection across items.
 
STAC's flexibility is a blessing and a curse. The flexibility of schemaless JSON allows for very easy writing as each object can be dumped separately to JSON. Every item is allowed to have a different schema. And newer items are free to have a different schema than older items in the same collection. But this write-time flexibility makes it harder to read as there are no guarantees (outside STAC's few required fields) about what fields exist.
 
Parquet is the complete opposite of JSON. Parquet has a strict schema that must be known before writing can start. This puts the burden of work onto the writer instead of the reader. Reading Parquet is very efficient because the file's metadata defines the exact schema of every record. This also enables use cases like reading specific columns that would not be possible without a strict schema.
 
This conversion from schemaless to strict-schema is the difficult part of converting STAC from JSON to GeoParquet, especially for large input datasets like STAC that are often larger than memory.
"},{"location":"schema/#full-scan-over-input-data","title":"Full scan over input data","text":"
The most foolproof way to convert STAC JSON to GeoParquet is to perform a full scan over input data. This is done automatically by parse_stac_ndjson_to_arrow when a schema is not provided.
 
This is time consuming as it requires two full passes over the input data: once to infer a common schema and again to actually write to Parquet (though items are never fully held in memory, allowing this process to scale).
"},{"location":"schema/#user-provided-schema","title":"User-provided schema","text":"
Alternatively, the user can pass in an Arrow schema themselves using the schema parameter of parse_stac_ndjson_to_arrow. This schema must match the on-disk schema of the the STAC JSON data.
"},{"location":"schema/#multiple-schemas-per-collection","title":"Multiple schemas per collection","text":"
It is also possible to write multiple Parquet files with STAC data where each Parquet file may have a different schema. This simplifies the conversion and writing process but makes reading and using the Parquet data harder.
"},{"location":"schema/#merging-data-with-schema-mismatch","title":"Merging data with schema mismatch","text":"
If you've created STAC GeoParquet data where the schema has updated, you can use pyarrow.concat_tables with promote_options=\"permissive\" to combine multiple STAC GeoParquet files.
 
import pyarrow as pa\nimport pyarrow.parquet as pq\n\ntable_1 = pq.read_table(\"stac1.parquet\")\ntable_2 = pq.read_table(\"stac2.parquet\")\ncombined_table = pa.concat_tables([table1, table2], promote_options=\"permissive\")\n
"},{"location":"schema/#future-work","title":"Future work","text":"
Schema operations is an area where future work can improve reliability and ease of use of STAC GeoParquet.
 
It's possible that in the future we could automatically infer an Arrow schema from the STAC specification's published JSON Schema files. If you're interested in this, open an issue and discuss.
"},{"location":"usage/","title":"Usage","text":"
Except for the legacy API, Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.
 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.
"},{"location":"usage/#dictjson-arrow-conversion","title":"dict/JSON - Arrow conversion","text":""},{"location":"usage/#convert-dicts-to-arrow","title":"Convert dicts to Arrow","text":"
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.
"},{"location":"usage/#convert-json-to-arrow","title":"Convert JSON to Arrow","text":"
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.
"},{"location":"usage/#convert-arrow-to-dicts","title":"Convert Arrow to dicts","text":"
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#convert-arrow-to-json","title":"Convert Arrow to JSON","text":"
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#parquet","title":"Parquet","text":"
Use to_parquet to write STAC Arrow data in memory. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.
 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.
 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.
"},{"location":"usage/#delta-lake","title":"Delta Lake","text":"
Use parse_stac_ndjson_to_delta_lake to read (newline-delimited) JSON on disk and write out to a Delta Lake table.
 
No special API is required for reading a STAC Delta Lake table back into Arrow. You can use the DeltaTable class directly to read the data back into Arrow.
 
Important
 
Arrow has a null data type, where every value in the column is always null, but Delta Lake does not. This means that for any column inferred to have a null data type, writing to Delta Lake will error with 
_internal.SchemaMismatchError: Invalid data type for Delta Lake: Null\n
 
This is a problem because if all items in a STAC Collection have a null JSON key, it gets inferred as an Arrow null type. For example, in the 3dep-lidar-copc collection in the tests, it has start_datetime and end_datetime fields, and so according to the spec, datetime is always null. This column would need to be casted to a timestamp type before being written to Delta Lake.
 
This means we cannot write this collection to Delta Lake solely with automatic schema inference.
 
In such cases, users may need to manually update the inferred schema to cast any null type to another Delta Lake-compatible type.
"},{"location":"api/arrow/","title":"stac_geoparquet.arrow","text":"
Arrow-based format conversions.
"},{"location":"api/arrow/#stac_geoparquet.arrow","title":"stac_geoparquet.arrow","text":""},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_JSON_CHUNK_SIZE","title":"DEFAULT_JSON_CHUNK_SIZE  module-attribute","text":"
DEFAULT_JSON_CHUNK_SIZE = 65536\n
 
The default chunk size to use for reading JSON into memory.
"},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_PARQUET_SCHEMA_VERSION","title":"DEFAULT_PARQUET_SCHEMA_VERSION  module-attribute","text":"
DEFAULT_PARQUET_SCHEMA_VERSION: SUPPORTED_PARQUET_SCHEMA_VERSIONS = '1.1.0'\n
 
The default GeoParquet schema version written to file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.SUPPORTED_PARQUET_SCHEMA_VERSIONS","title":"SUPPORTED_PARQUET_SCHEMA_VERSIONS  module-attribute","text":"
SUPPORTED_PARQUET_SCHEMA_VERSIONS = Literal['1.0.0', '1.1.0']\n
 
A Literal type with the supported GeoParquet schema versions.
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_items_to_arrow","title":"parse_stac_items_to_arrow","text":"
parse_stac_items_to_arrow(\n    items: Iterable[Item | dict[str, Any]],\n    *,\n    chunk_size: int = 8192,\n    schema: Schema | InferredSchema | None = None\n) -> Iterable[RecordBatch]\n
 
Parse a collection of STAC Items to an iterable of pyarrow.RecordBatch.
 
The objects under properties are moved up to the top-level of the Table, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Iterable[Item | dict[str, Any]])           \u2013            
    the STAC Items to convert
     
    •  
  •  chunk_size               (int, default:                   8192 )           \u2013            
    The chunk size to use for Arrow record batches. This only takes effect if schema is not None. When schema is None, the input will be parsed into a single contiguous record batch. Defaults to 8192.
     
    •  
  •  schema               (Schema | InferredSchema | None, default:                   None )           \u2013            
    The schema of the input data. If provided, can improve memory use; otherwise all items need to be parsed into a single array for schema inference. Defaults to None.
     
    •  
 
Returns:
 
     
  •  Iterable[RecordBatch]           \u2013            
    an iterable of pyarrow RecordBatches with the STAC-GeoParquet representation of items.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_arrow","title":"parse_stac_ndjson_to_arrow","text":"
parse_stac_ndjson_to_arrow(\n    path: str | Path | Iterable[str | Path],\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None\n) -> Iterator[RecordBatch]\n
 
Convert one or more newline-delimited JSON STAC files to a generator of Arrow RecordBatches.
 
Each RecordBatch in the returned iterator is guaranteed to have an identical schema, and can be used to write to one or more Parquet files.
 
Parameters:
 
     
  •  path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data.
     
    •  
 
Other Parameters:
 
     
  •  limit               (int | None)           \u2013            
    The maximum number of JSON Items to use for schema inference
     
    •  
 
Yields:
 
     
  •  RecordBatch           \u2013            
    Arrow RecordBatch with a single chunk of Item data.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_delta_lake","title":"parse_stac_ndjson_to_delta_lake","text":"
parse_stac_ndjson_to_delta_lake(\n    input_path: str | Path | Iterable[str | Path],\n    table_or_uri: str | Path | DeltaTable,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to Delta Lake
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  table_or_uri               (str | Path | DeltaTable)           \u2013            
    A path to the output Delta Lake table
     
    •  
 
Parameters:
 
     
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size to use for reading JSON into memory. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None, default:                   None )           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS, default:                   DEFAULT_PARQUET_SCHEMA_VERSION )           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_parquet","title":"parse_stac_ndjson_to_parquet","text":"
parse_stac_ndjson_to_parquet(\n    input_path: str | Path | Iterable[str | Path],\n    output_path: str | Path,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | InferredSchema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to GeoParquet
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  output_path               (str | Path)           \u2013            
    A path to the output Parquet file.
     
    •  
 
Other Parameters:
 
     
  •  chunk_size               (int)           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | InferredSchema | None)           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None)           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_items","title":"stac_table_to_items","text":"
stac_table_to_items(table: Table) -> Iterable[dict]\n
 
Convert a STAC Table to a generator of STAC Item dicts
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_ndjson","title":"stac_table_to_ndjson","text":"
stac_table_to_ndjson(table: Table, dest: str | Path | PathLike[bytes]) -> None\n
 
Write a STAC Table to a newline-delimited JSON file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.to_parquet","title":"to_parquet","text":"
to_parquet(\n    table: Table,\n    where: Any,\n    *,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Write an Arrow table with STAC data to GeoParquet
 
This writes metadata compliant with either GeoParquet 1.0 or 1.1.
 
Parameters:
 
     
  •  table               (Table)           \u2013            
    The table to write to Parquet
     
    •  
  •  where               (Any)           \u2013            
    The destination for saving.
     
    •  
 
Other Parameters:
 
     
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/legacy/","title":"Direct GeoPandas conversion (Legacy)","text":"
The API listed here was the initial non-Arrow-based STAC-GeoParquet implementation, converting between JSON and GeoPandas directly. For large collections of STAC items, using the new Arrow-based functionality (under the stac_geoparquet.arrow namespace) will be more performant.
 
Note that stac_geoparquet lifts the keys in the item properties up to the top level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
>>> import requests\n>>> import stac_geoparquet.arrow\n>>> import pyarrow.parquet\n>>> import pyarrow as pa\n\n>>> items = requests.get(\n...     \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items\"\n... ).json()[\"features\"]\n>>> table = pa.Table.from_batches(stac_geoparquet.arrow.parse_stac_items_to_arrow(items))\n>>> stac_geoparquet.arrow.to_parquet(table, \"items.parquet\")\n>>> table2 = pyarrow.parquet.read_table(\"items.parquet\")\n>>> items2 = list(stac_geoparquet.arrow.stac_table_to_items(table2))\n
"},{"location":"api/legacy/#stac_geoparquet.to_geodataframe","title":"stac_geoparquet.to_geodataframe","text":"
to_geodataframe(\n    items: Sequence[dict[str, Any]],\n    add_self_link: bool = False,\n    dtype_backend: DTYPE_BACKEND | None = None,\n    datetime_precision: str = \"ns\",\n) -> GeoDataFrame\n
 
Convert a sequence of STAC items to a geopandas.GeoDataFrame.
 
The objects under properties are moved up to the top-level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Sequence[dict[str, Any]])           \u2013            
    A sequence of STAC items.
     
    •  
  •  add_self_link               (bool, default:                   False )           \u2013            
    bool, default False Add the absolute link (if available) to the source STAC Item as a separate column named \"self_link\"
     
    •  
  •  dtype_backend               (DTYPE_BACKEND | None, default:                   None )           \u2013            
    {'pyarrow', 'numpy_nullable'}, optional The dtype backend to use for storing arrays.
     
    By default, this will use 'numpy_nullable' and emit a FutureWarning that the default will change to 'pyarrow' in the next release.
     
    Set to 'numpy_nullable' to silence the warning and accept the old behavior.
     
    Set to 'pyarrow' to silence the warning and accept the new behavior.
     
    There are some difference in the output as well: with dtype_backend=\"pyarrow\", struct-like fields will explicitly contain null values for fields that appear in only some of the records. For example, given an assets like::
     
    {\n    \"a\": {\n        \"href\": \"a.tif\",\n    },\n    \"b\": {\n        \"href\": \"b.tif\",\n        \"title\": \"B\",\n    }\n}\n
     
    The assets field of the output for the first row with dtype_backend=\"numpy_nullable\" will be a Python dictionary with just {\"href\": \"a.tiff\"}.
     
    With dtype_backend=\"pyarrow\", this will be a pyarrow struct with fields {\"href\": \"a.tif\", \"title\", None}. pyarrow will infer that the struct field asset.title is nullable.
     
    •  
  •  datetime_precision               (str, default:                   'ns' )           \u2013            
    str, default \"ns\" The precision to use for the datetime columns. For example, \"us\" is microsecond and \"ns\" is nanosecond.
     
    •  
 
Returns:
 
     
  •  GeoDataFrame           \u2013            
    The converted GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_item_collection","title":"stac_geoparquet.to_item_collection","text":"
to_item_collection(df: GeoDataFrame) -> ItemCollection\n
 
Convert a GeoDataFrame of STAC items to a pystac.ItemCollection.
 
Parameters:
 
     
  •  df               (GeoDataFrame)           \u2013            
    A GeoDataFrame with a schema similar to that exported by stac-geoparquet.
     
    •  
 
Returns:
 
     
  •  ItemCollection           \u2013            
    The converted ItemCollection. There will be one record / feature per row in the in the GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_dict","title":"stac_geoparquet.to_dict","text":"
to_dict(record: dict) -> dict\n
 
Create a dictionary representing a STAC item from a row of the GeoDataFrame.
 
Parameters:
 
     
  •  record               (dict)           \u2013            
    dict
     
    •  
"},{"location":"api/pgstac/","title":"pgstac integration","text":"
stac_geoparquet.pgstac_reader has some helpers for working with items coming from a pgstac.items table. It takes care of
 
     
  • Rehydrating the dehydrated items
    •  
  • Partitioning by time
    •  
  • Injecting dynamic links and assets from a STAC API
    •  
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig","title":"stac_geoparquet.pgstac_reader.CollectionConfig  dataclass","text":"
Additional collection-based configuration to inject, matching the dynamic properties from the API.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection","title":"collection  property","text":"
collection: Collection\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection_id","title":"collection_id  instance-attribute","text":"
collection_id: str\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.partition_frequency","title":"partition_frequency  class-attribute instance-attribute","text":"
partition_frequency: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.render_config","title":"render_config  class-attribute instance-attribute","text":"
render_config: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.should_inject_dynamic_properties","title":"should_inject_dynamic_properties  class-attribute instance-attribute","text":"
should_inject_dynamic_properties: bool = True\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.stac_api","title":"stac_api  class-attribute instance-attribute","text":"
stac_api: str = 'https://planetarycomputer.microsoft.com/api/stac/v1'\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__init__","title":"__init__","text":"
__init__(\n    collection_id: str,\n    partition_frequency: str | None = None,\n    stac_api: str = \"https://planetarycomputer.microsoft.com/api/stac/v1\",\n    should_inject_dynamic_properties: bool = True,\n    render_config: str | None = None,\n) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__post_init__","title":"__post_init__","text":"
__post_init__() -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_collection","title":"export_collection","text":"
export_collection(\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> list[str | None]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition","title":"export_partition","text":"
export_partition(\n    conninfo: str,\n    query: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any] | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition_for_endpoints","title":"export_partition_for_endpoints","text":"
export_partition_for_endpoints(\n    endpoints: tuple[datetime, datetime],\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    part_number: int | None = None,\n    total: int | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
 
Export results for a pair of endpoints.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.generate_endpoints","title":"generate_endpoints","text":"
generate_endpoints(\n    since: datetime | None = None,\n) -> list[tuple[datetime, datetime]]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_assets","title":"inject_assets","text":"
inject_assets(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_links","title":"inject_links","text":"
inject_links(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.make_pgstac_items","title":"make_pgstac_items","text":"
make_pgstac_items(\n    records: list[tuple[str, str, str, datetime, datetime, dict[str, Any]]],\n    base_item: dict[str, Any],\n) -> list[dict[str, Any]]\n
 
Make STAC items out of pgstac records.
 
Parameters:
 
     
  •  records               (list[tuple[str, str, str, datetime, datetime, dict[str, Any]]])           \u2013            
    list[tuple] The dehydrated records from pgstac.items table.
     
    •  
  •  base_item               (dict[str, Any])           \u2013            
    dict[str, Any] The base item from the collection_base_item pgstac function for this collection. Used for rehydration
     
    •  
"},{"location":"spec/stac-geoparquet-spec/","title":"STAC GeoParquet Specification","text":""},{"location":"spec/stac-geoparquet-spec/#overview","title":"Overview","text":"
This document specifies how to map a set of STAC Items into GeoParquet. It is directly inspired by the STAC GeoParquet library, but aims to provide guidance for anyone putting STAC data into GeoParquet.
"},{"location":"spec/stac-geoparquet-spec/#use-cases","title":"Use cases","text":"
     
  • Provide a STAC GeoParquet that mirrors a static Collection as a way to query the whole dataset instead of reading every specific GeoJSON file.
    •  
  • As an output format for STAC API responses that is more efficient than paging through thousands of pages of GeoJSON.
    •  
  • Provide efficient access to specific fields of a STAC item, thanks to Parquet's columnar format.
    •  
"},{"location":"spec/stac-geoparquet-spec/#guidelines","title":"Guidelines","text":"
Each row in the Parquet Dataset represents a single STAC item. Most all the fields in a STAC Item should be mapped to a column in GeoParquet. We embrace Parquet structures where possible, mapping from JSON into nested structures. We do pull the properties to the top level, so that it is easier to query and use them. The names of most of the fields should be the same in STAC and in GeoParquet.
 Field GeoParquet Type Required Details type String Optional This is just needed for GeoJSON, so it is optional and not recommended to include in GeoParquet stac_extensions List of Strings Required This column is required, but can be empty if no STAC extensions were used id String Required Required, should be unique within each collection geometry Binary (WKB) Required For GeoParquet 1.0 this must be well-known Binary bbox Struct of Floats Required Can be a 4 or 6 value struct, depending on dimension of the data. It must conform to the \"Bounding Box Columns\" definition of GeoParquet 1.1. links List of Link structs Required See Link Struct for more info assets An Assets struct Required See Asset Struct for more info collection String Optional The ID of the collection this Item is a part of. See notes below on 'Collection' and 'Collection JSON' in the Parquet metadata property columns varies - Each property should use the relevant Parquet type, and be pulled out of the properties object to be a top-level Parquet field 
     
  • Must be valid GeoParquet, with proper metadata. Ideally the geometry types are defined and as narrow as possible.
    •  
  • Strongly recommend to only have one GeoParquet per STAC 'Collection'. Not doing this will lead to an expanded GeoParquet schema (the union of all the schemas of the collection) with lots of empty data
    •  
  • Any field in 'properties' of the STAC item should be moved up to be a top-level field in the GeoParquet.
    •  
  • STAC GeoParquet does not support properties that are named such that they collide with a top-level key.
    •  
  • datetime columns should be stored as a native timestamp, not as a string
    •  
  • The Collection JSON should be included in the Parquet metadata. See Collection JSON below.
    •  
  • Any other properties that would be stored as GeoJSON in a STAC JSON Item (e.g. proj:geometry) should be stored as a binary column with WKB encoding. This simplifies the handling of collections with multiple geometry types.
    •  
"},{"location":"spec/stac-geoparquet-spec/#link-struct","title":"Link Struct","text":"
The GeoParquet dataset can contain zero or more Link Structs. Each Link Struct has 2 required fields and 2 optional ones:
 Field Name Type Description href string REQUIRED. The actual link in the format of an URL. Relative and absolute links are both allowed. rel string REQUIRED. Relationship between the current document and the linked document. See chapter \"Relation types\" for more information. type string Media type of the referenced entity. title string A human readable title to be used in rendered displays of the link. 
See Link Object for more.
"},{"location":"spec/stac-geoparquet-spec/#asset-struct","title":"Asset Struct","text":"
The GeoParquet dataset can contain zero or more Asset Structs. Each Asset Struct can have the following fields:
 Field Name Type Description href string REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. title string The displayed title for clients and users. description string A description of the Asset providing additional details, such as how it was processed or created. CommonMark 0.29 syntax MAY be used for rich text representation. type string Media type of the asset. See the common media types in the best practice doc for commonly used asset types. roles [string] The semantic roles of the asset, similar to the use of rel in links. 
Each struct has each full asset key and object as a sub-struct, it's a direct mapping from the JSON to Parquet
 
To take advantage of Parquet's columnar nature and compression, the assets should be uniform so they can be represented by a simple schema, which in turn means every item should probably come from the same STAC collection.
 
See Asset Object for more.
"},{"location":"spec/stac-geoparquet-spec/#including-a-stac-collection-json-in-a-stac-geoparquet-collection","title":"Including a STAC Collection JSON in a STAC Geoparquet Collection","text":"
To make a stac-geoparquet file a fully self-contained representation, you can include the Collection JSON in the Parquet metadata. If present in the Parquet file metadata, the key must be stac:collection and the value must be a JSON string with the Collection JSON.
"},{"location":"spec/stac-geoparquet-spec/#referencing-a-stac-geoparquet-collections-in-a-stac-collection-json","title":"Referencing a STAC Geoparquet Collections in a STAC Collection JSON","text":"
A common use case of stac-geoparquet is to create a mirror of a STAC collection. To refer to this mirror in the original collection, use an Asset Object at the collection level of the STAC JSON that includes the application/vnd.apache.parquet Media type and collection-mirror Role type to describe the function of the Geoparquet STAC Collection Asset.
 
For example:
 Field Name Type Value href string s3://example/uri/to/file.parquet title string An example STAC GeoParquet. description string Example description. type string application/vnd.apache.parquet roles [string] [collection-mirror]* 
*Note the IANA has not approved the new Media type application/vnd.apache.parquet yet, it's been submitted for approval.
 
The description should ideally include details about the spatial partitioning method.
"},{"location":"spec/stac-geoparquet-spec/#mapping-to-other-geospatial-data-formats","title":"Mapping to other geospatial data formats","text":"
The principles here can likely be used to map into other geospatial data formats (GeoPackage, FlatGeobuf, etc), but we embrace Parquet's nested 'structs' for some of the mappings, so other formats will need to do something different. The obvious thing to do is to dump JSON into those fields, but that's outside the scope of this document, and we recommend creating a general document for that.
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"STAC-geoparquet","text":"
Convert STAC items between JSON, GeoParquet, pgstac, and Delta Lake.
"},{"location":"#purpose","title":"Purpose","text":"
The STAC spec defines a JSON-based schema. But it can be hard to manage and search through many millions of STAC items in JSON format. For one, JSON is very large on disk. And you need to parse the entire JSON data into memory to extract just a small piece of information, say the datetime and one asset of an Item.
 
GeoParquet can be a good complement to JSON for many bulk-access and analytic use cases. While STAC Items are commonly distributed as individual JSON files on object storage or through a STAC API, STAC GeoParquet allows users to access a large number of STAC items in bulk without making repeated HTTP requests.
 
For analytic questions like \"find the items in the Sentinel-2 collection in June 2024 over New York City with cloud cover of less than 20%\" it can be much, much faster to find the relevant data from a GeoParquet source than from JSON, because GeoParquet needs to load only the relevant columns for that query, not the full data.
 
See the STAC-GeoParquet specification for details on the exact schema of the written Parquet files.
"},{"location":"#documentation","title":"Documentation","text":"
Documentation website
"},{"location":"drawbacks/","title":"Drawbacks","text":"
Trying to represent STAC data in GeoParquet has some drawbacks.
"},{"location":"drawbacks/#unable-to-represent-undefined-values","title":"Unable to represent undefined values","text":"
Parquet is unable to represent the difference between undefined and null, and so is unable to perfectly round-trip STAC data with undefined values.
 
In JSON a value can have one of three states: defined, undefined, or null. The \"b\" key in the next three examples illustrates this:
 
Defined:
 
{\n  \"a\": 1,\n  \"b\": \"foo\"\n}\n
 
Undefined:
 
{\n  \"a\": 2\n}\n
 
Null:
 
{\n  \"a\": 3,\n  \"b\": null\n}\n
 
Because Parquet is a columnar format, it is only able to represent undefined at the column level. So if those three JSON items above were converted to Parquet, the column \"b\" would exist because it exists in the first and third item, and the second item would have \"b\" inferred as null:
 a b 1 \"foo\" 2 null 3 null 
Then when the second item is converted back to JSON, it will be returned as
 
{\n  \"a\": 2\n  \"b\": null\n}\n
 
which is not strictly equal to the input.
"},{"location":"drawbacks/#schema-difficulties","title":"Schema difficulties","text":"
JSON is schemaless while Parquet requires a strict schema, and it can be very difficult to unite these two systems. This is such an important consideration that we have a documentation page just to discuss this point.
"},{"location":"schema/","title":"Schema considerations","text":"
A STAC Item is a JSON object to describe an external geospatial dataset. The STAC specification defines a common core, plus a variety of extensions. Additionally, STAC Items may include custom extensions outside the common ones. Crucially, the majority of the specified fields in the core spec and extensions define optional keys. Those keys often differ across STAC collections and may even differ within a single collection across items.
 
STAC's flexibility is a blessing and a curse. The flexibility of schemaless JSON allows for very easy writing as each object can be dumped separately to JSON. Every item is allowed to have a different schema. And newer items are free to have a different schema than older items in the same collection. But this write-time flexibility makes it harder to read as there are no guarantees (outside STAC's few required fields) about what fields exist.
 
Parquet is the complete opposite of JSON. Parquet has a strict schema that must be known before writing can start. This puts the burden of work onto the writer instead of the reader. Reading Parquet is very efficient because the file's metadata defines the exact schema of every record. This also enables use cases like reading specific columns that would not be possible without a strict schema.
 
This conversion from schemaless to strict-schema is the difficult part of converting STAC from JSON to GeoParquet, especially for large input datasets like STAC that are often larger than memory.
"},{"location":"schema/#full-scan-over-input-data","title":"Full scan over input data","text":"
The most foolproof way to convert STAC JSON to GeoParquet is to perform a full scan over input data. This is done automatically by parse_stac_ndjson_to_arrow when a schema is not provided.
 
This is time consuming as it requires two full passes over the input data: once to infer a common schema and again to actually write to Parquet (though items are never fully held in memory, allowing this process to scale).
"},{"location":"schema/#user-provided-schema","title":"User-provided schema","text":"
Alternatively, the user can pass in an Arrow schema themselves using the schema parameter of parse_stac_ndjson_to_arrow. This schema must match the on-disk schema of the the STAC JSON data.
"},{"location":"schema/#multiple-schemas-per-collection","title":"Multiple schemas per collection","text":"
It is also possible to write multiple Parquet files with STAC data where each Parquet file may have a different schema. This simplifies the conversion and writing process but makes reading and using the Parquet data harder.
"},{"location":"schema/#merging-data-with-schema-mismatch","title":"Merging data with schema mismatch","text":"
If you've created STAC GeoParquet data where the schema has updated, you can use pyarrow.concat_tables with promote_options=\"permissive\" to combine multiple STAC GeoParquet files.
 
import pyarrow as pa\nimport pyarrow.parquet as pq\n\ntable_1 = pq.read_table(\"stac1.parquet\")\ntable_2 = pq.read_table(\"stac2.parquet\")\ncombined_table = pa.concat_tables([table1, table2], promote_options=\"permissive\")\n
"},{"location":"schema/#future-work","title":"Future work","text":"
Schema operations is an area where future work can improve reliability and ease of use of STAC GeoParquet.
 
It's possible that in the future we could automatically infer an Arrow schema from the STAC specification's published JSON Schema files. If you're interested in this, open an issue and discuss.
"},{"location":"usage/","title":"Usage","text":"
Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.
 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.
"},{"location":"usage/#dictjson-arrow-conversion","title":"dict/JSON - Arrow conversion","text":""},{"location":"usage/#convert-dicts-to-arrow","title":"Convert dicts to Arrow","text":"
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.
 
For example:
 
import pyarrow as pa\nimport pystac\n\nimport stac_geoparquet\n\nitem = pystac.read_file(\n    \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items/S2A_MSIL2A_20230112T104411_R008_T29NPE_20230113T053333\"\n)\nassert isinstance(item, pystac.Item)\n\nrecord_batch_reader = stac_geoparquet.arrow.parse_stac_items_to_arrow([item])\ntable = record_batch_reader.read_all()\n
"},{"location":"usage/#convert-json-to-arrow","title":"Convert JSON to Arrow","text":"
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.
"},{"location":"usage/#convert-arrow-to-dicts","title":"Convert Arrow to dicts","text":"
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#convert-arrow-to-json","title":"Convert Arrow to JSON","text":"
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a newline-delimited JSON file. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#parquet","title":"Parquet","text":"
Use to_parquet to write STAC Arrow data from memory to a path or file-like object. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.
 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.
 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.
"},{"location":"usage/#delta-lake","title":"Delta Lake","text":"
Use parse_stac_ndjson_to_delta_lake to read (newline-delimited) JSON on disk and write out to a Delta Lake table.
 
No special API is required for reading a STAC Delta Lake table back into Arrow. You can use the DeltaTable class directly to read the data back into Arrow.
 
Important
 
Arrow has a null data type, where every value in the column is always null, but Delta Lake does not. This means that for any column inferred to have a null data type, writing to Delta Lake will error with 
_internal.SchemaMismatchError: Invalid data type for Delta Lake: Null\n
 
This is a problem because if all items in a STAC Collection have a null JSON key, it gets inferred as an Arrow null type. For example, in the 3dep-lidar-copc collection in the tests, it has start_datetime and end_datetime fields, and so according to the spec, datetime is always null. This column would need to be casted to a timestamp type before being written to Delta Lake.
 
This means we cannot write this collection to Delta Lake solely with automatic schema inference.
 
In such cases, users may need to manually update the inferred schema to cast any null type to another Delta Lake-compatible type.
"},{"location":"api/arrow/","title":"stac_geoparquet.arrow","text":"
Arrow-based format conversions.
"},{"location":"api/arrow/#stac_geoparquet.arrow","title":"stac_geoparquet.arrow","text":""},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_JSON_CHUNK_SIZE","title":"DEFAULT_JSON_CHUNK_SIZE  module-attribute","text":"
DEFAULT_JSON_CHUNK_SIZE = 65536\n
 
The default chunk size to use for reading JSON into memory.
"},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_PARQUET_SCHEMA_VERSION","title":"DEFAULT_PARQUET_SCHEMA_VERSION  module-attribute","text":"
DEFAULT_PARQUET_SCHEMA_VERSION: SUPPORTED_PARQUET_SCHEMA_VERSIONS = '1.1.0'\n
 
The default GeoParquet schema version written to file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.SUPPORTED_PARQUET_SCHEMA_VERSIONS","title":"SUPPORTED_PARQUET_SCHEMA_VERSIONS  module-attribute","text":"
SUPPORTED_PARQUET_SCHEMA_VERSIONS = Literal['1.0.0', '1.1.0']\n
 
A Literal type with the supported GeoParquet schema versions.
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_items_to_arrow","title":"parse_stac_items_to_arrow","text":"
parse_stac_items_to_arrow(\n    items: Iterable[Item | dict[str, Any]],\n    *,\n    chunk_size: int = 8192,\n    schema: Schema | InferredSchema | None = None\n) -> RecordBatchReader\n
 
Parse a collection of STAC Items to an iterable of pyarrow.RecordBatch.
 
The objects under properties are moved up to the top-level of the Table, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Iterable[Item | dict[str, Any]])           \u2013            
    the STAC Items to convert
     
    •  
  •  chunk_size               (int, default:                   8192 )           \u2013            
    The chunk size to use for Arrow record batches. This only takes effect if schema is not None. When schema is None, the input will be parsed into a single contiguous record batch. Defaults to 8192.
     
    •  
  •  schema               (Schema | InferredSchema | None, default:                   None )           \u2013            
    The schema of the input data. If provided, can improve memory use; otherwise all items need to be parsed into a single array for schema inference. Defaults to None.
     
    •  
 
Returns:
 
     
  •  RecordBatchReader           \u2013            
    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_arrow","title":"parse_stac_ndjson_to_arrow","text":"
parse_stac_ndjson_to_arrow(\n    path: str | Path | Iterable[str | Path],\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None\n) -> RecordBatchReader\n
 
Convert one or more newline-delimited JSON STAC files to a generator of Arrow RecordBatches.
 
Each RecordBatch in the returned iterator is guaranteed to have an identical schema, and can be used to write to one or more Parquet files.
 
Parameters:
 
     
  •  path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data.
     
    •  
 
Other Parameters:
 
     
  •  limit               (int | None)           \u2013            
    The maximum number of JSON Items to use for schema inference
     
    •  
 
Returns:
 
     
  •  RecordBatchReader           \u2013            
    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_delta_lake","title":"parse_stac_ndjson_to_delta_lake","text":"
parse_stac_ndjson_to_delta_lake(\n    input_path: str | Path | Iterable[str | Path],\n    table_or_uri: str | Path | DeltaTable,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to Delta Lake
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  table_or_uri               (str | Path | DeltaTable)           \u2013            
    A path to the output Delta Lake table
     
    •  
 
Parameters:
 
     
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size to use for reading JSON into memory. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None, default:                   None )           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS, default:                   DEFAULT_PARQUET_SCHEMA_VERSION )           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_parquet","title":"parse_stac_ndjson_to_parquet","text":"
parse_stac_ndjson_to_parquet(\n    input_path: str | Path | Iterable[str | Path],\n    output_path: str | Path,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | InferredSchema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to GeoParquet
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  output_path               (str | Path)           \u2013            
    A path to the output Parquet file.
     
    •  
 
Other Parameters:
 
     
  •  chunk_size               (int)           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | InferredSchema | None)           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None)           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
 
All other keyword args are passed on to pyarrow.parquet.ParquetWriter.
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_items","title":"stac_table_to_items","text":"
stac_table_to_items(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n) -> Iterable[dict]\n
 
Convert STAC Arrow to a generator of STAC Item dicts.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
 
Yields:
 
     
  •  Iterable[dict]           \u2013            
    A STAC dict for each input row.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_ndjson","title":"stac_table_to_ndjson","text":"
stac_table_to_ndjson(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n    dest: str | Path | PathLike[bytes],\n) -> None\n
 
Write STAC Arrow to a newline-delimited JSON file.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
  •  dest               (str | Path | PathLike[bytes])           \u2013            
    The destination where newline-delimited JSON should be written.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.to_parquet","title":"to_parquet","text":"
to_parquet(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n    output_path: str | Path,\n    *,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Write an Arrow table with STAC data to GeoParquet
 
This writes metadata compliant with either GeoParquet 1.0 or 1.1.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
  •  output_path               (str | Path)           \u2013            
    The destination for saving.
     
    •  
 
Other Parameters:
 
     
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
 
All other keyword args are passed on to pyarrow.parquet.ParquetWriter.
"},{"location":"api/legacy/","title":"Direct GeoPandas conversion (Legacy)","text":"
The API listed here was the initial non-Arrow-based STAC-GeoParquet implementation, converting between JSON and GeoPandas directly. For large collections of STAC items, using the new Arrow-based functionality (under the stac_geoparquet.arrow namespace) will be more performant.
 
Note that stac_geoparquet lifts the keys in the item properties up to the top level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
>>> import requests\n>>> import stac_geoparquet.arrow\n>>> import pyarrow.parquet\n>>> import pyarrow as pa\n\n>>> items = requests.get(\n...     \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items\"\n... ).json()[\"features\"]\n>>> table = pa.Table.from_batches(stac_geoparquet.arrow.parse_stac_items_to_arrow(items))\n>>> stac_geoparquet.arrow.to_parquet(table, \"items.parquet\")\n>>> table2 = pyarrow.parquet.read_table(\"items.parquet\")\n>>> items2 = list(stac_geoparquet.arrow.stac_table_to_items(table2))\n
"},{"location":"api/legacy/#stac_geoparquet.to_geodataframe","title":"stac_geoparquet.to_geodataframe","text":"
to_geodataframe(\n    items: Sequence[dict[str, Any]],\n    add_self_link: bool = False,\n    dtype_backend: DTYPE_BACKEND | None = None,\n    datetime_precision: str = \"ns\",\n) -> GeoDataFrame\n
 
Convert a sequence of STAC items to a geopandas.GeoDataFrame.
 
The objects under properties are moved up to the top-level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Sequence[dict[str, Any]])           \u2013            
    A sequence of STAC items.
     
    •  
  •  add_self_link               (bool, default:                   False )           \u2013            
    bool, default False Add the absolute link (if available) to the source STAC Item as a separate column named \"self_link\"
     
    •  
  •  dtype_backend               (DTYPE_BACKEND | None, default:                   None )           \u2013            
    {'pyarrow', 'numpy_nullable'}, optional The dtype backend to use for storing arrays.
     
    By default, this will use 'numpy_nullable' and emit a FutureWarning that the default will change to 'pyarrow' in the next release.
     
    Set to 'numpy_nullable' to silence the warning and accept the old behavior.
     
    Set to 'pyarrow' to silence the warning and accept the new behavior.
     
    There are some difference in the output as well: with dtype_backend=\"pyarrow\", struct-like fields will explicitly contain null values for fields that appear in only some of the records. For example, given an assets like::
     
    {\n    \"a\": {\n        \"href\": \"a.tif\",\n    },\n    \"b\": {\n        \"href\": \"b.tif\",\n        \"title\": \"B\",\n    }\n}\n
     
    The assets field of the output for the first row with dtype_backend=\"numpy_nullable\" will be a Python dictionary with just {\"href\": \"a.tiff\"}.
     
    With dtype_backend=\"pyarrow\", this will be a pyarrow struct with fields {\"href\": \"a.tif\", \"title\", None}. pyarrow will infer that the struct field asset.title is nullable.
     
    •  
  •  datetime_precision               (str, default:                   'ns' )           \u2013            
    str, default \"ns\" The precision to use for the datetime columns. For example, \"us\" is microsecond and \"ns\" is nanosecond.
     
    •  
 
Returns:
 
     
  •  GeoDataFrame           \u2013            
    The converted GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_item_collection","title":"stac_geoparquet.to_item_collection","text":"
to_item_collection(df: GeoDataFrame) -> ItemCollection\n
 
Convert a GeoDataFrame of STAC items to a pystac.ItemCollection.
 
Parameters:
 
     
  •  df               (GeoDataFrame)           \u2013            
    A GeoDataFrame with a schema similar to that exported by stac-geoparquet.
     
    •  
 
Returns:
 
     
  •  ItemCollection           \u2013            
    The converted ItemCollection. There will be one record / feature per row in the in the GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_dict","title":"stac_geoparquet.to_dict","text":"
to_dict(record: dict) -> dict\n
 
Create a dictionary representing a STAC item from a row of the GeoDataFrame.
 
Parameters:
 
     
  •  record               (dict)           \u2013            
    dict
     
    •  
"},{"location":"api/pgstac/","title":"pgstac integration","text":"
stac_geoparquet.pgstac_reader has some helpers for working with items coming from a pgstac.items table. It takes care of
 
     
  • Rehydrating the dehydrated items
    •  
  • Partitioning by time
    •  
  • Injecting dynamic links and assets from a STAC API
    •  
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig","title":"stac_geoparquet.pgstac_reader.CollectionConfig  dataclass","text":"
Additional collection-based configuration to inject, matching the dynamic properties from the API.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection","title":"collection  property","text":"
collection: Collection\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection_id","title":"collection_id  instance-attribute","text":"
collection_id: str\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.partition_frequency","title":"partition_frequency  class-attribute instance-attribute","text":"
partition_frequency: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.render_config","title":"render_config  class-attribute instance-attribute","text":"
render_config: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.should_inject_dynamic_properties","title":"should_inject_dynamic_properties  class-attribute instance-attribute","text":"
should_inject_dynamic_properties: bool = True\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.stac_api","title":"stac_api  class-attribute instance-attribute","text":"
stac_api: str = 'https://planetarycomputer.microsoft.com/api/stac/v1'\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__init__","title":"__init__","text":"
__init__(\n    collection_id: str,\n    partition_frequency: str | None = None,\n    stac_api: str = \"https://planetarycomputer.microsoft.com/api/stac/v1\",\n    should_inject_dynamic_properties: bool = True,\n    render_config: str | None = None,\n) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__post_init__","title":"__post_init__","text":"
__post_init__() -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_collection","title":"export_collection","text":"
export_collection(\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> list[str | None]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition","title":"export_partition","text":"
export_partition(\n    conninfo: str,\n    query: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any] | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition_for_endpoints","title":"export_partition_for_endpoints","text":"
export_partition_for_endpoints(\n    endpoints: tuple[datetime, datetime],\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    part_number: int | None = None,\n    total: int | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
 
Export results for a pair of endpoints.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.generate_endpoints","title":"generate_endpoints","text":"
generate_endpoints(\n    since: datetime | None = None,\n) -> list[tuple[datetime, datetime]]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_assets","title":"inject_assets","text":"
inject_assets(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_links","title":"inject_links","text":"
inject_links(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.make_pgstac_items","title":"make_pgstac_items","text":"
make_pgstac_items(\n    records: list[tuple[str, str, str, datetime, datetime, dict[str, Any]]],\n    base_item: dict[str, Any],\n) -> list[dict[str, Any]]\n
 
Make STAC items out of pgstac records.
 
Parameters:
 
     
  •  records               (list[tuple[str, str, str, datetime, datetime, dict[str, Any]]])           \u2013            
    list[tuple] The dehydrated records from pgstac.items table.
     
    •  
  •  base_item               (dict[str, Any])           \u2013            
    dict[str, Any] The base item from the collection_base_item pgstac function for this collection. Used for rehydration
     
    •  
"},{"location":"spec/stac-geoparquet-spec/","title":"STAC GeoParquet Specification","text":""},{"location":"spec/stac-geoparquet-spec/#overview","title":"Overview","text":"
This document specifies how to map a set of STAC Items into GeoParquet. It is directly inspired by the STAC GeoParquet library, but aims to provide guidance for anyone putting STAC data into GeoParquet.
"},{"location":"spec/stac-geoparquet-spec/#use-cases","title":"Use cases","text":"
     
  • Provide a STAC GeoParquet that mirrors a static Collection as a way to query the whole dataset instead of reading every specific GeoJSON file.
    •  
  • As an output format for STAC API responses that is more efficient than paging through thousands of pages of GeoJSON.
    •  
  • Provide efficient access to specific fields of a STAC item, thanks to Parquet's columnar format.
    •  
"},{"location":"spec/stac-geoparquet-spec/#guidelines","title":"Guidelines","text":"
Each row in the Parquet Dataset represents a single STAC item. Most all the fields in a STAC Item should be mapped to a column in GeoParquet. We embrace Parquet structures where possible, mapping from JSON into nested structures. We do pull the properties to the top level, so that it is easier to query and use them. The names of most of the fields should be the same in STAC and in GeoParquet.
 Field GeoParquet Type Required Details type String Optional This is just needed for GeoJSON, so it is optional and not recommended to include in GeoParquet stac_extensions List of Strings Required This column is required, but can be empty if no STAC extensions were used id String Required Required, should be unique within each collection geometry Binary (WKB) Required For GeoParquet 1.0 this must be well-known Binary bbox Struct of Floats Required Can be a 4 or 6 value struct, depending on dimension of the data. It must conform to the \"Bounding Box Columns\" definition of GeoParquet 1.1. links List of Link structs Required See Link Struct for more info assets An Assets struct Required See Asset Struct for more info collection String Optional The ID of the collection this Item is a part of. See notes below on 'Collection' and 'Collection JSON' in the Parquet metadata property columns varies - Each property should use the relevant Parquet type, and be pulled out of the properties object to be a top-level Parquet field 
     
  • Must be valid GeoParquet, with proper metadata. Ideally the geometry types are defined and as narrow as possible.
    •  
  • Strongly recommend to only have one GeoParquet per STAC 'Collection'. Not doing this will lead to an expanded GeoParquet schema (the union of all the schemas of the collection) with lots of empty data
    •  
  • Any field in 'properties' of the STAC item should be moved up to be a top-level field in the GeoParquet.
    •  
  • STAC GeoParquet does not support properties that are named such that they collide with a top-level key.
    •  
  • datetime columns should be stored as a native timestamp, not as a string
    •  
  • The Collection JSON should be included in the Parquet metadata. See Collection JSON below.
    •  
  • Any other properties that would be stored as GeoJSON in a STAC JSON Item (e.g. proj:geometry) should be stored as a binary column with WKB encoding. This simplifies the handling of collections with multiple geometry types.
    •  
"},{"location":"spec/stac-geoparquet-spec/#link-struct","title":"Link Struct","text":"
The GeoParquet dataset can contain zero or more Link Structs. Each Link Struct has 2 required fields and 2 optional ones:
 Field Name Type Description href string REQUIRED. The actual link in the format of an URL. Relative and absolute links are both allowed. rel string REQUIRED. Relationship between the current document and the linked document. See chapter \"Relation types\" for more information. type string Media type of the referenced entity. title string A human readable title to be used in rendered displays of the link. 
See Link Object for more.
"},{"location":"spec/stac-geoparquet-spec/#asset-struct","title":"Asset Struct","text":"
The GeoParquet dataset can contain zero or more Asset Structs. Each Asset Struct can have the following fields:
 Field Name Type Description href string REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. title string The displayed title for clients and users. description string A description of the Asset providing additional details, such as how it was processed or created. CommonMark 0.29 syntax MAY be used for rich text representation. type string Media type of the asset. See the common media types in the best practice doc for commonly used asset types. roles [string] The semantic roles of the asset, similar to the use of rel in links. 
Each struct has each full asset key and object as a sub-struct, it's a direct mapping from the JSON to Parquet
 
To take advantage of Parquet's columnar nature and compression, the assets should be uniform so they can be represented by a simple schema, which in turn means every item should probably come from the same STAC collection.
 
See Asset Object for more.
"},{"location":"spec/stac-geoparquet-spec/#including-a-stac-collection-json-in-a-stac-geoparquet-collection","title":"Including a STAC Collection JSON in a STAC Geoparquet Collection","text":"
To make a stac-geoparquet file a fully self-contained representation, you can include the Collection JSON in the Parquet metadata. If present in the Parquet file metadata, the key must be stac:collection and the value must be a JSON string with the Collection JSON.
"},{"location":"spec/stac-geoparquet-spec/#referencing-a-stac-geoparquet-collections-in-a-stac-collection-json","title":"Referencing a STAC Geoparquet Collections in a STAC Collection JSON","text":"
A common use case of stac-geoparquet is to create a mirror of a STAC collection. To refer to this mirror in the original collection, use an Asset Object at the collection level of the STAC JSON that includes the application/vnd.apache.parquet Media type and collection-mirror Role type to describe the function of the Geoparquet STAC Collection Asset.
 
For example:
 Field Name Type Value href string s3://example/uri/to/file.parquet title string An example STAC GeoParquet. description string Example description. type string application/vnd.apache.parquet roles [string] [collection-mirror]* 
*Note the IANA has not approved the new Media type application/vnd.apache.parquet yet, it's been submitted for approval.
 
The description should ideally include details about the spatial partitioning method.
"},{"location":"spec/stac-geoparquet-spec/#mapping-to-other-geospatial-data-formats","title":"Mapping to other geospatial data formats","text":"
The principles here can likely be used to map into other geospatial data formats (GeoPackage, FlatGeobuf, etc), but we embrace Parquet's nested 'structs' for some of the mappings, so other formats will need to do something different. The obvious thing to do is to dump JSON into those fields, but that's outside the scope of this document, and we recommend creating a general document for that.
"}]}
\ No newline at end of file
diff --git a/0.5.1/sitemap.xml b/0.5.1/sitemap.xml
index c7f9c19..ddbc218 100644
--- a/0.5.1/sitemap.xml
+++ b/0.5.1/sitemap.xml
@@ -2,42 +2,42 @@
 
     
          https://stac-utils.github.io/stac-geoparquet/latest/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/drawbacks/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/schema/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/usage/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/arrow/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/legacy/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/pgstac/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/spec/stac-geoparquet-spec/
-         2024-06-24
+         2024-06-25
          daily
     
 
\ No newline at end of file
diff --git a/0.5.1/sitemap.xml.gz b/0.5.1/sitemap.xml.gz
index ff0ffba75439d5d6bfa656bfc6c1d004f0296c58..207add5b981efa0ac6084b502afcbb3a5081310a 100644
GIT binary patch
literal 278
zcmV+x0qOo9iwFpS2zq7$|8r?{Wo=<_E_iKh0M(SiPQ)M(hVT0n4fjHA)Tn9Mn@^w*
zFiS^DmKK(w)_r?vyD^PtJunyGpPxS;A=fGA0H909${5XS#VC=0CJ7psjKZt(?+Xa{}pgt%s?ClowVx9;z=7kL22nfN=`C(hspl
cdyO8|?_Rlz|9*^hNZ-Ev1}tNkza|C%09hZ0G5`Po

diff --git a/0.5.1/usage/index.html b/0.5.1/usage/index.html
index d1a9546..2b0a3f1 100644
--- a/0.5.1/usage/index.html
+++ b/0.5.1/usage/index.html
@@ -733,19 +733,33 @@
 
 
 
Usage

-
Except for the legacy API, Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.

+
Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.

 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.

 
dict/JSON - Arrow conversion

 
Convert dicts to Arrow

 
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.

+
For example:

+
import pyarrow as pa
+import pystac
+
+import stac_geoparquet
+
+item = pystac.read_file(
+    "https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items/S2A_MSIL2A_20230112T104411_R008_T29NPE_20230113T053333"
+)
+assert isinstance(item, pystac.Item)
+
+record_batch_reader = stac_geoparquet.arrow.parse_stac_items_to_arrow([item])
+table = record_batch_reader.read_all()
+

 
Convert JSON to Arrow

 
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.

 
Convert Arrow to dicts

 
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

 
Convert Arrow to JSON

-
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

+
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a newline-delimited JSON file. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

 
Parquet

-
Use to_parquet to write STAC Arrow data in memory. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.

+
Use to_parquet to write STAC Arrow data from memory to a path or file-like object. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.

 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.

 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.

 
Delta Lake

diff --git a/latest/api/arrow/index.html b/latest/api/arrow/index.html
index 17bb1c8..304788a 100644
--- a/latest/api/arrow/index.html
+++ b/latest/api/arrow/index.html
@@ -916,7 +916,7 @@ 
*,
     chunk_size: int = 8192,
     schema: Schema | InferredSchema | None = None
-) -> Iterable[RecordBatch]
+) -> RecordBatchReader
 

 
     

@@ -968,10 +968,10 @@ 
Returns:

     
    
         
  • 
-              Iterable[RecordBatch]
+              RecordBatchReader
           –
           
    
-            
    an iterable of pyarrow RecordBatches with the STAC-GeoParquet representation of items.
    
+            
    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.
    
           
    
         
    • 
     

@@ -994,7 +994,7 @@ 
chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,
     schema: Schema | None = None,
     limit: int | None = None
-) -> Iterator[RecordBatch]
+) -> RecordBatchReader
@@ -1054,13 +1054,13 @@

Yields:

+

Returns:

  • - RecordBatch + RecordBatchReader –
    -

    Arrow RecordBatch with a single chunk of Item data.

    +

    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.

@@ -1255,6 +1255,8 @@

pyarrow.parquet.ParquetWriter.

@@ -1268,12 +1270,43 @@

-
stac_table_to_items(table: Table) -> Iterable[dict]
+
stac_table_to_items(
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+) -> Iterable[dict]
 

 
     

 
-      
Convert a STAC Table to a generator of STAC Item dicts

+      
Convert STAC Arrow to a generator of STAC Item dicts.

+
+
+
Parameters:

+    
    
+        
  • 
+          table
+              (Table | RecordBatchReader | ArrowStreamExportable)
+          –
+          
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
+          
    
+        
    • 
+    

+
+
+
Yields:

+    
    
+        
  • 
+              Iterable[dict]
+          –
+          
    
+            
    A STAC dict for each input row.
    
+          
    
+        
    • 
+    

 
     

 
@@ -1287,12 +1320,40 @@ 

 
 
 

-
stac_table_to_ndjson(table: Table, dest: str | Path | PathLike[bytes]) -> None
+
stac_table_to_ndjson(
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+    dest: str | Path | PathLike[bytes],
+) -> None
 

 
     

 
-      
Write a STAC Table to a newline-delimited JSON file.

+      
Write STAC Arrow to a newline-delimited JSON file.

+
+
+
Parameters:

+    
    
+        
  • 
+          table
+              (Table | RecordBatchReader | ArrowStreamExportable)
+          –
+          
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
+          
    
+        
    • 
+        
  • 
+          dest
+              (str | Path | PathLike[bytes])
+          –
+          
    
+            
    The destination where newline-delimited JSON should be written.
    
+          
    
+        
    • 
+    

 
     

 
@@ -1307,8 +1368,8 @@ 

 
 

 
to_parquet(
-    table: Table,
-    where: Any,
+    table: Table | RecordBatchReader | ArrowStreamExportable,
+    output_path: str | Path,
     *,
     schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,
     **kwargs: Any
@@ -1325,15 +1386,19 @@ 

     
    
         
  • 
           table
-              (Table)
+              (Table | RecordBatchReader | ArrowStreamExportable)
           –
           
    
-            
    The table to write to Parquet
    
+            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow
+RecordBatchReader, or any other Arrow stream object exposed through the
+Arrow PyCapsule
+Interface.
+A RecordBatchReader or stream object will not be materialized in memory.
    
           
    
         
    • 
         
  • 
-          where
-              (Any)
+          output_path
+              (str | Path)
           –
           
    
             
    The destination for saving.
    
@@ -1354,6 +1419,8 @@ 
    
           
    
         
    • 
     

+      
All other keyword args are passed on to
+pyarrow.parquet.ParquetWriter.

 
     

 
diff --git a/latest/search/search_index.json b/latest/search/search_index.json
index 582ab72..e02f440 100644
--- a/latest/search/search_index.json
+++ b/latest/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"STAC-geoparquet","text":"
Convert STAC items between JSON, GeoParquet, pgstac, and Delta Lake.
"},{"location":"#purpose","title":"Purpose","text":"
The STAC spec defines a JSON-based schema. But it can be hard to manage and search through many millions of STAC items in JSON format. For one, JSON is very large on disk. And you need to parse the entire JSON data into memory to extract just a small piece of information, say the datetime and one asset of an Item.
 
GeoParquet can be a good complement to JSON for many bulk-access and analytic use cases. While STAC Items are commonly distributed as individual JSON files on object storage or through a STAC API, STAC GeoParquet allows users to access a large number of STAC items in bulk without making repeated HTTP requests.
 
For analytic questions like \"find the items in the Sentinel-2 collection in June 2024 over New York City with cloud cover of less than 20%\" it can be much, much faster to find the relevant data from a GeoParquet source than from JSON, because GeoParquet needs to load only the relevant columns for that query, not the full data.
 
See the STAC-GeoParquet specification for details on the exact schema of the written Parquet files.
"},{"location":"#documentation","title":"Documentation","text":"
Documentation website
"},{"location":"drawbacks/","title":"Drawbacks","text":"
Trying to represent STAC data in GeoParquet has some drawbacks.
"},{"location":"drawbacks/#unable-to-represent-undefined-values","title":"Unable to represent undefined values","text":"
Parquet is unable to represent the difference between undefined and null, and so is unable to perfectly round-trip STAC data with undefined values.
 
In JSON a value can have one of three states: defined, undefined, or null. The \"b\" key in the next three examples illustrates this:
 
Defined:
 
{\n  \"a\": 1,\n  \"b\": \"foo\"\n}\n
 
Undefined:
 
{\n  \"a\": 2\n}\n
 
Null:
 
{\n  \"a\": 3,\n  \"b\": null\n}\n
 
Because Parquet is a columnar format, it is only able to represent undefined at the column level. So if those three JSON items above were converted to Parquet, the column \"b\" would exist because it exists in the first and third item, and the second item would have \"b\" inferred as null:
 a b 1 \"foo\" 2 null 3 null 
Then when the second item is converted back to JSON, it will be returned as
 
{\n  \"a\": 2\n  \"b\": null\n}\n
 
which is not strictly equal to the input.
"},{"location":"drawbacks/#schema-difficulties","title":"Schema difficulties","text":"
JSON is schemaless while Parquet requires a strict schema, and it can be very difficult to unite these two systems. This is such an important consideration that we have a documentation page just to discuss this point.
"},{"location":"schema/","title":"Schema considerations","text":"
A STAC Item is a JSON object to describe an external geospatial dataset. The STAC specification defines a common core, plus a variety of extensions. Additionally, STAC Items may include custom extensions outside the common ones. Crucially, the majority of the specified fields in the core spec and extensions define optional keys. Those keys often differ across STAC collections and may even differ within a single collection across items.
 
STAC's flexibility is a blessing and a curse. The flexibility of schemaless JSON allows for very easy writing as each object can be dumped separately to JSON. Every item is allowed to have a different schema. And newer items are free to have a different schema than older items in the same collection. But this write-time flexibility makes it harder to read as there are no guarantees (outside STAC's few required fields) about what fields exist.
 
Parquet is the complete opposite of JSON. Parquet has a strict schema that must be known before writing can start. This puts the burden of work onto the writer instead of the reader. Reading Parquet is very efficient because the file's metadata defines the exact schema of every record. This also enables use cases like reading specific columns that would not be possible without a strict schema.
 
This conversion from schemaless to strict-schema is the difficult part of converting STAC from JSON to GeoParquet, especially for large input datasets like STAC that are often larger than memory.
"},{"location":"schema/#full-scan-over-input-data","title":"Full scan over input data","text":"
The most foolproof way to convert STAC JSON to GeoParquet is to perform a full scan over input data. This is done automatically by parse_stac_ndjson_to_arrow when a schema is not provided.
 
This is time consuming as it requires two full passes over the input data: once to infer a common schema and again to actually write to Parquet (though items are never fully held in memory, allowing this process to scale).
"},{"location":"schema/#user-provided-schema","title":"User-provided schema","text":"
Alternatively, the user can pass in an Arrow schema themselves using the schema parameter of parse_stac_ndjson_to_arrow. This schema must match the on-disk schema of the the STAC JSON data.
"},{"location":"schema/#multiple-schemas-per-collection","title":"Multiple schemas per collection","text":"
It is also possible to write multiple Parquet files with STAC data where each Parquet file may have a different schema. This simplifies the conversion and writing process but makes reading and using the Parquet data harder.
"},{"location":"schema/#merging-data-with-schema-mismatch","title":"Merging data with schema mismatch","text":"
If you've created STAC GeoParquet data where the schema has updated, you can use pyarrow.concat_tables with promote_options=\"permissive\" to combine multiple STAC GeoParquet files.
 
import pyarrow as pa\nimport pyarrow.parquet as pq\n\ntable_1 = pq.read_table(\"stac1.parquet\")\ntable_2 = pq.read_table(\"stac2.parquet\")\ncombined_table = pa.concat_tables([table1, table2], promote_options=\"permissive\")\n
"},{"location":"schema/#future-work","title":"Future work","text":"
Schema operations is an area where future work can improve reliability and ease of use of STAC GeoParquet.
 
It's possible that in the future we could automatically infer an Arrow schema from the STAC specification's published JSON Schema files. If you're interested in this, open an issue and discuss.
"},{"location":"usage/","title":"Usage","text":"
Except for the legacy API, Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.
 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.
"},{"location":"usage/#dictjson-arrow-conversion","title":"dict/JSON - Arrow conversion","text":""},{"location":"usage/#convert-dicts-to-arrow","title":"Convert dicts to Arrow","text":"
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.
"},{"location":"usage/#convert-json-to-arrow","title":"Convert JSON to Arrow","text":"
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.
"},{"location":"usage/#convert-arrow-to-dicts","title":"Convert Arrow to dicts","text":"
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#convert-arrow-to-json","title":"Convert Arrow to JSON","text":"
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#parquet","title":"Parquet","text":"
Use to_parquet to write STAC Arrow data in memory. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.
 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.
 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.
"},{"location":"usage/#delta-lake","title":"Delta Lake","text":"
Use parse_stac_ndjson_to_delta_lake to read (newline-delimited) JSON on disk and write out to a Delta Lake table.
 
No special API is required for reading a STAC Delta Lake table back into Arrow. You can use the DeltaTable class directly to read the data back into Arrow.
 
Important
 
Arrow has a null data type, where every value in the column is always null, but Delta Lake does not. This means that for any column inferred to have a null data type, writing to Delta Lake will error with 
_internal.SchemaMismatchError: Invalid data type for Delta Lake: Null\n
 
This is a problem because if all items in a STAC Collection have a null JSON key, it gets inferred as an Arrow null type. For example, in the 3dep-lidar-copc collection in the tests, it has start_datetime and end_datetime fields, and so according to the spec, datetime is always null. This column would need to be casted to a timestamp type before being written to Delta Lake.
 
This means we cannot write this collection to Delta Lake solely with automatic schema inference.
 
In such cases, users may need to manually update the inferred schema to cast any null type to another Delta Lake-compatible type.
"},{"location":"api/arrow/","title":"stac_geoparquet.arrow","text":"
Arrow-based format conversions.
"},{"location":"api/arrow/#stac_geoparquet.arrow","title":"stac_geoparquet.arrow","text":""},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_JSON_CHUNK_SIZE","title":"DEFAULT_JSON_CHUNK_SIZE  module-attribute","text":"
DEFAULT_JSON_CHUNK_SIZE = 65536\n
 
The default chunk size to use for reading JSON into memory.
"},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_PARQUET_SCHEMA_VERSION","title":"DEFAULT_PARQUET_SCHEMA_VERSION  module-attribute","text":"
DEFAULT_PARQUET_SCHEMA_VERSION: SUPPORTED_PARQUET_SCHEMA_VERSIONS = '1.1.0'\n
 
The default GeoParquet schema version written to file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.SUPPORTED_PARQUET_SCHEMA_VERSIONS","title":"SUPPORTED_PARQUET_SCHEMA_VERSIONS  module-attribute","text":"
SUPPORTED_PARQUET_SCHEMA_VERSIONS = Literal['1.0.0', '1.1.0']\n
 
A Literal type with the supported GeoParquet schema versions.
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_items_to_arrow","title":"parse_stac_items_to_arrow","text":"
parse_stac_items_to_arrow(\n    items: Iterable[Item | dict[str, Any]],\n    *,\n    chunk_size: int = 8192,\n    schema: Schema | InferredSchema | None = None\n) -> Iterable[RecordBatch]\n
 
Parse a collection of STAC Items to an iterable of pyarrow.RecordBatch.
 
The objects under properties are moved up to the top-level of the Table, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Iterable[Item | dict[str, Any]])           \u2013            
    the STAC Items to convert
     
    •  
  •  chunk_size               (int, default:                   8192 )           \u2013            
    The chunk size to use for Arrow record batches. This only takes effect if schema is not None. When schema is None, the input will be parsed into a single contiguous record batch. Defaults to 8192.
     
    •  
  •  schema               (Schema | InferredSchema | None, default:                   None )           \u2013            
    The schema of the input data. If provided, can improve memory use; otherwise all items need to be parsed into a single array for schema inference. Defaults to None.
     
    •  
 
Returns:
 
     
  •  Iterable[RecordBatch]           \u2013            
    an iterable of pyarrow RecordBatches with the STAC-GeoParquet representation of items.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_arrow","title":"parse_stac_ndjson_to_arrow","text":"
parse_stac_ndjson_to_arrow(\n    path: str | Path | Iterable[str | Path],\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None\n) -> Iterator[RecordBatch]\n
 
Convert one or more newline-delimited JSON STAC files to a generator of Arrow RecordBatches.
 
Each RecordBatch in the returned iterator is guaranteed to have an identical schema, and can be used to write to one or more Parquet files.
 
Parameters:
 
     
  •  path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data.
     
    •  
 
Other Parameters:
 
     
  •  limit               (int | None)           \u2013            
    The maximum number of JSON Items to use for schema inference
     
    •  
 
Yields:
 
     
  •  RecordBatch           \u2013            
    Arrow RecordBatch with a single chunk of Item data.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_delta_lake","title":"parse_stac_ndjson_to_delta_lake","text":"
parse_stac_ndjson_to_delta_lake(\n    input_path: str | Path | Iterable[str | Path],\n    table_or_uri: str | Path | DeltaTable,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to Delta Lake
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  table_or_uri               (str | Path | DeltaTable)           \u2013            
    A path to the output Delta Lake table
     
    •  
 
Parameters:
 
     
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size to use for reading JSON into memory. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None, default:                   None )           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS, default:                   DEFAULT_PARQUET_SCHEMA_VERSION )           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_parquet","title":"parse_stac_ndjson_to_parquet","text":"
parse_stac_ndjson_to_parquet(\n    input_path: str | Path | Iterable[str | Path],\n    output_path: str | Path,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | InferredSchema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to GeoParquet
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  output_path               (str | Path)           \u2013            
    A path to the output Parquet file.
     
    •  
 
Other Parameters:
 
     
  •  chunk_size               (int)           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | InferredSchema | None)           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None)           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_items","title":"stac_table_to_items","text":"
stac_table_to_items(table: Table) -> Iterable[dict]\n
 
Convert a STAC Table to a generator of STAC Item dicts
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_ndjson","title":"stac_table_to_ndjson","text":"
stac_table_to_ndjson(table: Table, dest: str | Path | PathLike[bytes]) -> None\n
 
Write a STAC Table to a newline-delimited JSON file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.to_parquet","title":"to_parquet","text":"
to_parquet(\n    table: Table,\n    where: Any,\n    *,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Write an Arrow table with STAC data to GeoParquet
 
This writes metadata compliant with either GeoParquet 1.0 or 1.1.
 
Parameters:
 
     
  •  table               (Table)           \u2013            
    The table to write to Parquet
     
    •  
  •  where               (Any)           \u2013            
    The destination for saving.
     
    •  
 
Other Parameters:
 
     
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/legacy/","title":"Direct GeoPandas conversion (Legacy)","text":"
The API listed here was the initial non-Arrow-based STAC-GeoParquet implementation, converting between JSON and GeoPandas directly. For large collections of STAC items, using the new Arrow-based functionality (under the stac_geoparquet.arrow namespace) will be more performant.
 
Note that stac_geoparquet lifts the keys in the item properties up to the top level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
>>> import requests\n>>> import stac_geoparquet.arrow\n>>> import pyarrow.parquet\n>>> import pyarrow as pa\n\n>>> items = requests.get(\n...     \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items\"\n... ).json()[\"features\"]\n>>> table = pa.Table.from_batches(stac_geoparquet.arrow.parse_stac_items_to_arrow(items))\n>>> stac_geoparquet.arrow.to_parquet(table, \"items.parquet\")\n>>> table2 = pyarrow.parquet.read_table(\"items.parquet\")\n>>> items2 = list(stac_geoparquet.arrow.stac_table_to_items(table2))\n
"},{"location":"api/legacy/#stac_geoparquet.to_geodataframe","title":"stac_geoparquet.to_geodataframe","text":"
to_geodataframe(\n    items: Sequence[dict[str, Any]],\n    add_self_link: bool = False,\n    dtype_backend: DTYPE_BACKEND | None = None,\n    datetime_precision: str = \"ns\",\n) -> GeoDataFrame\n
 
Convert a sequence of STAC items to a geopandas.GeoDataFrame.
 
The objects under properties are moved up to the top-level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Sequence[dict[str, Any]])           \u2013            
    A sequence of STAC items.
     
    •  
  •  add_self_link               (bool, default:                   False )           \u2013            
    bool, default False Add the absolute link (if available) to the source STAC Item as a separate column named \"self_link\"
     
    •  
  •  dtype_backend               (DTYPE_BACKEND | None, default:                   None )           \u2013            
    {'pyarrow', 'numpy_nullable'}, optional The dtype backend to use for storing arrays.
     
    By default, this will use 'numpy_nullable' and emit a FutureWarning that the default will change to 'pyarrow' in the next release.
     
    Set to 'numpy_nullable' to silence the warning and accept the old behavior.
     
    Set to 'pyarrow' to silence the warning and accept the new behavior.
     
    There are some difference in the output as well: with dtype_backend=\"pyarrow\", struct-like fields will explicitly contain null values for fields that appear in only some of the records. For example, given an assets like::
     
    {\n    \"a\": {\n        \"href\": \"a.tif\",\n    },\n    \"b\": {\n        \"href\": \"b.tif\",\n        \"title\": \"B\",\n    }\n}\n
     
    The assets field of the output for the first row with dtype_backend=\"numpy_nullable\" will be a Python dictionary with just {\"href\": \"a.tiff\"}.
     
    With dtype_backend=\"pyarrow\", this will be a pyarrow struct with fields {\"href\": \"a.tif\", \"title\", None}. pyarrow will infer that the struct field asset.title is nullable.
     
    •  
  •  datetime_precision               (str, default:                   'ns' )           \u2013            
    str, default \"ns\" The precision to use for the datetime columns. For example, \"us\" is microsecond and \"ns\" is nanosecond.
     
    •  
 
Returns:
 
     
  •  GeoDataFrame           \u2013            
    The converted GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_item_collection","title":"stac_geoparquet.to_item_collection","text":"
to_item_collection(df: GeoDataFrame) -> ItemCollection\n
 
Convert a GeoDataFrame of STAC items to a pystac.ItemCollection.
 
Parameters:
 
     
  •  df               (GeoDataFrame)           \u2013            
    A GeoDataFrame with a schema similar to that exported by stac-geoparquet.
     
    •  
 
Returns:
 
     
  •  ItemCollection           \u2013            
    The converted ItemCollection. There will be one record / feature per row in the in the GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_dict","title":"stac_geoparquet.to_dict","text":"
to_dict(record: dict) -> dict\n
 
Create a dictionary representing a STAC item from a row of the GeoDataFrame.
 
Parameters:
 
     
  •  record               (dict)           \u2013            
    dict
     
    •  
"},{"location":"api/pgstac/","title":"pgstac integration","text":"
stac_geoparquet.pgstac_reader has some helpers for working with items coming from a pgstac.items table. It takes care of
 
     
  • Rehydrating the dehydrated items
    •  
  • Partitioning by time
    •  
  • Injecting dynamic links and assets from a STAC API
    •  
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig","title":"stac_geoparquet.pgstac_reader.CollectionConfig  dataclass","text":"
Additional collection-based configuration to inject, matching the dynamic properties from the API.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection","title":"collection  property","text":"
collection: Collection\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection_id","title":"collection_id  instance-attribute","text":"
collection_id: str\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.partition_frequency","title":"partition_frequency  class-attribute instance-attribute","text":"
partition_frequency: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.render_config","title":"render_config  class-attribute instance-attribute","text":"
render_config: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.should_inject_dynamic_properties","title":"should_inject_dynamic_properties  class-attribute instance-attribute","text":"
should_inject_dynamic_properties: bool = True\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.stac_api","title":"stac_api  class-attribute instance-attribute","text":"
stac_api: str = 'https://planetarycomputer.microsoft.com/api/stac/v1'\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__init__","title":"__init__","text":"
__init__(\n    collection_id: str,\n    partition_frequency: str | None = None,\n    stac_api: str = \"https://planetarycomputer.microsoft.com/api/stac/v1\",\n    should_inject_dynamic_properties: bool = True,\n    render_config: str | None = None,\n) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__post_init__","title":"__post_init__","text":"
__post_init__() -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_collection","title":"export_collection","text":"
export_collection(\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> list[str | None]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition","title":"export_partition","text":"
export_partition(\n    conninfo: str,\n    query: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any] | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition_for_endpoints","title":"export_partition_for_endpoints","text":"
export_partition_for_endpoints(\n    endpoints: tuple[datetime, datetime],\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    part_number: int | None = None,\n    total: int | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
 
Export results for a pair of endpoints.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.generate_endpoints","title":"generate_endpoints","text":"
generate_endpoints(\n    since: datetime | None = None,\n) -> list[tuple[datetime, datetime]]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_assets","title":"inject_assets","text":"
inject_assets(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_links","title":"inject_links","text":"
inject_links(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.make_pgstac_items","title":"make_pgstac_items","text":"
make_pgstac_items(\n    records: list[tuple[str, str, str, datetime, datetime, dict[str, Any]]],\n    base_item: dict[str, Any],\n) -> list[dict[str, Any]]\n
 
Make STAC items out of pgstac records.
 
Parameters:
 
     
  •  records               (list[tuple[str, str, str, datetime, datetime, dict[str, Any]]])           \u2013            
    list[tuple] The dehydrated records from pgstac.items table.
     
    •  
  •  base_item               (dict[str, Any])           \u2013            
    dict[str, Any] The base item from the collection_base_item pgstac function for this collection. Used for rehydration
     
    •  
"},{"location":"spec/stac-geoparquet-spec/","title":"STAC GeoParquet Specification","text":""},{"location":"spec/stac-geoparquet-spec/#overview","title":"Overview","text":"
This document specifies how to map a set of STAC Items into GeoParquet. It is directly inspired by the STAC GeoParquet library, but aims to provide guidance for anyone putting STAC data into GeoParquet.
"},{"location":"spec/stac-geoparquet-spec/#use-cases","title":"Use cases","text":"
     
  • Provide a STAC GeoParquet that mirrors a static Collection as a way to query the whole dataset instead of reading every specific GeoJSON file.
    •  
  • As an output format for STAC API responses that is more efficient than paging through thousands of pages of GeoJSON.
    •  
  • Provide efficient access to specific fields of a STAC item, thanks to Parquet's columnar format.
    •  
"},{"location":"spec/stac-geoparquet-spec/#guidelines","title":"Guidelines","text":"
Each row in the Parquet Dataset represents a single STAC item. Most all the fields in a STAC Item should be mapped to a column in GeoParquet. We embrace Parquet structures where possible, mapping from JSON into nested structures. We do pull the properties to the top level, so that it is easier to query and use them. The names of most of the fields should be the same in STAC and in GeoParquet.
 Field GeoParquet Type Required Details type String Optional This is just needed for GeoJSON, so it is optional and not recommended to include in GeoParquet stac_extensions List of Strings Required This column is required, but can be empty if no STAC extensions were used id String Required Required, should be unique within each collection geometry Binary (WKB) Required For GeoParquet 1.0 this must be well-known Binary bbox Struct of Floats Required Can be a 4 or 6 value struct, depending on dimension of the data. It must conform to the \"Bounding Box Columns\" definition of GeoParquet 1.1. links List of Link structs Required See Link Struct for more info assets An Assets struct Required See Asset Struct for more info collection String Optional The ID of the collection this Item is a part of. See notes below on 'Collection' and 'Collection JSON' in the Parquet metadata property columns varies - Each property should use the relevant Parquet type, and be pulled out of the properties object to be a top-level Parquet field 
     
  • Must be valid GeoParquet, with proper metadata. Ideally the geometry types are defined and as narrow as possible.
    •  
  • Strongly recommend to only have one GeoParquet per STAC 'Collection'. Not doing this will lead to an expanded GeoParquet schema (the union of all the schemas of the collection) with lots of empty data
    •  
  • Any field in 'properties' of the STAC item should be moved up to be a top-level field in the GeoParquet.
    •  
  • STAC GeoParquet does not support properties that are named such that they collide with a top-level key.
    •  
  • datetime columns should be stored as a native timestamp, not as a string
    •  
  • The Collection JSON should be included in the Parquet metadata. See Collection JSON below.
    •  
  • Any other properties that would be stored as GeoJSON in a STAC JSON Item (e.g. proj:geometry) should be stored as a binary column with WKB encoding. This simplifies the handling of collections with multiple geometry types.
    •  
"},{"location":"spec/stac-geoparquet-spec/#link-struct","title":"Link Struct","text":"
The GeoParquet dataset can contain zero or more Link Structs. Each Link Struct has 2 required fields and 2 optional ones:
 Field Name Type Description href string REQUIRED. The actual link in the format of an URL. Relative and absolute links are both allowed. rel string REQUIRED. Relationship between the current document and the linked document. See chapter \"Relation types\" for more information. type string Media type of the referenced entity. title string A human readable title to be used in rendered displays of the link. 
See Link Object for more.
"},{"location":"spec/stac-geoparquet-spec/#asset-struct","title":"Asset Struct","text":"
The GeoParquet dataset can contain zero or more Asset Structs. Each Asset Struct can have the following fields:
 Field Name Type Description href string REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. title string The displayed title for clients and users. description string A description of the Asset providing additional details, such as how it was processed or created. CommonMark 0.29 syntax MAY be used for rich text representation. type string Media type of the asset. See the common media types in the best practice doc for commonly used asset types. roles [string] The semantic roles of the asset, similar to the use of rel in links. 
Each struct has each full asset key and object as a sub-struct, it's a direct mapping from the JSON to Parquet
 
To take advantage of Parquet's columnar nature and compression, the assets should be uniform so they can be represented by a simple schema, which in turn means every item should probably come from the same STAC collection.
 
See Asset Object for more.
"},{"location":"spec/stac-geoparquet-spec/#including-a-stac-collection-json-in-a-stac-geoparquet-collection","title":"Including a STAC Collection JSON in a STAC Geoparquet Collection","text":"
To make a stac-geoparquet file a fully self-contained representation, you can include the Collection JSON in the Parquet metadata. If present in the Parquet file metadata, the key must be stac:collection and the value must be a JSON string with the Collection JSON.
"},{"location":"spec/stac-geoparquet-spec/#referencing-a-stac-geoparquet-collections-in-a-stac-collection-json","title":"Referencing a STAC Geoparquet Collections in a STAC Collection JSON","text":"
A common use case of stac-geoparquet is to create a mirror of a STAC collection. To refer to this mirror in the original collection, use an Asset Object at the collection level of the STAC JSON that includes the application/vnd.apache.parquet Media type and collection-mirror Role type to describe the function of the Geoparquet STAC Collection Asset.
 
For example:
 Field Name Type Value href string s3://example/uri/to/file.parquet title string An example STAC GeoParquet. description string Example description. type string application/vnd.apache.parquet roles [string] [collection-mirror]* 
*Note the IANA has not approved the new Media type application/vnd.apache.parquet yet, it's been submitted for approval.
 
The description should ideally include details about the spatial partitioning method.
"},{"location":"spec/stac-geoparquet-spec/#mapping-to-other-geospatial-data-formats","title":"Mapping to other geospatial data formats","text":"
The principles here can likely be used to map into other geospatial data formats (GeoPackage, FlatGeobuf, etc), but we embrace Parquet's nested 'structs' for some of the mappings, so other formats will need to do something different. The obvious thing to do is to dump JSON into those fields, but that's outside the scope of this document, and we recommend creating a general document for that.
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"STAC-geoparquet","text":"
Convert STAC items between JSON, GeoParquet, pgstac, and Delta Lake.
"},{"location":"#purpose","title":"Purpose","text":"
The STAC spec defines a JSON-based schema. But it can be hard to manage and search through many millions of STAC items in JSON format. For one, JSON is very large on disk. And you need to parse the entire JSON data into memory to extract just a small piece of information, say the datetime and one asset of an Item.
 
GeoParquet can be a good complement to JSON for many bulk-access and analytic use cases. While STAC Items are commonly distributed as individual JSON files on object storage or through a STAC API, STAC GeoParquet allows users to access a large number of STAC items in bulk without making repeated HTTP requests.
 
For analytic questions like \"find the items in the Sentinel-2 collection in June 2024 over New York City with cloud cover of less than 20%\" it can be much, much faster to find the relevant data from a GeoParquet source than from JSON, because GeoParquet needs to load only the relevant columns for that query, not the full data.
 
See the STAC-GeoParquet specification for details on the exact schema of the written Parquet files.
"},{"location":"#documentation","title":"Documentation","text":"
Documentation website
"},{"location":"drawbacks/","title":"Drawbacks","text":"
Trying to represent STAC data in GeoParquet has some drawbacks.
"},{"location":"drawbacks/#unable-to-represent-undefined-values","title":"Unable to represent undefined values","text":"
Parquet is unable to represent the difference between undefined and null, and so is unable to perfectly round-trip STAC data with undefined values.
 
In JSON a value can have one of three states: defined, undefined, or null. The \"b\" key in the next three examples illustrates this:
 
Defined:
 
{\n  \"a\": 1,\n  \"b\": \"foo\"\n}\n
 
Undefined:
 
{\n  \"a\": 2\n}\n
 
Null:
 
{\n  \"a\": 3,\n  \"b\": null\n}\n
 
Because Parquet is a columnar format, it is only able to represent undefined at the column level. So if those three JSON items above were converted to Parquet, the column \"b\" would exist because it exists in the first and third item, and the second item would have \"b\" inferred as null:
 a b 1 \"foo\" 2 null 3 null 
Then when the second item is converted back to JSON, it will be returned as
 
{\n  \"a\": 2\n  \"b\": null\n}\n
 
which is not strictly equal to the input.
"},{"location":"drawbacks/#schema-difficulties","title":"Schema difficulties","text":"
JSON is schemaless while Parquet requires a strict schema, and it can be very difficult to unite these two systems. This is such an important consideration that we have a documentation page just to discuss this point.
"},{"location":"schema/","title":"Schema considerations","text":"
A STAC Item is a JSON object to describe an external geospatial dataset. The STAC specification defines a common core, plus a variety of extensions. Additionally, STAC Items may include custom extensions outside the common ones. Crucially, the majority of the specified fields in the core spec and extensions define optional keys. Those keys often differ across STAC collections and may even differ within a single collection across items.
 
STAC's flexibility is a blessing and a curse. The flexibility of schemaless JSON allows for very easy writing as each object can be dumped separately to JSON. Every item is allowed to have a different schema. And newer items are free to have a different schema than older items in the same collection. But this write-time flexibility makes it harder to read as there are no guarantees (outside STAC's few required fields) about what fields exist.
 
Parquet is the complete opposite of JSON. Parquet has a strict schema that must be known before writing can start. This puts the burden of work onto the writer instead of the reader. Reading Parquet is very efficient because the file's metadata defines the exact schema of every record. This also enables use cases like reading specific columns that would not be possible without a strict schema.
 
This conversion from schemaless to strict-schema is the difficult part of converting STAC from JSON to GeoParquet, especially for large input datasets like STAC that are often larger than memory.
"},{"location":"schema/#full-scan-over-input-data","title":"Full scan over input data","text":"
The most foolproof way to convert STAC JSON to GeoParquet is to perform a full scan over input data. This is done automatically by parse_stac_ndjson_to_arrow when a schema is not provided.
 
This is time consuming as it requires two full passes over the input data: once to infer a common schema and again to actually write to Parquet (though items are never fully held in memory, allowing this process to scale).
"},{"location":"schema/#user-provided-schema","title":"User-provided schema","text":"
Alternatively, the user can pass in an Arrow schema themselves using the schema parameter of parse_stac_ndjson_to_arrow. This schema must match the on-disk schema of the the STAC JSON data.
"},{"location":"schema/#multiple-schemas-per-collection","title":"Multiple schemas per collection","text":"
It is also possible to write multiple Parquet files with STAC data where each Parquet file may have a different schema. This simplifies the conversion and writing process but makes reading and using the Parquet data harder.
"},{"location":"schema/#merging-data-with-schema-mismatch","title":"Merging data with schema mismatch","text":"
If you've created STAC GeoParquet data where the schema has updated, you can use pyarrow.concat_tables with promote_options=\"permissive\" to combine multiple STAC GeoParquet files.
 
import pyarrow as pa\nimport pyarrow.parquet as pq\n\ntable_1 = pq.read_table(\"stac1.parquet\")\ntable_2 = pq.read_table(\"stac2.parquet\")\ncombined_table = pa.concat_tables([table1, table2], promote_options=\"permissive\")\n
"},{"location":"schema/#future-work","title":"Future work","text":"
Schema operations is an area where future work can improve reliability and ease of use of STAC GeoParquet.
 
It's possible that in the future we could automatically infer an Arrow schema from the STAC specification's published JSON Schema files. If you're interested in this, open an issue and discuss.
"},{"location":"usage/","title":"Usage","text":"
Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.
 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.
"},{"location":"usage/#dictjson-arrow-conversion","title":"dict/JSON - Arrow conversion","text":""},{"location":"usage/#convert-dicts-to-arrow","title":"Convert dicts to Arrow","text":"
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.
 
For example:
 
import pyarrow as pa\nimport pystac\n\nimport stac_geoparquet\n\nitem = pystac.read_file(\n    \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items/S2A_MSIL2A_20230112T104411_R008_T29NPE_20230113T053333\"\n)\nassert isinstance(item, pystac.Item)\n\nrecord_batch_reader = stac_geoparquet.arrow.parse_stac_items_to_arrow([item])\ntable = record_batch_reader.read_all()\n
"},{"location":"usage/#convert-json-to-arrow","title":"Convert JSON to Arrow","text":"
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.
"},{"location":"usage/#convert-arrow-to-dicts","title":"Convert Arrow to dicts","text":"
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#convert-arrow-to-json","title":"Convert Arrow to JSON","text":"
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a newline-delimited JSON file. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.
"},{"location":"usage/#parquet","title":"Parquet","text":"
Use to_parquet to write STAC Arrow data from memory to a path or file-like object. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.
 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.
 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.
"},{"location":"usage/#delta-lake","title":"Delta Lake","text":"
Use parse_stac_ndjson_to_delta_lake to read (newline-delimited) JSON on disk and write out to a Delta Lake table.
 
No special API is required for reading a STAC Delta Lake table back into Arrow. You can use the DeltaTable class directly to read the data back into Arrow.
 
Important
 
Arrow has a null data type, where every value in the column is always null, but Delta Lake does not. This means that for any column inferred to have a null data type, writing to Delta Lake will error with 
_internal.SchemaMismatchError: Invalid data type for Delta Lake: Null\n
 
This is a problem because if all items in a STAC Collection have a null JSON key, it gets inferred as an Arrow null type. For example, in the 3dep-lidar-copc collection in the tests, it has start_datetime and end_datetime fields, and so according to the spec, datetime is always null. This column would need to be casted to a timestamp type before being written to Delta Lake.
 
This means we cannot write this collection to Delta Lake solely with automatic schema inference.
 
In such cases, users may need to manually update the inferred schema to cast any null type to another Delta Lake-compatible type.
"},{"location":"api/arrow/","title":"stac_geoparquet.arrow","text":"
Arrow-based format conversions.
"},{"location":"api/arrow/#stac_geoparquet.arrow","title":"stac_geoparquet.arrow","text":""},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_JSON_CHUNK_SIZE","title":"DEFAULT_JSON_CHUNK_SIZE  module-attribute","text":"
DEFAULT_JSON_CHUNK_SIZE = 65536\n
 
The default chunk size to use for reading JSON into memory.
"},{"location":"api/arrow/#stac_geoparquet.arrow.DEFAULT_PARQUET_SCHEMA_VERSION","title":"DEFAULT_PARQUET_SCHEMA_VERSION  module-attribute","text":"
DEFAULT_PARQUET_SCHEMA_VERSION: SUPPORTED_PARQUET_SCHEMA_VERSIONS = '1.1.0'\n
 
The default GeoParquet schema version written to file.
"},{"location":"api/arrow/#stac_geoparquet.arrow.SUPPORTED_PARQUET_SCHEMA_VERSIONS","title":"SUPPORTED_PARQUET_SCHEMA_VERSIONS  module-attribute","text":"
SUPPORTED_PARQUET_SCHEMA_VERSIONS = Literal['1.0.0', '1.1.0']\n
 
A Literal type with the supported GeoParquet schema versions.
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_items_to_arrow","title":"parse_stac_items_to_arrow","text":"
parse_stac_items_to_arrow(\n    items: Iterable[Item | dict[str, Any]],\n    *,\n    chunk_size: int = 8192,\n    schema: Schema | InferredSchema | None = None\n) -> RecordBatchReader\n
 
Parse a collection of STAC Items to an iterable of pyarrow.RecordBatch.
 
The objects under properties are moved up to the top-level of the Table, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Iterable[Item | dict[str, Any]])           \u2013            
    the STAC Items to convert
     
    •  
  •  chunk_size               (int, default:                   8192 )           \u2013            
    The chunk size to use for Arrow record batches. This only takes effect if schema is not None. When schema is None, the input will be parsed into a single contiguous record batch. Defaults to 8192.
     
    •  
  •  schema               (Schema | InferredSchema | None, default:                   None )           \u2013            
    The schema of the input data. If provided, can improve memory use; otherwise all items need to be parsed into a single array for schema inference. Defaults to None.
     
    •  
 
Returns:
 
     
  •  RecordBatchReader           \u2013            
    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_arrow","title":"parse_stac_ndjson_to_arrow","text":"
parse_stac_ndjson_to_arrow(\n    path: str | Path | Iterable[str | Path],\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None\n) -> RecordBatchReader\n
 
Convert one or more newline-delimited JSON STAC files to a generator of Arrow RecordBatches.
 
Each RecordBatch in the returned iterator is guaranteed to have an identical schema, and can be used to write to one or more Parquet files.
 
Parameters:
 
     
  •  path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data.
     
    •  
 
Other Parameters:
 
     
  •  limit               (int | None)           \u2013            
    The maximum number of JSON Items to use for schema inference
     
    •  
 
Returns:
 
     
  •  RecordBatchReader           \u2013            
    pyarrow RecordBatchReader with a stream of STAC Arrow RecordBatches.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_delta_lake","title":"parse_stac_ndjson_to_delta_lake","text":"
parse_stac_ndjson_to_delta_lake(\n    input_path: str | Path | Iterable[str | Path],\n    table_or_uri: str | Path | DeltaTable,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to Delta Lake
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  table_or_uri               (str | Path | DeltaTable)           \u2013            
    A path to the output Delta Lake table
     
    •  
 
Parameters:
 
     
  •  chunk_size               (int, default:                   DEFAULT_JSON_CHUNK_SIZE )           \u2013            
    The chunk size to use for reading JSON into memory. Defaults to 65536.
     
    •  
  •  schema               (Schema | None, default:                   None )           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None, default:                   None )           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS, default:                   DEFAULT_PARQUET_SCHEMA_VERSION )           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.parse_stac_ndjson_to_parquet","title":"parse_stac_ndjson_to_parquet","text":"
parse_stac_ndjson_to_parquet(\n    input_path: str | Path | Iterable[str | Path],\n    output_path: str | Path,\n    *,\n    chunk_size: int = DEFAULT_JSON_CHUNK_SIZE,\n    schema: Schema | InferredSchema | None = None,\n    limit: int | None = None,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Convert one or more newline-delimited JSON STAC files to GeoParquet
 
Parameters:
 
     
  •  input_path               (str | Path | Iterable[str | Path])           \u2013            
    One or more paths to files with STAC items.
     
    •  
  •  output_path               (str | Path)           \u2013            
    A path to the output Parquet file.
     
    •  
 
Other Parameters:
 
     
  •  chunk_size               (int)           \u2013            
    The chunk size. Defaults to 65536.
     
    •  
  •  schema               (Schema | InferredSchema | None)           \u2013            
    The schema to represent the input STAC data. Defaults to None, in which case the schema will first be inferred via a full pass over the input data. In this case, there will be two full passes over the input data: one to infer a common schema across all data and another to read the data and iteratively convert to GeoParquet.
     
    •  
  •  limit               (int | None)           \u2013            
    The maximum number of JSON records to convert.
     
    •  
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
 
All other keyword args are passed on to pyarrow.parquet.ParquetWriter.
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_items","title":"stac_table_to_items","text":"
stac_table_to_items(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n) -> Iterable[dict]\n
 
Convert STAC Arrow to a generator of STAC Item dicts.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
 
Yields:
 
     
  •  Iterable[dict]           \u2013            
    A STAC dict for each input row.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.stac_table_to_ndjson","title":"stac_table_to_ndjson","text":"
stac_table_to_ndjson(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n    dest: str | Path | PathLike[bytes],\n) -> None\n
 
Write STAC Arrow to a newline-delimited JSON file.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
  •  dest               (str | Path | PathLike[bytes])           \u2013            
    The destination where newline-delimited JSON should be written.
     
    •  
"},{"location":"api/arrow/#stac_geoparquet.arrow.to_parquet","title":"to_parquet","text":"
to_parquet(\n    table: Table | RecordBatchReader | ArrowStreamExportable,\n    output_path: str | Path,\n    *,\n    schema_version: SUPPORTED_PARQUET_SCHEMA_VERSIONS = DEFAULT_PARQUET_SCHEMA_VERSION,\n    **kwargs: Any\n) -> None\n
 
Write an Arrow table with STAC data to GeoParquet
 
This writes metadata compliant with either GeoParquet 1.0 or 1.1.
 
Parameters:
 
     
  •  table               (Table | RecordBatchReader | ArrowStreamExportable)           \u2013            
    STAC in Arrow form. This can be a pyarrow Table, a pyarrow RecordBatchReader, or any other Arrow stream object exposed through the Arrow PyCapsule Interface. A RecordBatchReader or stream object will not be materialized in memory.
     
    •  
  •  output_path               (str | Path)           \u2013            
    The destination for saving.
     
    •  
 
Other Parameters:
 
     
  •  schema_version               (SUPPORTED_PARQUET_SCHEMA_VERSIONS)           \u2013            
    GeoParquet specification version; if not provided will default to latest supported version.
     
    •  
 
All other keyword args are passed on to pyarrow.parquet.ParquetWriter.
"},{"location":"api/legacy/","title":"Direct GeoPandas conversion (Legacy)","text":"
The API listed here was the initial non-Arrow-based STAC-GeoParquet implementation, converting between JSON and GeoPandas directly. For large collections of STAC items, using the new Arrow-based functionality (under the stac_geoparquet.arrow namespace) will be more performant.
 
Note that stac_geoparquet lifts the keys in the item properties up to the top level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
>>> import requests\n>>> import stac_geoparquet.arrow\n>>> import pyarrow.parquet\n>>> import pyarrow as pa\n\n>>> items = requests.get(\n...     \"https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items\"\n... ).json()[\"features\"]\n>>> table = pa.Table.from_batches(stac_geoparquet.arrow.parse_stac_items_to_arrow(items))\n>>> stac_geoparquet.arrow.to_parquet(table, \"items.parquet\")\n>>> table2 = pyarrow.parquet.read_table(\"items.parquet\")\n>>> items2 = list(stac_geoparquet.arrow.stac_table_to_items(table2))\n
"},{"location":"api/legacy/#stac_geoparquet.to_geodataframe","title":"stac_geoparquet.to_geodataframe","text":"
to_geodataframe(\n    items: Sequence[dict[str, Any]],\n    add_self_link: bool = False,\n    dtype_backend: DTYPE_BACKEND | None = None,\n    datetime_precision: str = \"ns\",\n) -> GeoDataFrame\n
 
Convert a sequence of STAC items to a geopandas.GeoDataFrame.
 
The objects under properties are moved up to the top-level of the DataFrame, similar to geopandas.GeoDataFrame.from_features.
 
Parameters:
 
     
  •  items               (Sequence[dict[str, Any]])           \u2013            
    A sequence of STAC items.
     
    •  
  •  add_self_link               (bool, default:                   False )           \u2013            
    bool, default False Add the absolute link (if available) to the source STAC Item as a separate column named \"self_link\"
     
    •  
  •  dtype_backend               (DTYPE_BACKEND | None, default:                   None )           \u2013            
    {'pyarrow', 'numpy_nullable'}, optional The dtype backend to use for storing arrays.
     
    By default, this will use 'numpy_nullable' and emit a FutureWarning that the default will change to 'pyarrow' in the next release.
     
    Set to 'numpy_nullable' to silence the warning and accept the old behavior.
     
    Set to 'pyarrow' to silence the warning and accept the new behavior.
     
    There are some difference in the output as well: with dtype_backend=\"pyarrow\", struct-like fields will explicitly contain null values for fields that appear in only some of the records. For example, given an assets like::
     
    {\n    \"a\": {\n        \"href\": \"a.tif\",\n    },\n    \"b\": {\n        \"href\": \"b.tif\",\n        \"title\": \"B\",\n    }\n}\n
     
    The assets field of the output for the first row with dtype_backend=\"numpy_nullable\" will be a Python dictionary with just {\"href\": \"a.tiff\"}.
     
    With dtype_backend=\"pyarrow\", this will be a pyarrow struct with fields {\"href\": \"a.tif\", \"title\", None}. pyarrow will infer that the struct field asset.title is nullable.
     
    •  
  •  datetime_precision               (str, default:                   'ns' )           \u2013            
    str, default \"ns\" The precision to use for the datetime columns. For example, \"us\" is microsecond and \"ns\" is nanosecond.
     
    •  
 
Returns:
 
     
  •  GeoDataFrame           \u2013            
    The converted GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_item_collection","title":"stac_geoparquet.to_item_collection","text":"
to_item_collection(df: GeoDataFrame) -> ItemCollection\n
 
Convert a GeoDataFrame of STAC items to a pystac.ItemCollection.
 
Parameters:
 
     
  •  df               (GeoDataFrame)           \u2013            
    A GeoDataFrame with a schema similar to that exported by stac-geoparquet.
     
    •  
 
Returns:
 
     
  •  ItemCollection           \u2013            
    The converted ItemCollection. There will be one record / feature per row in the in the GeoDataFrame.
     
    •  
"},{"location":"api/legacy/#stac_geoparquet.to_dict","title":"stac_geoparquet.to_dict","text":"
to_dict(record: dict) -> dict\n
 
Create a dictionary representing a STAC item from a row of the GeoDataFrame.
 
Parameters:
 
     
  •  record               (dict)           \u2013            
    dict
     
    •  
"},{"location":"api/pgstac/","title":"pgstac integration","text":"
stac_geoparquet.pgstac_reader has some helpers for working with items coming from a pgstac.items table. It takes care of
 
     
  • Rehydrating the dehydrated items
    •  
  • Partitioning by time
    •  
  • Injecting dynamic links and assets from a STAC API
    •  
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig","title":"stac_geoparquet.pgstac_reader.CollectionConfig  dataclass","text":"
Additional collection-based configuration to inject, matching the dynamic properties from the API.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection","title":"collection  property","text":"
collection: Collection\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.collection_id","title":"collection_id  instance-attribute","text":"
collection_id: str\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.partition_frequency","title":"partition_frequency  class-attribute instance-attribute","text":"
partition_frequency: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.render_config","title":"render_config  class-attribute instance-attribute","text":"
render_config: str | None = None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.should_inject_dynamic_properties","title":"should_inject_dynamic_properties  class-attribute instance-attribute","text":"
should_inject_dynamic_properties: bool = True\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.stac_api","title":"stac_api  class-attribute instance-attribute","text":"
stac_api: str = 'https://planetarycomputer.microsoft.com/api/stac/v1'\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__init__","title":"__init__","text":"
__init__(\n    collection_id: str,\n    partition_frequency: str | None = None,\n    stac_api: str = \"https://planetarycomputer.microsoft.com/api/stac/v1\",\n    should_inject_dynamic_properties: bool = True,\n    render_config: str | None = None,\n) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.__post_init__","title":"__post_init__","text":"
__post_init__() -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_collection","title":"export_collection","text":"
export_collection(\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> list[str | None]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition","title":"export_partition","text":"
export_partition(\n    conninfo: str,\n    query: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any] | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.export_partition_for_endpoints","title":"export_partition_for_endpoints","text":"
export_partition_for_endpoints(\n    endpoints: tuple[datetime, datetime],\n    conninfo: str,\n    output_protocol: str,\n    output_path: str,\n    storage_options: dict[str, Any],\n    part_number: int | None = None,\n    total: int | None = None,\n    rewrite: bool = False,\n    skip_empty_partitions: bool = False,\n) -> str | None\n
 
Export results for a pair of endpoints.
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.generate_endpoints","title":"generate_endpoints","text":"
generate_endpoints(\n    since: datetime | None = None,\n) -> list[tuple[datetime, datetime]]\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_assets","title":"inject_assets","text":"
inject_assets(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.inject_links","title":"inject_links","text":"
inject_links(item: dict[str, Any]) -> None\n
"},{"location":"api/pgstac/#stac_geoparquet.pgstac_reader.CollectionConfig.make_pgstac_items","title":"make_pgstac_items","text":"
make_pgstac_items(\n    records: list[tuple[str, str, str, datetime, datetime, dict[str, Any]]],\n    base_item: dict[str, Any],\n) -> list[dict[str, Any]]\n
 
Make STAC items out of pgstac records.
 
Parameters:
 
     
  •  records               (list[tuple[str, str, str, datetime, datetime, dict[str, Any]]])           \u2013            
    list[tuple] The dehydrated records from pgstac.items table.
     
    •  
  •  base_item               (dict[str, Any])           \u2013            
    dict[str, Any] The base item from the collection_base_item pgstac function for this collection. Used for rehydration
     
    •  
"},{"location":"spec/stac-geoparquet-spec/","title":"STAC GeoParquet Specification","text":""},{"location":"spec/stac-geoparquet-spec/#overview","title":"Overview","text":"
This document specifies how to map a set of STAC Items into GeoParquet. It is directly inspired by the STAC GeoParquet library, but aims to provide guidance for anyone putting STAC data into GeoParquet.
"},{"location":"spec/stac-geoparquet-spec/#use-cases","title":"Use cases","text":"
     
  • Provide a STAC GeoParquet that mirrors a static Collection as a way to query the whole dataset instead of reading every specific GeoJSON file.
    •  
  • As an output format for STAC API responses that is more efficient than paging through thousands of pages of GeoJSON.
    •  
  • Provide efficient access to specific fields of a STAC item, thanks to Parquet's columnar format.
    •  
"},{"location":"spec/stac-geoparquet-spec/#guidelines","title":"Guidelines","text":"
Each row in the Parquet Dataset represents a single STAC item. Most all the fields in a STAC Item should be mapped to a column in GeoParquet. We embrace Parquet structures where possible, mapping from JSON into nested structures. We do pull the properties to the top level, so that it is easier to query and use them. The names of most of the fields should be the same in STAC and in GeoParquet.
 Field GeoParquet Type Required Details type String Optional This is just needed for GeoJSON, so it is optional and not recommended to include in GeoParquet stac_extensions List of Strings Required This column is required, but can be empty if no STAC extensions were used id String Required Required, should be unique within each collection geometry Binary (WKB) Required For GeoParquet 1.0 this must be well-known Binary bbox Struct of Floats Required Can be a 4 or 6 value struct, depending on dimension of the data. It must conform to the \"Bounding Box Columns\" definition of GeoParquet 1.1. links List of Link structs Required See Link Struct for more info assets An Assets struct Required See Asset Struct for more info collection String Optional The ID of the collection this Item is a part of. See notes below on 'Collection' and 'Collection JSON' in the Parquet metadata property columns varies - Each property should use the relevant Parquet type, and be pulled out of the properties object to be a top-level Parquet field 
     
  • Must be valid GeoParquet, with proper metadata. Ideally the geometry types are defined and as narrow as possible.
    •  
  • Strongly recommend to only have one GeoParquet per STAC 'Collection'. Not doing this will lead to an expanded GeoParquet schema (the union of all the schemas of the collection) with lots of empty data
    •  
  • Any field in 'properties' of the STAC item should be moved up to be a top-level field in the GeoParquet.
    •  
  • STAC GeoParquet does not support properties that are named such that they collide with a top-level key.
    •  
  • datetime columns should be stored as a native timestamp, not as a string
    •  
  • The Collection JSON should be included in the Parquet metadata. See Collection JSON below.
    •  
  • Any other properties that would be stored as GeoJSON in a STAC JSON Item (e.g. proj:geometry) should be stored as a binary column with WKB encoding. This simplifies the handling of collections with multiple geometry types.
    •  
"},{"location":"spec/stac-geoparquet-spec/#link-struct","title":"Link Struct","text":"
The GeoParquet dataset can contain zero or more Link Structs. Each Link Struct has 2 required fields and 2 optional ones:
 Field Name Type Description href string REQUIRED. The actual link in the format of an URL. Relative and absolute links are both allowed. rel string REQUIRED. Relationship between the current document and the linked document. See chapter \"Relation types\" for more information. type string Media type of the referenced entity. title string A human readable title to be used in rendered displays of the link. 
See Link Object for more.
"},{"location":"spec/stac-geoparquet-spec/#asset-struct","title":"Asset Struct","text":"
The GeoParquet dataset can contain zero or more Asset Structs. Each Asset Struct can have the following fields:
 Field Name Type Description href string REQUIRED. URI to the asset object. Relative and absolute URI are both allowed. title string The displayed title for clients and users. description string A description of the Asset providing additional details, such as how it was processed or created. CommonMark 0.29 syntax MAY be used for rich text representation. type string Media type of the asset. See the common media types in the best practice doc for commonly used asset types. roles [string] The semantic roles of the asset, similar to the use of rel in links. 
Each struct has each full asset key and object as a sub-struct, it's a direct mapping from the JSON to Parquet
 
To take advantage of Parquet's columnar nature and compression, the assets should be uniform so they can be represented by a simple schema, which in turn means every item should probably come from the same STAC collection.
 
See Asset Object for more.
"},{"location":"spec/stac-geoparquet-spec/#including-a-stac-collection-json-in-a-stac-geoparquet-collection","title":"Including a STAC Collection JSON in a STAC Geoparquet Collection","text":"
To make a stac-geoparquet file a fully self-contained representation, you can include the Collection JSON in the Parquet metadata. If present in the Parquet file metadata, the key must be stac:collection and the value must be a JSON string with the Collection JSON.
"},{"location":"spec/stac-geoparquet-spec/#referencing-a-stac-geoparquet-collections-in-a-stac-collection-json","title":"Referencing a STAC Geoparquet Collections in a STAC Collection JSON","text":"
A common use case of stac-geoparquet is to create a mirror of a STAC collection. To refer to this mirror in the original collection, use an Asset Object at the collection level of the STAC JSON that includes the application/vnd.apache.parquet Media type and collection-mirror Role type to describe the function of the Geoparquet STAC Collection Asset.
 
For example:
 Field Name Type Value href string s3://example/uri/to/file.parquet title string An example STAC GeoParquet. description string Example description. type string application/vnd.apache.parquet roles [string] [collection-mirror]* 
*Note the IANA has not approved the new Media type application/vnd.apache.parquet yet, it's been submitted for approval.
 
The description should ideally include details about the spatial partitioning method.
"},{"location":"spec/stac-geoparquet-spec/#mapping-to-other-geospatial-data-formats","title":"Mapping to other geospatial data formats","text":"
The principles here can likely be used to map into other geospatial data formats (GeoPackage, FlatGeobuf, etc), but we embrace Parquet's nested 'structs' for some of the mappings, so other formats will need to do something different. The obvious thing to do is to dump JSON into those fields, but that's outside the scope of this document, and we recommend creating a general document for that.
"}]}
\ No newline at end of file
diff --git a/latest/sitemap.xml b/latest/sitemap.xml
index c7f9c19..ddbc218 100644
--- a/latest/sitemap.xml
+++ b/latest/sitemap.xml
@@ -2,42 +2,42 @@
 
     
          https://stac-utils.github.io/stac-geoparquet/latest/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/drawbacks/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/schema/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/usage/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/arrow/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/legacy/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/api/pgstac/
-         2024-06-24
+         2024-06-25
          daily
     
     
          https://stac-utils.github.io/stac-geoparquet/latest/spec/stac-geoparquet-spec/
-         2024-06-24
+         2024-06-25
          daily
     
 
\ No newline at end of file
diff --git a/latest/sitemap.xml.gz b/latest/sitemap.xml.gz
index ff0ffba75439d5d6bfa656bfc6c1d004f0296c58..207add5b981efa0ac6084b502afcbb3a5081310a 100644
GIT binary patch
literal 278
zcmV+x0qOo9iwFpS2zq7$|8r?{Wo=<_E_iKh0M(SiPQ)M(hVT0n4fjHA)Tn9Mn@^w*
zFiS^DmKK(w)_r?vyD^PtJunyGpPxS;A=fGA0H909${5XS#VC=0CJ7psjKZt(?+Xa{}pgt%s?ClowVx9;z=7kL22nfN=`C(hspl
cdyO8|?_Rlz|9*^hNZ-Ev1}tNkza|C%09hZ0G5`Po

diff --git a/latest/usage/index.html b/latest/usage/index.html
index d1a9546..2b0a3f1 100644
--- a/latest/usage/index.html
+++ b/latest/usage/index.html
@@ -733,19 +733,33 @@
 
 
 
Usage

-
Except for the legacy API, Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.

+
Apache Arrow is used as the in-memory interchange format between all formats. While some end-to-end helper functions are provided, the user can go through Arrow objects for maximal flexibility in the conversion process.

 
All functionality that goes through Arrow is currently exported via the stac_geoparquet.arrow namespace.

 
dict/JSON - Arrow conversion

 
Convert dicts to Arrow

 
Use parse_stac_items_to_arrow to convert STAC items either in memory or on disk to a stream of Arrow record batches. This accepts either an iterable of Python dicts or an iterable of pystac.Item objects.

+
For example:

+
import pyarrow as pa
+import pystac
+
+import stac_geoparquet
+
+item = pystac.read_file(
+    "https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a/items/S2A_MSIL2A_20230112T104411_R008_T29NPE_20230113T053333"
+)
+assert isinstance(item, pystac.Item)
+
+record_batch_reader = stac_geoparquet.arrow.parse_stac_items_to_arrow([item])
+table = record_batch_reader.read_all()
+

 
Convert JSON to Arrow

 
parse_stac_ndjson_to_arrow is a helper function to take one or more JSON or newline-delimited JSON files on disk, infer the schema from all of them, and convert the data to a stream of Arrow record batches.

 
Convert Arrow to dicts

 
Use stac_table_to_items to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

 
Convert Arrow to JSON

-
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a generator of Python dicts. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

+
Use stac_table_to_ndjson to convert a table or stream of Arrow record batches of STAC data to a newline-delimited JSON file. This accepts either a pyarrow.Table or a pyarrow.RecordBatchReader, which allows conversions of larger-than-memory files in a streaming manner.

 
Parquet

-
Use to_parquet to write STAC Arrow data in memory. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.

+
Use to_parquet to write STAC Arrow data from memory to a path or file-like object. This is a special function to ensure that GeoParquet 1.0 or 1.1 metadata is written to the Parquet file.

 
parse_stac_ndjson_to_parquet is a helper that connects reading (newline-delimited) JSON on disk to writing out to a Parquet file.

 
No special API is required for reading a STAC GeoParquet file back into Arrow. You can use pyarrow.parquet.read_table or pyarrow.parquet.ParquetFile directly to read the STAC GeoParquet data back into Arrow.

 
Delta Lake