diff --git a/.gitignore b/.gitignore index 02e651c..1368305 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ dist stac_geoparquet/_version.py .cache site +docs/**/*.jsonl +docs/**/*table +docs/**/*.parquet +.ipynb_checkpoints diff --git a/docs/examples/naip.ipynb b/docs/examples/naip.ipynb new file mode 100644 index 0000000..e452da3 --- /dev/null +++ b/docs/examples/naip.ipynb @@ -0,0 +1,455 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NAIP example\n", + "\n", + "We'll use STAC Items from the [NAIP STAC Collection](https://planetarycomputer.microsoft.com/dataset/naip) on Microsoft's Planetary Computer to illustrate how to use the `stac-geoparquet` library.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are a few libraries we need to install to run this notebook:\n", + "\n", + "```\n", + "pip install planetary-computer pystac-client stac-geoparquet pyarrow\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "\n", + "import planetary_computer\n", + "import pyarrow as pa\n", + "import pyarrow.parquet as pq\n", + "import pystac_client\n", + "import deltalake\n", + "import stac_geoparquet" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can open the Planetary Computer STAC Collection with `pystac_client.Client.open`, ensuring we also sign the returned URLs in each STAC Item.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "catalog = pystac_client.Client.open(\n", + " \"https://planetarycomputer.microsoft.com/api/stac/v1\",\n", + " modifier=planetary_computer.sign_inplace,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we'll access the NAIP collection from the Planetary Computer catalog and download 1000 items from this collection, writing them to a newline-delimited JSON file in the current directory.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "items_iter = catalog.get_collection(\"naip\").get_items()\n", + "\n", + "max_items = 1000\n", + "naip_json_path = Path(\"naip.jsonl\")\n", + "if not naip_json_path.exists():\n", + " with open(naip_json_path, \"w\") as f:\n", + " count = 0\n", + "\n", + " for item in items_iter:\n", + " json.dump(item.to_dict(), f, separators=(\",\", \":\"))\n", + " f.write(\"\\n\")\n", + "\n", + " count += 1\n", + " if count >= max_items:\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now use `stac-geoparquet` APIs on this data.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Loading to Arrow\n", + "\n", + "We can load to an Arrow `RecordBatchReader` by using the `parse_stac_ndjson_to_arrow` function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "record_batch_reader = stac_geoparquet.arrow.parse_stac_ndjson_to_arrow(naip_json_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Arrow `RecordBatchReader` represents a _stream_ of Arrow batches, which can be useful when converting a very large STAC collection, which you don't want to materialize in memory at once.\n", + "\n", + "We can convert this to an Arrow table with `read_all`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "assets: struct>, href: string, roles: list, title: string, type: string>, rendered_preview: struct, title: string, type: string>, thumbnail: struct, title: string, type: string>, tilejson: struct, title: string, type: string>>\n", + " child 0, image: struct>, href: string, roles: list, title: string, type: string>\n", + " child 0, eo:bands: list>\n", + " child 0, item: struct\n", + " child 0, common_name: string\n", + " child 1, description: string\n", + " child 2, name: string\n", + " child 1, href: string\n", + " child 2, roles: list\n", + " child 0, item: string\n", + " child 3, title: string\n", + " child 4, type: string\n", + " child 1, rendered_preview: struct, title: string, type: string>\n", + " child 0, href: string\n", + " child 1, rel: string\n", + " child 2, roles: list\n", + " child 0, item: string\n", + " child 3, title: string\n", + " child 4, type: string\n", + " child 2, thumbnail: struct, title: string, type: string>\n", + " child 0, href: string\n", + " child 1, roles: list\n", + " child 0, item: string\n", + " child 2, title: string\n", + " child 3, type: string\n", + " child 3, tilejson: struct, title: string, type: string>\n", + " child 0, href: string\n", + " child 1, roles: list\n", + " child 0, item: string\n", + " child 2, title: string\n", + " child 3, type: string\n", + "bbox: struct\n", + " child 0, xmin: double\n", + " child 1, ymin: double\n", + " child 2, xmax: double\n", + " child 3, ymax: double\n", + "collection: string\n", + "geometry: binary\n", + " -- field metadata --\n", + " ARROW:extension:name: 'geoarrow.wkb'\n", + " ARROW:extension:metadata: '{\"crs\":{\"$schema\":\"https://proj.org/schemas/' + 1296\n", + "id: string\n", + "links: list>\n", + " child 0, item: struct\n", + " child 0, href: string\n", + " child 1, rel: string\n", + " child 2, title: string\n", + " child 3, type: string\n", + "stac_extensions: list\n", + " child 0, item: string\n", + "stac_version: string\n", + "type: string\n", + "datetime: timestamp[us, tz=UTC]\n", + "gsd: double\n", + "naip:state: string\n", + "naip:year: string\n", + "proj:bbox: list\n", + " child 0, item: double\n", + "proj:centroid: struct\n", + " child 0, lat: double\n", + " child 1, lon: double\n", + "proj:epsg: int64\n", + "proj:shape: list\n", + " child 0, item: int64\n", + "proj:transform: list\n", + " child 0, item: double\n", + "providers: list, url: string>>\n", + " child 0, item: struct, url: string>\n", + " child 0, name: string\n", + " child 1, roles: list\n", + " child 0, item: string\n", + " child 2, url: string" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table = record_batch_reader.read_all()\n", + "table.schema" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also pass a small chunk size into `parse_stac_ndjson_to_arrow` to show how the streaming works.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "record_batch_reader = stac_geoparquet.arrow.parse_stac_ndjson_to_arrow(\n", + " naip_json_path, chunk_size=100\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`record_batch_reader` is an iterator that yields Arrow `RecordBatch` objects. If we load just the first one, we'll see that it contains 100 rows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "100" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "first_batch = next(record_batch_reader)\n", + "first_batch.num_rows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Materializing the rest of the batches from the iterator into a table gives us the other 900 rows in the dataset.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "900" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "other_batches = record_batch_reader.read_all()\n", + "other_batches.num_rows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All batches from the RecordBatchReader have the same schema, so we can concatenate them back into a single table:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "combined_table = pa.concat_tables([pa.Table.from_batches([first_batch]), other_batches])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Both the original `table` object and this `combined_table` object have the exact same data:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table == combined_table" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Converting to Parquet\n", + "\n", + "We can use the utility function `parse_stac_ndjson_to_parquet` to convert the items directly to GeoParquet.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "naip_parquet_path = \"naip.parquet\"\n", + "stac_geoparquet.arrow.parse_stac_ndjson_to_parquet(naip_json_path, naip_parquet_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Reading that Parquet data back into Arrow with `pyarrow.parquet.read_table` gives us the exact same Arrow data as before.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pq.read_table(naip_parquet_path) == table" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Converting to Delta Lake\n", + "\n", + "We can use the utility function `parse_stac_ndjson_to_delta_lake` to convert items directly to Delta Lake.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "naip_delta_lake_path = \"naip_table\"\n", + "stac_geoparquet.arrow.parse_stac_ndjson_to_delta_lake(\n", + " naip_json_path, naip_delta_lake_path, mode=\"overwrite\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Reading the Delta Lake table back into Arrow with `deltalake.DeltaTable` gives us the exact same Arrow data as before.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "deltalake.DeltaTable(naip_delta_lake_path).to_pyarrow_table() == table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "stac-geoparquet", + "language": "python", + "name": "stac-geoparquet" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/mkdocs.yml b/mkdocs.yml index 0d253da..d0250b6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,6 +18,8 @@ nav: - index.md - usage.md - schema.md + - Examples: + - examples/naip.ipynb - Specification: spec/stac-geoparquet-spec.md - API Reference: - api/arrow.md @@ -74,6 +76,9 @@ plugins: - mike: alias_type: "copy" canonical_version: "latest" + - mkdocs-jupyter: + include_source: true + ignore: ["**/.ipynb_checkpoints/*.ipynb"] - mkdocstrings: enable_inventory: true handlers: diff --git a/pyproject.toml b/pyproject.toml index 99ee7d5..c759a16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ docs = [ "black", "griffe-inherited-docstrings", "mike>=2", + "mkdocs-jupyter", "mkdocs-material[imaging]>=9.5", "mkdocs", "mkdocstrings[python]>=0.25.1",