From b00de2b1f2773112ef07610239761c35633c02a3 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Tue, 3 Oct 2023 09:46:52 +0200 Subject: [PATCH 01/24] geos: use v3.11 fix deps --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b6a7ce2..fb2a921 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ path-slash = "0.1" [features] extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] -static = [ "geos/static" ] +static = [ "geos/static", "geos/v3_11_0" ] nightly = [] default = [] From e9fa78adf755a756c49978ffe1ec9eb07fa3891d Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Tue, 3 Oct 2023 09:52:17 +0200 Subject: [PATCH 02/24] ci: global static --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d741c7b..a1954a7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,8 +28,8 @@ jobs: run: | pip install numpy - - run: cargo build --features geos/static --verbose - - run: cargo test --features geos/static --verbose + - run: cargo build --features static --verbose + - run: cargo test --features static --verbose nightly: runs-on: ubuntu-latest From 3953fa94674d8a24ad33729241f653ae2f11a777 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Fri, 26 Jan 2024 09:54:32 +0100 Subject: [PATCH 03/24] update deps --- Cargo.toml | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fb2a921..4abf663 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,8 +20,8 @@ path = "src/devel/make_bitmap.rs" [dependencies] geos = { version = "8" } lazy_static = "1.4" -numpy = { version = "0.19" } -pyo3 = { version = "0.19" , features = [ "abi3-py38" ] } +numpy = { version = "0.20" } +pyo3 = { version = "0.20" , features = [ "abi3-py39" ] } roaring = "0.10" rust-embed = "8" xz2 = "0.1" @@ -32,8 +32,8 @@ rayon = "1" [build-dependencies] reqwest = { version = "0.11", default-features = false, features = [ "blocking", "rustls-tls" ] } -ring = "0.16" -path-slash = "0.1" +ring = "0.17" +path-slash = "0.2" [features] extension-module = ["pyo3/extension-module"] diff --git a/pyproject.toml b/pyproject.toml index 18f4082..b1f8b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1.4"] build-backend = "maturin" [tool.maturin] From f77429f175f78fb480839c42ab52d7709a260055 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Tue, 14 May 2024 15:19:26 +0200 Subject: [PATCH 04/24] update deps --- Cargo.toml | 8 ++++---- src/shapes.rs | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4abf663..601d57e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,10 +18,10 @@ name = "make_bitmap" path = "src/devel/make_bitmap.rs" [dependencies] -geos = { version = "8" } +geos = { version = "9" } lazy_static = "1.4" -numpy = { version = "0.20" } -pyo3 = { version = "0.20" , features = [ "abi3-py39" ] } +numpy = { version = "0.21" } +pyo3 = { version = "0.21" , features = [ "abi3-py311" ] } roaring = "0.10" rust-embed = "8" xz2 = "0.1" @@ -31,7 +31,7 @@ ndarray = { version = "0.15", features = [ "rayon" ] } rayon = "1" [build-dependencies] -reqwest = { version = "0.11", default-features = false, features = [ "blocking", "rustls-tls" ] } +reqwest = { version = "0.12", default-features = false, features = [ "blocking", "rustls-tls" ] } ring = "0.17" path-slash = "0.2" diff --git a/src/shapes.rs b/src/shapes.rs index 4917277..a2a6190 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -11,10 +11,10 @@ pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N #[pyclass] pub struct Gshhg { - geom: *mut Geometry<'static>, + geom: *mut Geometry, // prepped requires `geom` above to be around, and is valid as long as geom is alive. - prepped: PreparedGeometry<'static>, + prepped: PreparedGeometry, } impl Drop for Gshhg { @@ -29,7 +29,7 @@ unsafe impl Sync for Gshhg {} // `PreparededGeometry::contains` needs a call to `contains` before it is thread-safe: // https://github.com/georust/geos/issues/95 -fn warmup_prepped(prepped: &PreparedGeometry<'_>) { +fn warmup_prepped(prepped: &PreparedGeometry) { let point = CoordSeq::new_from_vec(&[&[0.0, 0.0]]).unwrap(); let point = Geometry::create_point(point).unwrap(); prepped.contains(&point).unwrap(); @@ -50,7 +50,7 @@ impl Clone for Gshhg { } impl Gshhg { - pub fn from_geom(geom: Geometry<'static>) -> io::Result { + pub fn from_geom(geom: Geometry) -> io::Result { let bxd = Box::new(geom); let gptr = Box::into_raw(bxd); let prepped = unsafe { (&*gptr).to_prepared_geom() } @@ -69,7 +69,7 @@ impl Gshhg { Gshhg::from_geom(g) } - pub fn get_geometry_from_compressed>(path: P) -> io::Result> { + pub fn get_geometry_from_compressed>(path: P) -> io::Result { let fd = File::open(path)?; let fd = io::BufReader::new(fd); let mut fd = xz2::bufread::XzDecoder::new(fd); From 7872c9f8d3e861e2301e7997fa9389a3c5ba5650 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Tue, 14 May 2024 15:38:40 +0200 Subject: [PATCH 05/24] py 3.11 --- .github/workflows/package.yml | 8 ++++---- .github/workflows/rust.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index a717cf1..08ba4a6 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Install Rust toolchain uses: actions-rs/toolchain@v1 @@ -56,7 +56,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: ${{ matrix.target }} - name: Install Rust toolchain uses: actions-rs/toolchain@v1 @@ -89,7 +89,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Build Wheels uses: messense/maturin-action@v1 @@ -162,7 +162,7 @@ jobs: name: wheels - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - name: Publish to PyPi env: TWINE_USERNAME: __token__ diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a1954a7..b8c456e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Install deps @@ -45,7 +45,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Install deps From 5554415f861b1bf09655f9bf9f286418090d597d Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 08:57:16 +0200 Subject: [PATCH 06/24] abi 3.10, default simd --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 601d57e..5d0a98b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ path = "src/devel/make_bitmap.rs" geos = { version = "9" } lazy_static = "1.4" numpy = { version = "0.21" } -pyo3 = { version = "0.21" , features = [ "abi3-py311" ] } +pyo3 = { version = "0.21" , features = [ "abi3-py310" ] } roaring = "0.10" rust-embed = "8" xz2 = "0.1" @@ -40,7 +40,7 @@ extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] static = [ "geos/static", "geos/v3_11_0" ] nightly = [] -default = [] +default = [ "simd" ] [profile.release] debug = true From d92b2d6b57dc8a677bf2a32ac71966734250c1d6 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 18:14:40 +0200 Subject: [PATCH 07/24] convert to geo types --- Cargo.toml | 4 +-- src/shapes.rs | 69 ++++++++++----------------------------------------- 2 files changed, 15 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5d0a98b..9ec0208 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ name = "make_bitmap" path = "src/devel/make_bitmap.rs" [dependencies] -geos = { version = "9" } +geo = "0.28" lazy_static = "1.4" numpy = { version = "0.21" } pyo3 = { version = "0.21" , features = [ "abi3-py310" ] } @@ -26,6 +26,7 @@ roaring = "0.10" rust-embed = "8" xz2 = "0.1" ndarray = { version = "0.15", features = [ "rayon" ] } +shapefile = { version = "0.6.0", features = ["geo-types"] } [dev-dependencies] rayon = "1" @@ -38,7 +39,6 @@ path-slash = "0.2" [features] extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] -static = [ "geos/static", "geos/v3_11_0" ] nightly = [] default = [ "simd" ] diff --git a/src/shapes.rs b/src/shapes.rs index a2a6190..d2ff232 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,65 +1,24 @@ use pyo3::{prelude::*, types::PyBytes}; use std::borrow::Borrow; use std::fs::File; -use std::io::{self, prelude::*}; +use std::io::{self, prelude::*, Cursor}; use std::path::Path; -use geos::{CoordSeq, Geom, Geometry, PreparedGeometry}; +use geo::{Geometry, Contains, point}; use numpy::{PyArray, PyReadonlyArrayDyn}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; #[pyclass] +#[derive(Clone)] pub struct Gshhg { - geom: *mut Geometry, - - // prepped requires `geom` above to be around, and is valid as long as geom is alive. - prepped: PreparedGeometry, -} - -impl Drop for Gshhg { - fn drop(&mut self) { - unsafe { drop(Box::from_raw(self.geom)) } - } -} - -// PreparedGeometry is Send+Sync, Geometry is Send+Sync. *mut Geometry is never modified. -unsafe impl Send for Gshhg {} -unsafe impl Sync for Gshhg {} - -// `PreparededGeometry::contains` needs a call to `contains` before it is thread-safe: -// https://github.com/georust/geos/issues/95 -fn warmup_prepped(prepped: &PreparedGeometry) { - let point = CoordSeq::new_from_vec(&[&[0.0, 0.0]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); -} - -impl Clone for Gshhg { - fn clone(&self) -> Self { - let gptr = self.geom.clone(); - debug_assert!(gptr != self.geom); - let prepped = unsafe { (&*gptr).to_prepared_geom().unwrap() }; - warmup_prepped(&prepped); - - Gshhg { - geom: gptr, - prepped, - } - } + geom: Geometry, } impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { - let bxd = Box::new(geom); - let gptr = Box::into_raw(bxd); - let prepped = unsafe { (&*gptr).to_prepared_geom() } - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "cannot prepare geomtry"))?; - warmup_prepped(&prepped); - Ok(Gshhg { - geom: gptr, - prepped, + geom }) } @@ -75,8 +34,9 @@ impl Gshhg { let mut fd = xz2::bufread::XzDecoder::new(fd); let mut buf = Vec::new(); fd.read_to_end(&mut buf)?; - - Ok(geos::Geometry::new_from_wkb(&buf).unwrap()) + let mut buf = Cursor::new(buf); + let shape = shapefile::ShapeReader::new(buf).unwrap(); + shape.read_as() } } @@ -85,8 +45,7 @@ impl Gshhg { /// Make a new Gshhg shapes instance. #[staticmethod] pub fn new(py: Python) -> io::Result { - let buf = Gshhg::wkb(py)?; - let g = geos::Geometry::new_from_wkb(buf.as_bytes()).unwrap(); + let g = Gshhg::get_geometry_from_compressed(&GSHHS_F)?; Gshhg::from_geom(g) } @@ -117,16 +76,14 @@ impl Gshhg { debug_assert!(x >= -180. && x <= 180.); assert!(y > -90. && y <= 90.); - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - self.prepped.contains(&point).unwrap() + let p = point!(x: x, y: y); + self.geom.contains(&p) } /// Same as `contains`, but does not check for bounds. pub(crate) fn contains_unchecked(&self, x: f64, y: f64) -> bool { - let point = CoordSeq::new_from_vec(&[&[x, y]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - self.prepped.contains(&point).unwrap() + let p = point!(x: x, y: y); + self.geom.contains(&p) } pub fn contains_many( From d0e190eb100fcdb8bc26b9b1751a9a34414e3c9f Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 20:39:24 +0200 Subject: [PATCH 08/24] load wkb and use geo types --- Cargo.toml | 2 +- src/shapes.rs | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9ec0208..efa8972 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ roaring = "0.10" rust-embed = "8" xz2 = "0.1" ndarray = { version = "0.15", features = [ "rayon" ] } -shapefile = { version = "0.6.0", features = ["geo-types"] } +wkb = "0.7.1" [dev-dependencies] rayon = "1" diff --git a/src/shapes.rs b/src/shapes.rs index d2ff232..ab3fe85 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,10 +1,11 @@ use pyo3::{prelude::*, types::PyBytes}; use std::borrow::Borrow; +use std::convert::TryFrom; use std::fs::File; use std::io::{self, prelude::*, Cursor}; use std::path::Path; -use geo::{Geometry, Contains, point}; +use geo::{point, Contains, Geometry}; use numpy::{PyArray, PyReadonlyArrayDyn}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; @@ -17,9 +18,7 @@ pub struct Gshhg { impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { - Ok(Gshhg { - geom - }) + Ok(Gshhg { geom }) } pub fn from_compressed>(path: P) -> io::Result { @@ -32,11 +31,8 @@ impl Gshhg { let fd = File::open(path)?; let fd = io::BufReader::new(fd); let mut fd = xz2::bufread::XzDecoder::new(fd); - let mut buf = Vec::new(); - fd.read_to_end(&mut buf)?; - let mut buf = Cursor::new(buf); - let shape = shapefile::ShapeReader::new(buf).unwrap(); - shape.read_as() + let geom = wkb::wkb_to_geom(&mut fd).unwrap(); + Ok(geom) } } @@ -45,8 +41,14 @@ impl Gshhg { /// Make a new Gshhg shapes instance. #[staticmethod] pub fn new(py: Python) -> io::Result { - let g = Gshhg::get_geometry_from_compressed(&GSHHS_F)?; - Gshhg::from_geom(g) + use crate::GsshgData; + + let buf = GsshgData::get(&GSHHS_F) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "cannot find shapes"))?; + let buf: &[u8] = buf.data.borrow(); + let mut fd = xz2::read::XzDecoder::new(buf); + let geom = wkb::wkb_to_geom(&mut fd).unwrap(); + Gshhg::from_geom(geom) } /// Get the WKB for the GSHHG shapes (full resolution). From 7b208126bbb5a413abe634434c331a5011eb93c4 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 20:39:34 +0200 Subject: [PATCH 09/24] tests: remove geos test --- tests/test_geos_par_prepped.rs | 54 ---------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 tests/test_geos_par_prepped.rs diff --git a/tests/test_geos_par_prepped.rs b/tests/test_geos_par_prepped.rs deleted file mode 100644 index 4460fa8..0000000 --- a/tests/test_geos_par_prepped.rs +++ /dev/null @@ -1,54 +0,0 @@ -use roaring_landmask::Gshhg; -use geos::{CoordSeq, Geom, Geometry}; - -#[ignore] -#[test] -fn test_par_prepped_no_warmup() { - use rayon::prelude::*; - - // let s = Gshhg::new().unwrap(); - // let prepped = &s.prepped; - - let g = Gshhg::get_geometry_from_compressed( - "gshhs/gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz", - ).unwrap(); - let prepped = g.to_prepared_geom().unwrap(); - - (0..10000).into_par_iter().for_each(|k| { - - let x = k % 180; - let y = (k / 180) % 90; - - - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - }); -} - -#[test] -fn test_par_prepped_with_warmup() { - use rayon::prelude::*; - - // let s = Gshhg::new().unwrap(); - // let prepped = &s.prepped; - - let g = Gshhg::get_geometry_from_compressed( - "gshhs/gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz", - ).unwrap(); - let prepped = g.to_prepared_geom().unwrap(); - - let point = CoordSeq::new_from_vec(&[&[10., 50.]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - - (0..10000).into_par_iter().for_each(|k| { - let x = k % 180; - let y = (k / 180) % 90; - - - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - }); -} From 11a94eabcf4596e3f9adefefb7a34bc650cd4af4 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 20:47:52 +0200 Subject: [PATCH 10/24] pyproj, ci: remove refs to geos and static --- .github/workflows/rust.yml | 8 ++++---- pyproject.toml | 2 +- src/mask.rs | 4 ++-- src/shapes.rs | 7 +++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b8c456e..56edf74 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,8 +28,8 @@ jobs: run: | pip install numpy - - run: cargo build --features static --verbose - - run: cargo test --features static --verbose + - run: cargo build --verbose + - run: cargo test --verbose nightly: runs-on: ubuntu-latest @@ -54,5 +54,5 @@ jobs: sudo apt-get -y install build-essential libssl-dev pip install numpy - - run: cargo build --features geos/static,nightly --verbose - - run: cargo test --features geos/static,nightly --verbose + - run: cargo build --features nightly --verbose + - run: cargo test --features nightly --verbose diff --git a/pyproject.toml b/pyproject.toml index b1f8b24..7d66f3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.4"] build-backend = "maturin" [tool.maturin] -features = [ "extension-module" , "static" ] +features = [ "extension-module" ] [tool.pytest.ini_options] minversion = "6.0" diff --git a/src/mask.rs b/src/mask.rs index 4470173..07f3502 100644 --- a/src/mask.rs +++ b/src/mask.rs @@ -154,11 +154,11 @@ impl RoaringMask { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned() + .unbind() } pub fn contains_many_par( diff --git a/src/shapes.rs b/src/shapes.rs index ab3fe85..52472f3 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,8 +1,7 @@ use pyo3::{prelude::*, types::PyBytes}; use std::borrow::Borrow; -use std::convert::TryFrom; use std::fs::File; -use std::io::{self, prelude::*, Cursor}; +use std::io::{self, prelude::*}; use std::path::Path; use geo::{point, Contains, Geometry}; @@ -97,11 +96,11 @@ impl Gshhg { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned() + .to_owned().into() } pub fn contains_many_par( From 6fb76402dd352eb77ff27b9d974b83f9366fd2aa Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 20:53:16 +0200 Subject: [PATCH 11/24] nightly: simd --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index efa8972..7aad679 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,8 +39,8 @@ path-slash = "0.2" [features] extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] -nightly = [] -default = [ "simd" ] +nightly = [ "simd" ] +default = [ ] [profile.release] debug = true From 90d5479905f1f05fa41b1c98af7ea674d5bd9fa7 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 20:58:32 +0200 Subject: [PATCH 12/24] py03: fix deprecated --- build.rs | 7 +++++-- src/devel/make_bitmap.rs | 9 ++++++--- src/lib.rs | 9 ++++++--- src/mask.rs | 4 +++- src/shapes.rs | 29 ++++++++--------------------- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/build.rs b/build.rs index e0249b4..ad52f64 100644 --- a/build.rs +++ b/build.rs @@ -1,8 +1,8 @@ +use path_slash::PathExt; use std::env; use std::fs; use std::io::prelude::*; use std::path::Path; -use path_slash::PathExt; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; pub static GSHHS_F_CS: &str = "05bdf3089407b9829a7a5be7ee43f1e4205f2bbc641e4778af77e4814be216da"; @@ -83,6 +83,9 @@ fn copy_or_download(from: impl AsRef, csum: &str) { if &expected != &actual.as_ref() { // Delete erronous file fs::remove_file(&full_to).unwrap(); - panic!("Checksum mismatched for {:?}, downloaded file deleted..", &from); + panic!( + "Checksum mismatched for {:?}, downloaded file deleted..", + &from + ); } } diff --git a/src/devel/make_bitmap.rs b/src/devel/make_bitmap.rs index 4bd42d8..9873f56 100644 --- a/src/devel/make_bitmap.rs +++ b/src/devel/make_bitmap.rs @@ -1,6 +1,6 @@ -use std::io::prelude::*; -use std::fs::File; use roaring::*; +use std::fs::File; +use std::io::prelude::*; fn main() -> std::io::Result<()> { println!("opening mask.bin.."); @@ -33,7 +33,10 @@ fn main() -> std::io::Result<()> { } } - println!("serialized size: {} mb", tmap.serialized_size() / 1024 / 1024); + println!( + "serialized size: {} mb", + tmap.serialized_size() / 1024 / 1024 + ); println!("serializing bitmap to file: mask.tbmap.."); { diff --git a/src/lib.rs b/src/lib.rs index 3d8f7e6..6348a8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -136,11 +136,12 @@ impl RoaringLandmask { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned() + .unbind() + .into() } pub fn contains_many_par( @@ -156,7 +157,9 @@ impl RoaringLandmask { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } diff --git a/src/mask.rs b/src/mask.rs index 07f3502..15e86df 100644 --- a/src/mask.rs +++ b/src/mask.rs @@ -174,7 +174,9 @@ impl RoaringMask { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } diff --git a/src/shapes.rs b/src/shapes.rs index 52472f3..967a62c 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,7 +1,7 @@ -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use std::borrow::Borrow; use std::fs::File; -use std::io::{self, prelude::*}; +use std::io; use std::path::Path; use geo::{point, Contains, Geometry}; @@ -39,7 +39,7 @@ impl Gshhg { impl Gshhg { /// Make a new Gshhg shapes instance. #[staticmethod] - pub fn new(py: Python) -> io::Result { + pub fn new(_py: Python) -> io::Result { use crate::GsshgData; let buf = GsshgData::get(&GSHHS_F) @@ -50,22 +50,6 @@ impl Gshhg { Gshhg::from_geom(geom) } - /// Get the WKB for the GSHHG shapes (full resolution). - #[staticmethod] - pub fn wkb(py: Python) -> io::Result<&PyBytes> { - use crate::GsshgData; - - let buf = GsshgData::get(&GSHHS_F) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "cannot find shapes"))?; - let buf: &[u8] = buf.data.borrow(); - let mut fd = xz2::read::XzDecoder::new(buf); - - let mut buf = Vec::new(); - fd.read_to_end(&mut buf)?; - - Ok(PyBytes::new(py, &buf)) - } - /// Check if point (x, y) is on land. /// /// `x` is longitude, [-180, 180] east @@ -100,7 +84,8 @@ impl Gshhg { py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned().into() + .to_owned() + .into() } pub fn contains_many_par( @@ -116,7 +101,9 @@ impl Gshhg { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } From 478e7de2bcb190e74477bd9107c363776db2143c Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 21:01:34 +0200 Subject: [PATCH 13/24] ci: mac add target --- .github/workflows/package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 08ba4a6..7a9a699 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -36,6 +36,7 @@ jobs: - name: Build wheels - universal2 uses: messense/maturin-action@v1 with: + target: ${{ matrix.target }} args: --release --universal2 --out dist --sdist - name: Install built wheel - universal2 run: | From 532cba2e28bd017f728f514ec2c919a1f3463798 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 21:06:29 +0200 Subject: [PATCH 14/24] ci: package -> python --- .github/workflows/package.yml | 173 ---------------------------------- .github/workflows/python.yml | 119 +++++++++++++++++++++++ 2 files changed, 119 insertions(+), 173 deletions(-) delete mode 100644 .github/workflows/package.yml create mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index 7a9a699..0000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: Python Build and test wheels - -on: - push: - branches: - - main - tags: - - v* - pull_request: - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.11 - architecture: x64 - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - default: true - - name: Build wheels - x86_64 - uses: messense/maturin-action@v1 - with: - target: x86_64 - args: --release --out dist - - name: Install built wheel - x86_64 - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - - name: Build wheels - universal2 - uses: messense/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --universal2 --out dist --sdist - - name: Install built wheel - universal2 - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - cd tests && pytest - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - windows: - runs-on: windows-latest - strategy: - matrix: - target: [x64] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.11 - architecture: ${{ matrix.target }} - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - default: true - - name: Build wheels - uses: messense/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist - - name: Install built wheel - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - # pip install pytest pytest-benchmark numpy - # cd tests && pytest -sv --log-cli-level=debug - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - linux: - runs-on: ubuntu-latest - strategy: - matrix: - target: [x86_64, i686] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.11 - architecture: x64 - - name: Build Wheels - uses: messense/maturin-action@v1 - with: - rust-toolchain: stable - target: ${{ matrix.target }} - manylinux: auto - args: --release --out dist - - name: Install built wheel - if: matrix.target == 'x86_64' - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - # linux-cross: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # target: [aarch64, armv7, s390x, ppc64le, ppc64] - # steps: - # - uses: actions/checkout@v2 - # - uses: actions/setup-python@v2 - # with: - # python-version: 3.9 - # - name: Build Wheels - # uses: messense/maturin-action@v1 - # with: - # rust-toolchain: stable - # target: ${{ matrix.target }} - # manylinux: auto - # args: --release --out dist --no-sdist - # - uses: uraimo/run-on-arch-action@v2.0.5 - # if: matrix.target != 'ppc64' - # name: Install built wheel - # with: - # arch: ${{ matrix.target }} - # distro: ubuntu18.04 - # githubToken: ${{ github.token }} - # # Mount the dist directory as /artifacts in the container - # dockerRunArgs: | - # --volume "${PWD}/dist:/artifacts" - # install: | - # apt-get update - # apt-get install -y --no-install-recommends python3 python3-pip - # pip3 install -U pip pytest - # run: | - # ls -lrth /artifacts - # pip3 install roaring-landmask --no-index --find-links /artifacts --force-reinstall - # cd tests && pytest - # - name: Upload wheels - # uses: actions/upload-artifact@v2 - # with: - # name: wheels - # path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ macos, windows, linux ] - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - - uses: actions/setup-python@v2 - with: - python-version: 3.11 - - name: Publish to PyPi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - pip install --upgrade twine - twine upload --skip-existing * diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..89cdabd --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,119 @@ +# This file is autogenerated by maturin v1.3.2 +# To update, run +# +# maturin generate-ci github +# +name: Python Build and test wheels + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64, x86] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: ${{ matrix.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + macos: + runs-on: macos-latest + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v3 + with: + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }} + with: + command: upload + args: --non-interactive --skip-existing * From a2c6f427d07bdf1871334e7fc8aa6c6a1fe15ca3 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 21:16:27 +0200 Subject: [PATCH 15/24] ci: python also test --- .github/workflows/python.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 89cdabd..b148c89 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -35,6 +35,11 @@ jobs: args: --release --out dist sccache: 'true' manylinux: auto + - name: Install and test + run: | + pip install roaring-landmask --no-index --find-links dist --force-reinstall + pip install pytest pytest-benchmark numpy shapely + cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -58,6 +63,11 @@ jobs: target: ${{ matrix.target }} args: --release --out dist sccache: 'true' + - name: Install and test + run: | + pip install roaring-landmask --no-index --find-links dist --force-reinstall + pip install pytest pytest-benchmark numpy shapely + cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -80,6 +90,11 @@ jobs: target: ${{ matrix.target }} args: --release --out dist sccache: 'true' + - name: Install and test + run: | + pip install roaring-landmask --no-index --find-links dist --force-reinstall + pip install pytest pytest-benchmark numpy shapely + cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: From 01ca7dcc3d0f66bf74503b25665750f8ced8993b Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 21:21:19 +0200 Subject: [PATCH 16/24] Revert "ci: python also test" This reverts commit a2c6f427d07bdf1871334e7fc8aa6c6a1fe15ca3. --- .github/workflows/python.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b148c89..89cdabd 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -35,11 +35,6 @@ jobs: args: --release --out dist sccache: 'true' manylinux: auto - - name: Install and test - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -63,11 +58,6 @@ jobs: target: ${{ matrix.target }} args: --release --out dist sccache: 'true' - - name: Install and test - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -90,11 +80,6 @@ jobs: target: ${{ matrix.target }} args: --release --out dist sccache: 'true' - - name: Install and test - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - name: Upload wheels uses: actions/upload-artifact@v3 with: From a6c90143a3c3ba4bd192964b5276400cccd573b4 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 18 May 2024 21:23:12 +0200 Subject: [PATCH 17/24] nightly: -simd --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7aad679..bd46847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ path-slash = "0.2" [features] extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] -nightly = [ "simd" ] +nightly = [ ] default = [ ] [profile.release] From b8e3a21cf7147fe29aa6ed84135c322111ca9485 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sun, 19 May 2024 09:22:24 +0200 Subject: [PATCH 18/24] wip: use RTree --- Cargo.toml | 1 + src/shapes.rs | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd46847..829648f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ rust-embed = "8" xz2 = "0.1" ndarray = { version = "0.15", features = [ "rayon" ] } wkb = "0.7.1" +rstar = "0.12.0" [dev-dependencies] rayon = "1" diff --git a/src/shapes.rs b/src/shapes.rs index 967a62c..b4a22ac 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,10 +1,11 @@ use pyo3::prelude::*; -use std::borrow::Borrow; +use std::{borrow::Borrow, convert::TryInto}; use std::fs::File; use std::io; use std::path::Path; -use geo::{point, Contains, Geometry}; +use geo::{point, Contains, Geometry, Polygon, MultiPolygon}; +use rstar::RTree; use numpy::{PyArray, PyReadonlyArrayDyn}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; @@ -12,11 +13,15 @@ pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N #[pyclass] #[derive(Clone)] pub struct Gshhg { - geom: Geometry, + geom: RTree, } impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { + let geom: MultiPolygon = geom.try_into().unwrap(); + assert!(geom.0.len() > 10); + + let geom = RTree::bulk_load(geom.0); Ok(Gshhg { geom }) } @@ -60,15 +65,16 @@ impl Gshhg { let x = super::modulate_longitude(x); debug_assert!(x >= -180. && x <= 180.); assert!(y > -90. && y <= 90.); - - let p = point!(x: x, y: y); - self.geom.contains(&p) + self.contains_unchecked(x, y) } /// Same as `contains`, but does not check for bounds. pub(crate) fn contains_unchecked(&self, x: f64, y: f64) -> bool { let p = point!(x: x, y: y); - self.geom.contains(&p) + // let po = self.geom.nearest_neighbor(&p); + // true + self.geom.locate_at_point(&p).is_some() + // self.geom.contains(&p) } pub fn contains_many( @@ -120,7 +126,7 @@ mod tests { } #[test] - fn test_load() { + fn test_load_embedded() { pyo3::prepare_freethreaded_python(); Python::with_gil(|py| Gshhg::new(py)).unwrap(); } From 60bc592b01c781af591f74fc3229c454e5fbf4c9 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sun, 19 May 2024 09:56:43 +0200 Subject: [PATCH 19/24] wrap polygon and implement contains --- src/shapes.rs | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/shapes.rs b/src/shapes.rs index b4a22ac..013049e 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,27 +1,53 @@ use pyo3::prelude::*; -use std::{borrow::Borrow, convert::TryInto}; use std::fs::File; use std::io; use std::path::Path; +use std::{borrow::Borrow, convert::TryInto}; -use geo::{point, Contains, Geometry, Polygon, MultiPolygon}; -use rstar::RTree; +use geo::{point, Contains, Geometry, MultiPolygon, Point, Polygon}; use numpy::{PyArray, PyReadonlyArrayDyn}; +use rstar::{PointDistance, RTree, RTreeObject, AABB}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; #[pyclass] #[derive(Clone)] pub struct Gshhg { - geom: RTree, + geom: RTree, +} + +#[derive(Clone)] +struct PolW(Polygon); + +impl RTreeObject for PolW { + type Envelope = AABB>; + + fn envelope(&self) -> Self::Envelope { + self.0.envelope() + } +} + +impl PointDistance for PolW { + fn distance_2(&self, _point: &Point) -> f64 { + panic!("this should only be used for contains, the distance will give the wrong answer"); + } + + fn contains_point(&self, point: &Point) -> bool { + self.0.contains(point) + } + + fn distance_2_if_less_or_equal(&self, _point: &Point, _max_distance: f64) -> Option { + panic!("this should only be used for contains, the distance will give the wrong answer"); + } } impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { let geom: MultiPolygon = geom.try_into().unwrap(); assert!(geom.0.len() > 10); + let geoms = geom.0.into_iter().map(|p| PolW(p)).collect(); - let geom = RTree::bulk_load(geom.0); + let geom = RTree::bulk_load(geoms); Ok(Gshhg { geom }) } @@ -71,10 +97,7 @@ impl Gshhg { /// Same as `contains`, but does not check for bounds. pub(crate) fn contains_unchecked(&self, x: f64, y: f64) -> bool { let p = point!(x: x, y: y); - // let po = self.geom.nearest_neighbor(&p); - // true self.geom.locate_at_point(&p).is_some() - // self.geom.contains(&p) } pub fn contains_many( From 47b478385903f83ebc1a1501c2ae26963feaa52a Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sun, 19 May 2024 17:08:03 +0200 Subject: [PATCH 20/24] check if envelope first --- src/shapes.rs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/shapes.rs b/src/shapes.rs index 013049e..8caefb9 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -6,7 +6,7 @@ use std::{borrow::Borrow, convert::TryInto}; use geo::{point, Contains, Geometry, MultiPolygon, Point, Polygon}; use numpy::{PyArray, PyReadonlyArrayDyn}; -use rstar::{PointDistance, RTree, RTreeObject, AABB}; +use rstar::{PointDistance, RTree, RTreeObject, AABB, Envelope}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; @@ -17,13 +17,25 @@ pub struct Gshhg { } #[derive(Clone)] -struct PolW(Polygon); +struct PolW { + p: Polygon, + e: AABB>, +} + +impl PolW { + pub fn from(p: Polygon) -> PolW { + PolW { + p: p.clone(), + e: p.envelope() + } + } +} impl RTreeObject for PolW { type Envelope = AABB>; fn envelope(&self) -> Self::Envelope { - self.0.envelope() + self.e } } @@ -33,7 +45,13 @@ impl PointDistance for PolW { } fn contains_point(&self, point: &Point) -> bool { - self.0.contains(point) + // fast contains from libgeos + // https://github.com/libgeos/geos/blob/main/src/geom/prep/PreparedPolygonContainsProperly.cpp + if !self.e.contains_point(point) { + return false; + } + + self.p.contains(point) } fn distance_2_if_less_or_equal(&self, _point: &Point, _max_distance: f64) -> Option { @@ -45,7 +63,7 @@ impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { let geom: MultiPolygon = geom.try_into().unwrap(); assert!(geom.0.len() > 10); - let geoms = geom.0.into_iter().map(|p| PolW(p)).collect(); + let geoms = geom.0.into_iter().map(|p| PolW::from(p)).collect(); let geom = RTree::bulk_load(geoms); Ok(Gshhg { geom }) From f8899692a7cda54d16e4b08519bbccea6a16f7e8 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Fri, 16 Aug 2024 15:16:28 +0200 Subject: [PATCH 21/24] rtree of prepared geoms --- Cargo.toml | 2 ++ src/shapes.rs | 11 ++++++----- tests/test_open_wkb.py | 3 +++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 829648f..201a338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,3 +46,5 @@ default = [ ] [profile.release] debug = true +[patch.crates-io] +geo = { path = "/home/gauteh/dev/misc/geo/geo" } diff --git a/src/shapes.rs b/src/shapes.rs index 8caefb9..6685617 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -4,7 +4,7 @@ use std::io; use std::path::Path; use std::{borrow::Borrow, convert::TryInto}; -use geo::{point, Contains, Geometry, MultiPolygon, Point, Polygon}; +use geo::{point, Relate, Contains, Geometry, MultiPolygon, Point, Polygon, PreparedGeometry}; use numpy::{PyArray, PyReadonlyArrayDyn}; use rstar::{PointDistance, RTree, RTreeObject, AABB, Envelope}; @@ -18,15 +18,15 @@ pub struct Gshhg { #[derive(Clone)] struct PolW { - p: Polygon, + p: PreparedGeometry<'static>, e: AABB>, } impl PolW { pub fn from(p: Polygon) -> PolW { PolW { - p: p.clone(), - e: p.envelope() + e: p.envelope(), + p: PreparedGeometry::from(p), } } } @@ -45,13 +45,14 @@ impl PointDistance for PolW { } fn contains_point(&self, point: &Point) -> bool { + // return self.p.covers(point); // fast contains from libgeos // https://github.com/libgeos/geos/blob/main/src/geom/prep/PreparedPolygonContainsProperly.cpp if !self.e.contains_point(point) { return false; } - self.p.contains(point) + self.p.relate(point).is_contains() } fn distance_2_if_less_or_equal(&self, _point: &Point, _max_distance: f64) -> Option { diff --git a/tests/test_open_wkb.py b/tests/test_open_wkb.py index 75b14d3..77dbded 100644 --- a/tests/test_open_wkb.py +++ b/tests/test_open_wkb.py @@ -1,9 +1,12 @@ from roaring_landmask import Gshhg import shapely.wkb as wkb +import pytest +@pytest.mark.skip def test_read_wkb(): w = Gshhg.wkb() +@pytest.mark.skip def test_load_wkb(): w = Gshhg.wkb() polys = wkb.loads(w) From c192eb49c3e04a01832248f38399ff8207a4cd91 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Fri, 16 Aug 2024 17:59:38 +0200 Subject: [PATCH 22/24] shapes: prepare multipolygon --- src/shapes.rs | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/shapes.rs b/src/shapes.rs index 6685617..c17176a 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -4,16 +4,16 @@ use std::io; use std::path::Path; use std::{borrow::Borrow, convert::TryInto}; -use geo::{point, Relate, Contains, Geometry, MultiPolygon, Point, Polygon, PreparedGeometry}; +use geo::{point, Contains, Geometry, MultiPolygon, Point, Polygon, PreparedGeometry, Relate}; use numpy::{PyArray, PyReadonlyArrayDyn}; -use rstar::{PointDistance, RTree, RTreeObject, AABB, Envelope}; +use rstar::{Envelope, PointDistance, RTree, RTreeObject, AABB}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; #[pyclass] #[derive(Clone)] pub struct Gshhg { - geom: RTree, + geom: PreparedGeometry<'static>, } #[derive(Clone)] @@ -63,11 +63,13 @@ impl PointDistance for PolW { impl Gshhg { pub fn from_geom(geom: Geometry) -> io::Result { let geom: MultiPolygon = geom.try_into().unwrap(); - assert!(geom.0.len() > 10); - let geoms = geom.0.into_iter().map(|p| PolW::from(p)).collect(); + // assert!(geom.0.len() > 10); + // let geoms = geom.0.into_iter().map(|p| PolW::from(p)).collect(); - let geom = RTree::bulk_load(geoms); - Ok(Gshhg { geom }) + // let geom = RTree::bulk_load(geoms); + Ok(Gshhg { + geom: PreparedGeometry::from(geom), + }) } pub fn from_compressed>(path: P) -> io::Result { @@ -83,13 +85,8 @@ impl Gshhg { let geom = wkb::wkb_to_geom(&mut fd).unwrap(); Ok(geom) } -} -#[pymethods] -impl Gshhg { - /// Make a new Gshhg shapes instance. - #[staticmethod] - pub fn new(_py: Python) -> io::Result { + pub fn geom_from_embedded() -> io::Result { use crate::GsshgData; let buf = GsshgData::get(&GSHHS_F) @@ -97,7 +94,17 @@ impl Gshhg { let buf: &[u8] = buf.data.borrow(); let mut fd = xz2::read::XzDecoder::new(buf); let geom = wkb::wkb_to_geom(&mut fd).unwrap(); - Gshhg::from_geom(geom) + + Ok(geom) + } +} + +#[pymethods] +impl Gshhg { + /// Make a new Gshhg shapes instance. + #[staticmethod] + pub fn new(_py: Python) -> io::Result { + Gshhg::from_geom(Gshhg::geom_from_embedded()?) } /// Check if point (x, y) is on land. @@ -116,7 +123,9 @@ impl Gshhg { /// Same as `contains`, but does not check for bounds. pub(crate) fn contains_unchecked(&self, x: f64, y: f64) -> bool { let p = point!(x: x, y: y); - self.geom.locate_at_point(&p).is_some() + // self.geom.locate_at_point(&p).is_some() + // self.geom.relate(&p).is_contains() + self.geom.relate(&p).is_covers() } pub fn contains_many( From 00b02928b9e98643dd6987a3cdb1400cd8cb58b5 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Fri, 16 Aug 2024 17:59:52 +0200 Subject: [PATCH 23/24] test: test load prepared --- src/shapes.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/shapes.rs b/src/shapes.rs index c17176a..08c1e5c 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -200,6 +200,12 @@ mod tests { }) } + #[test] + fn prepare_geometry() { + let geom = Gshhg::geom_from_embedded().unwrap(); + let prep = PreparedGeometry::from(geom); + } + #[cfg(feature = "nightly")] mod benches { use super::*; From b955480c6ed427ea6f26ef2e73987b6fc185ae42 Mon Sep 17 00:00:00 2001 From: Gaute Hope Date: Sat, 17 Aug 2024 15:35:21 +0200 Subject: [PATCH 24/24] debug contains --- pyproject.toml | 2 +- src/shapes.rs | 1 + tests/test_dateline.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d66f3e..28b8f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.4"] build-backend = "maturin" [tool.maturin] -features = [ "extension-module" ] +features = [ "extension-module", "simd", "nightly" ] [tool.pytest.ini_options] minversion = "6.0" diff --git a/src/shapes.rs b/src/shapes.rs index 08c1e5c..0f69931 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -125,6 +125,7 @@ impl Gshhg { let p = point!(x: x, y: y); // self.geom.locate_at_point(&p).is_some() // self.geom.relate(&p).is_contains() + println!("contains unchecked"); self.geom.relate(&p).is_covers() } diff --git a/tests/test_dateline.py b/tests/test_dateline.py index f877073..4e46b18 100644 --- a/tests/test_dateline.py +++ b/tests/test_dateline.py @@ -9,7 +9,7 @@ def test_dateline(): xx, yy = np.meshgrid(x, y) xx, yy = xx.ravel(), yy.ravel() - mm = mask.contains_many(xx, yy) + mm = mask.contains_many_par(xx, yy) # Offset x2 = np.linspace(180, 540, 100) @@ -18,7 +18,7 @@ def test_dateline(): xx, yy = np.meshgrid(x2, y2) xx, yy = xx.ravel(), yy.ravel() - MM = mask.contains_many(xx, yy) + MM = mask.contains_many_par(xx, yy) np.testing.assert_array_equal(mm, MM)