diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index a717cf1..0000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,172 +0,0 @@ -name: Python Build and test wheels - -on: - push: - branches: - - main - tags: - - v* - pull_request: - -jobs: - macos: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - architecture: x64 - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - default: true - - name: Build wheels - x86_64 - uses: messense/maturin-action@v1 - with: - target: x86_64 - args: --release --out dist - - name: Install built wheel - x86_64 - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - - name: Build wheels - universal2 - uses: messense/maturin-action@v1 - with: - args: --release --universal2 --out dist --sdist - - name: Install built wheel - universal2 - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - cd tests && pytest - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - windows: - runs-on: windows-latest - strategy: - matrix: - target: [x64] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - architecture: ${{ matrix.target }} - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - default: true - - name: Build wheels - uses: messense/maturin-action@v1 - with: - target: ${{ matrix.target }} - args: --release --out dist - - name: Install built wheel - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - # pip install pytest pytest-benchmark numpy - # cd tests && pytest -sv --log-cli-level=debug - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - linux: - runs-on: ubuntu-latest - strategy: - matrix: - target: [x86_64, i686] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - architecture: x64 - - name: Build Wheels - uses: messense/maturin-action@v1 - with: - rust-toolchain: stable - target: ${{ matrix.target }} - manylinux: auto - args: --release --out dist - - name: Install built wheel - if: matrix.target == 'x86_64' - run: | - pip install roaring-landmask --no-index --find-links dist --force-reinstall - pip install pytest pytest-benchmark numpy shapely - cd tests && pytest - - name: Upload wheels - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist - - # linux-cross: - # runs-on: ubuntu-latest - # strategy: - # matrix: - # target: [aarch64, armv7, s390x, ppc64le, ppc64] - # steps: - # - uses: actions/checkout@v2 - # - uses: actions/setup-python@v2 - # with: - # python-version: 3.9 - # - name: Build Wheels - # uses: messense/maturin-action@v1 - # with: - # rust-toolchain: stable - # target: ${{ matrix.target }} - # manylinux: auto - # args: --release --out dist --no-sdist - # - uses: uraimo/run-on-arch-action@v2.0.5 - # if: matrix.target != 'ppc64' - # name: Install built wheel - # with: - # arch: ${{ matrix.target }} - # distro: ubuntu18.04 - # githubToken: ${{ github.token }} - # # Mount the dist directory as /artifacts in the container - # dockerRunArgs: | - # --volume "${PWD}/dist:/artifacts" - # install: | - # apt-get update - # apt-get install -y --no-install-recommends python3 python3-pip - # pip3 install -U pip pytest - # run: | - # ls -lrth /artifacts - # pip3 install roaring-landmask --no-index --find-links /artifacts --force-reinstall - # cd tests && pytest - # - name: Upload wheels - # uses: actions/upload-artifact@v2 - # with: - # name: wheels - # path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - needs: [ macos, windows, linux ] - steps: - - uses: actions/download-artifact@v2 - with: - name: wheels - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Publish to PyPi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - pip install --upgrade twine - twine upload --skip-existing * diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..89cdabd --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,119 @@ +# This file is autogenerated by maturin v1.3.2 +# To update, run +# +# maturin generate-ci github +# +name: Python Build and test wheels + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64, x86] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: ${{ matrix.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + macos: + runs-on: macos-latest + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v3 + with: + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }} + with: + command: upload + args: --non-interactive --skip-existing * diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d741c7b..56edf74 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -21,15 +21,15 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Install deps run: | pip install numpy - - run: cargo build --features geos/static --verbose - - run: cargo test --features geos/static --verbose + - run: cargo build --verbose + - run: cargo test --verbose nightly: runs-on: ubuntu-latest @@ -45,7 +45,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 architecture: x64 - name: Install deps @@ -54,5 +54,5 @@ jobs: sudo apt-get -y install build-essential libssl-dev pip install numpy - - run: cargo build --features geos/static,nightly --verbose - - run: cargo test --features geos/static,nightly --verbose + - run: cargo build --features nightly --verbose + - run: cargo test --features nightly --verbose diff --git a/Cargo.toml b/Cargo.toml index b6a7ce2..201a338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,30 +18,33 @@ name = "make_bitmap" path = "src/devel/make_bitmap.rs" [dependencies] -geos = { version = "8" } +geo = "0.28" lazy_static = "1.4" -numpy = { version = "0.19" } -pyo3 = { version = "0.19" , features = [ "abi3-py38" ] } +numpy = { version = "0.21" } +pyo3 = { version = "0.21" , features = [ "abi3-py310" ] } roaring = "0.10" rust-embed = "8" xz2 = "0.1" ndarray = { version = "0.15", features = [ "rayon" ] } +wkb = "0.7.1" +rstar = "0.12.0" [dev-dependencies] rayon = "1" [build-dependencies] -reqwest = { version = "0.11", default-features = false, features = [ "blocking", "rustls-tls" ] } -ring = "0.16" -path-slash = "0.1" +reqwest = { version = "0.12", default-features = false, features = [ "blocking", "rustls-tls" ] } +ring = "0.17" +path-slash = "0.2" [features] extension-module = ["pyo3/extension-module"] simd = [ "roaring/simd" ] -static = [ "geos/static" ] -nightly = [] -default = [] +nightly = [ ] +default = [ ] [profile.release] debug = true +[patch.crates-io] +geo = { path = "/home/gauteh/dev/misc/geo/geo" } diff --git a/build.rs b/build.rs index e0249b4..ad52f64 100644 --- a/build.rs +++ b/build.rs @@ -1,8 +1,8 @@ +use path_slash::PathExt; use std::env; use std::fs; use std::io::prelude::*; use std::path::Path; -use path_slash::PathExt; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; pub static GSHHS_F_CS: &str = "05bdf3089407b9829a7a5be7ee43f1e4205f2bbc641e4778af77e4814be216da"; @@ -83,6 +83,9 @@ fn copy_or_download(from: impl AsRef, csum: &str) { if &expected != &actual.as_ref() { // Delete erronous file fs::remove_file(&full_to).unwrap(); - panic!("Checksum mismatched for {:?}, downloaded file deleted..", &from); + panic!( + "Checksum mismatched for {:?}, downloaded file deleted..", + &from + ); } } diff --git a/pyproject.toml b/pyproject.toml index 18f4082..28b8f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1.4"] build-backend = "maturin" [tool.maturin] -features = [ "extension-module" , "static" ] +features = [ "extension-module", "simd", "nightly" ] [tool.pytest.ini_options] minversion = "6.0" diff --git a/src/devel/make_bitmap.rs b/src/devel/make_bitmap.rs index 4bd42d8..9873f56 100644 --- a/src/devel/make_bitmap.rs +++ b/src/devel/make_bitmap.rs @@ -1,6 +1,6 @@ -use std::io::prelude::*; -use std::fs::File; use roaring::*; +use std::fs::File; +use std::io::prelude::*; fn main() -> std::io::Result<()> { println!("opening mask.bin.."); @@ -33,7 +33,10 @@ fn main() -> std::io::Result<()> { } } - println!("serialized size: {} mb", tmap.serialized_size() / 1024 / 1024); + println!( + "serialized size: {} mb", + tmap.serialized_size() / 1024 / 1024 + ); println!("serializing bitmap to file: mask.tbmap.."); { diff --git a/src/lib.rs b/src/lib.rs index 3d8f7e6..6348a8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -136,11 +136,12 @@ impl RoaringLandmask { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned() + .unbind() + .into() } pub fn contains_many_par( @@ -156,7 +157,9 @@ impl RoaringLandmask { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } diff --git a/src/mask.rs b/src/mask.rs index 4470173..15e86df 100644 --- a/src/mask.rs +++ b/src/mask.rs @@ -154,11 +154,11 @@ impl RoaringMask { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) - .to_owned() + .unbind() } pub fn contains_many_par( @@ -174,7 +174,9 @@ impl RoaringMask { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } diff --git a/src/shapes.rs b/src/shapes.rs index 4917277..0f69931 100644 --- a/src/shapes.rs +++ b/src/shapes.rs @@ -1,65 +1,74 @@ -use pyo3::{prelude::*, types::PyBytes}; -use std::borrow::Borrow; +use pyo3::prelude::*; use std::fs::File; -use std::io::{self, prelude::*}; +use std::io; use std::path::Path; +use std::{borrow::Borrow, convert::TryInto}; -use geos::{CoordSeq, Geom, Geometry, PreparedGeometry}; +use geo::{point, Contains, Geometry, MultiPolygon, Point, Polygon, PreparedGeometry, Relate}; use numpy::{PyArray, PyReadonlyArrayDyn}; +use rstar::{Envelope, PointDistance, RTree, RTreeObject, AABB}; pub static GSHHS_F: &str = "gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz"; #[pyclass] +#[derive(Clone)] pub struct Gshhg { - geom: *mut Geometry<'static>, + geom: PreparedGeometry<'static>, +} - // prepped requires `geom` above to be around, and is valid as long as geom is alive. - prepped: PreparedGeometry<'static>, +#[derive(Clone)] +struct PolW { + p: PreparedGeometry<'static>, + e: AABB>, } -impl Drop for Gshhg { - fn drop(&mut self) { - unsafe { drop(Box::from_raw(self.geom)) } +impl PolW { + pub fn from(p: Polygon) -> PolW { + PolW { + e: p.envelope(), + p: PreparedGeometry::from(p), + } } } -// PreparedGeometry is Send+Sync, Geometry is Send+Sync. *mut Geometry is never modified. -unsafe impl Send for Gshhg {} -unsafe impl Sync for Gshhg {} +impl RTreeObject for PolW { + type Envelope = AABB>; -// `PreparededGeometry::contains` needs a call to `contains` before it is thread-safe: -// https://github.com/georust/geos/issues/95 -fn warmup_prepped(prepped: &PreparedGeometry<'_>) { - let point = CoordSeq::new_from_vec(&[&[0.0, 0.0]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); + fn envelope(&self) -> Self::Envelope { + self.e + } } -impl Clone for Gshhg { - fn clone(&self) -> Self { - let gptr = self.geom.clone(); - debug_assert!(gptr != self.geom); - let prepped = unsafe { (&*gptr).to_prepared_geom().unwrap() }; - warmup_prepped(&prepped); +impl PointDistance for PolW { + fn distance_2(&self, _point: &Point) -> f64 { + panic!("this should only be used for contains, the distance will give the wrong answer"); + } - Gshhg { - geom: gptr, - prepped, + fn contains_point(&self, point: &Point) -> bool { + // return self.p.covers(point); + // fast contains from libgeos + // https://github.com/libgeos/geos/blob/main/src/geom/prep/PreparedPolygonContainsProperly.cpp + if !self.e.contains_point(point) { + return false; } + + self.p.relate(point).is_contains() + } + + fn distance_2_if_less_or_equal(&self, _point: &Point, _max_distance: f64) -> Option { + panic!("this should only be used for contains, the distance will give the wrong answer"); } } impl Gshhg { - pub fn from_geom(geom: Geometry<'static>) -> io::Result { - let bxd = Box::new(geom); - let gptr = Box::into_raw(bxd); - let prepped = unsafe { (&*gptr).to_prepared_geom() } - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "cannot prepare geomtry"))?; - warmup_prepped(&prepped); + pub fn from_geom(geom: Geometry) -> io::Result { + let geom: MultiPolygon = geom.try_into().unwrap(); + // assert!(geom.0.len() > 10); + // let geoms = geom.0.into_iter().map(|p| PolW::from(p)).collect(); + // let geom = RTree::bulk_load(geoms); Ok(Gshhg { - geom: gptr, - prepped, + geom: PreparedGeometry::from(geom), }) } @@ -69,41 +78,33 @@ impl Gshhg { Gshhg::from_geom(g) } - pub fn get_geometry_from_compressed>(path: P) -> io::Result> { + pub fn get_geometry_from_compressed>(path: P) -> io::Result { let fd = File::open(path)?; let fd = io::BufReader::new(fd); let mut fd = xz2::bufread::XzDecoder::new(fd); - let mut buf = Vec::new(); - fd.read_to_end(&mut buf)?; - - Ok(geos::Geometry::new_from_wkb(&buf).unwrap()) + let geom = wkb::wkb_to_geom(&mut fd).unwrap(); + Ok(geom) } -} -#[pymethods] -impl Gshhg { - /// Make a new Gshhg shapes instance. - #[staticmethod] - pub fn new(py: Python) -> io::Result { - let buf = Gshhg::wkb(py)?; - let g = geos::Geometry::new_from_wkb(buf.as_bytes()).unwrap(); - Gshhg::from_geom(g) - } - - /// Get the WKB for the GSHHG shapes (full resolution). - #[staticmethod] - pub fn wkb(py: Python) -> io::Result<&PyBytes> { + pub fn geom_from_embedded() -> io::Result { use crate::GsshgData; let buf = GsshgData::get(&GSHHS_F) .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "cannot find shapes"))?; let buf: &[u8] = buf.data.borrow(); let mut fd = xz2::read::XzDecoder::new(buf); + let geom = wkb::wkb_to_geom(&mut fd).unwrap(); - let mut buf = Vec::new(); - fd.read_to_end(&mut buf)?; + Ok(geom) + } +} - Ok(PyBytes::new(py, &buf)) +#[pymethods] +impl Gshhg { + /// Make a new Gshhg shapes instance. + #[staticmethod] + pub fn new(_py: Python) -> io::Result { + Gshhg::from_geom(Gshhg::geom_from_embedded()?) } /// Check if point (x, y) is on land. @@ -116,17 +117,16 @@ impl Gshhg { let x = super::modulate_longitude(x); debug_assert!(x >= -180. && x <= 180.); assert!(y > -90. && y <= 90.); - - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - self.prepped.contains(&point).unwrap() + self.contains_unchecked(x, y) } /// Same as `contains`, but does not check for bounds. pub(crate) fn contains_unchecked(&self, x: f64, y: f64) -> bool { - let point = CoordSeq::new_from_vec(&[&[x, y]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - self.prepped.contains(&point).unwrap() + let p = point!(x: x, y: y); + // self.geom.locate_at_point(&p).is_some() + // self.geom.relate(&p).is_contains() + println!("contains unchecked"); + self.geom.relate(&p).is_covers() } pub fn contains_many( @@ -138,11 +138,12 @@ impl Gshhg { let x = x.as_array(); let y = y.as_array(); - PyArray::from_iter( + PyArray::from_iter_bound( py, x.iter().zip(y.iter()).map(|(x, y)| self.contains(*x, *y)), ) .to_owned() + .into() } pub fn contains_many_par( @@ -158,7 +159,9 @@ impl Gshhg { let contains = Zip::from(&x) .and(&y) .par_map_collect(|x, y| self.contains(*x, *y)); - PyArray::from_owned_array(py, contains).to_owned() + PyArray::from_owned_array_bound(py, contains) + .unbind() + .into() } } @@ -175,7 +178,7 @@ mod tests { } #[test] - fn test_load() { + fn test_load_embedded() { pyo3::prepare_freethreaded_python(); Python::with_gil(|py| Gshhg::new(py)).unwrap(); } @@ -198,6 +201,12 @@ mod tests { }) } + #[test] + fn prepare_geometry() { + let geom = Gshhg::geom_from_embedded().unwrap(); + let prep = PreparedGeometry::from(geom); + } + #[cfg(feature = "nightly")] mod benches { use super::*; diff --git a/tests/test_dateline.py b/tests/test_dateline.py index f877073..4e46b18 100644 --- a/tests/test_dateline.py +++ b/tests/test_dateline.py @@ -9,7 +9,7 @@ def test_dateline(): xx, yy = np.meshgrid(x, y) xx, yy = xx.ravel(), yy.ravel() - mm = mask.contains_many(xx, yy) + mm = mask.contains_many_par(xx, yy) # Offset x2 = np.linspace(180, 540, 100) @@ -18,7 +18,7 @@ def test_dateline(): xx, yy = np.meshgrid(x2, y2) xx, yy = xx.ravel(), yy.ravel() - MM = mask.contains_many(xx, yy) + MM = mask.contains_many_par(xx, yy) np.testing.assert_array_equal(mm, MM) diff --git a/tests/test_geos_par_prepped.rs b/tests/test_geos_par_prepped.rs deleted file mode 100644 index 4460fa8..0000000 --- a/tests/test_geos_par_prepped.rs +++ /dev/null @@ -1,54 +0,0 @@ -use roaring_landmask::Gshhg; -use geos::{CoordSeq, Geom, Geometry}; - -#[ignore] -#[test] -fn test_par_prepped_no_warmup() { - use rayon::prelude::*; - - // let s = Gshhg::new().unwrap(); - // let prepped = &s.prepped; - - let g = Gshhg::get_geometry_from_compressed( - "gshhs/gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz", - ).unwrap(); - let prepped = g.to_prepared_geom().unwrap(); - - (0..10000).into_par_iter().for_each(|k| { - - let x = k % 180; - let y = (k / 180) % 90; - - - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - }); -} - -#[test] -fn test_par_prepped_with_warmup() { - use rayon::prelude::*; - - // let s = Gshhg::new().unwrap(); - // let prepped = &s.prepped; - - let g = Gshhg::get_geometry_from_compressed( - "gshhs/gshhs_f_-180.000000E-90.000000N180.000000E90.000000N.wkb.xz", - ).unwrap(); - let prepped = g.to_prepared_geom().unwrap(); - - let point = CoordSeq::new_from_vec(&[&[10., 50.]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - - (0..10000).into_par_iter().for_each(|k| { - let x = k % 180; - let y = (k / 180) % 90; - - - let point = CoordSeq::new_from_vec(&[&[x as f64, y as f64]]).unwrap(); - let point = Geometry::create_point(point).unwrap(); - prepped.contains(&point).unwrap(); - }); -} diff --git a/tests/test_open_wkb.py b/tests/test_open_wkb.py index 75b14d3..77dbded 100644 --- a/tests/test_open_wkb.py +++ b/tests/test_open_wkb.py @@ -1,9 +1,12 @@ from roaring_landmask import Gshhg import shapely.wkb as wkb +import pytest +@pytest.mark.skip def test_read_wkb(): w = Gshhg.wkb() +@pytest.mark.skip def test_load_wkb(): w = Gshhg.wkb() polys = wkb.loads(w)