diff --git a/.cirrus.yml b/.cirrus.yml index 4c49655..56c31a4 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -46,6 +46,8 @@ cirrus_wheels_macos_arm64_task: build_script: - which python + # needed for discover_version.py + - git fetch --all # needed for submodules - git submodule update --init - uname -m diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 11792d6..3083c00 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -3,6 +3,8 @@ on: branches: [ master ] tags: - v* + pull_request: + branches: [ master ] jobs: build_wheels: @@ -10,14 +12,33 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-12, windows-2019] + os: [ubuntu-22.04, macos-13, windows-2019] fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Checkout submodules - run: git submodule update --init --recursive + run: git submodule update --init --recursive + + - name: Install PCRE2 (Windows only) + if: matrix.os == 'windows-2019' + shell: bash + run: | + pcre2_version=10.42 + download_url=https://github.com/PCRE2Project/pcre2/releases/download/pcre2-${pcre2_version}/pcre2-${pcre2_version}.tar.gz + curl -L $download_url -o pcre2.tar.gz + tar xvzf pcre2.tar.gz + cd pcre2-${pcre2_version} + mkdir build + cd build + cmake .. + ninja + ninja install + - name: Build wheels uses: pypa/cibuildwheel@v2.12.1 env: @@ -28,11 +49,11 @@ jobs: CIBW_TEST_REQUIRES: pytest CIBW_TEST_COMMAND: "pytest -v {package}/tests" - # # Uncomment to get SSH access for testing - # - name: Setup tmate session - # if: failure() - # uses: mxschmitt/action-tmate@v3 - # timeout-minutes: 15 + # Uncomment to get SSH access for testing + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 15 - name: Upload artifacts uses: actions/upload-artifact@v2 @@ -50,10 +71,10 @@ jobs: - name: Check tag id: check-tag run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true + if [[ ${{ github.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "match=true" >> $GITHUB_OUTPUT fi - + - name: Deploy to PyPI if: steps.check-tag.outputs.match == 'true' run: | diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cab7b59..6041400 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Checkout submodules run: git submodule update --init --recursive - name: Set up Python ${{ matrix.python-version }} @@ -60,8 +62,8 @@ jobs: - name: Test with pytest run: | USE_FORTRAN=T pytest -v --ignore QUIP - # # Uncomment to get SSH access for testing - # - name: Setup tmate session - # if: failure() - # uses: mxschmitt/action-tmate@v3 - # timeout-minutes: 15 + # Uncomment to get SSH access for testing + - name: Setup tmate session + if: failure() + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 15 diff --git a/.gitmodules b/.gitmodules index dcd9747..d440586 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "extxyz/libcleri"] path = libcleri - url = https://github.com/transceptor-technology/libcleri + url = https://github.com/libAtoms/libcleri diff --git a/discover_version.py b/discover_version.py new file mode 100644 index 0000000..8719b33 --- /dev/null +++ b/discover_version.py @@ -0,0 +1,94 @@ +# +# Copyright 2022 Lars Pastewka +# +# ### MIT license +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +# +# This is the most minimal-idiotic way of discovering the version that I +# could come up with. It deals with the following issues: +# * If we are installed, we can get the version from package metadata, +# either via importlib.metadata or from pkg_resources. This also holds for +# wheels that contain the metadata. We are good! Yay! +# * If we are not installed, there are two options: +# - We are working within the source git repository. Then +# git describe --tags --always +# yields a reasonable version descriptor, but that is unfortunately not +# PEP 440 compliant (see https://peps.python.org/pep-0440/). We need to +# mangle the version string to yield something compatible. +# - If we install from a source tarball, we need to parse PKG-INFO manually. +# + +import re +import subprocess + +class CannotDiscoverVersion(Exception): + pass + + +def get_version_from_pkg_info(): + """ + Discover version from PKG-INFO file. + """ + f = open('PKG-INFO', 'r') + l = f.readline() + while l: + if l.startswith('Version:'): + return l[8:].strip() + l = f.readline() + raise CannotDiscoverVersion("No line starting with 'Version:' in 'PKG-INFO'.") + + +def get_version_from_git(): + """ + Discover version from git repository. + """ + git_describe = subprocess.run( + ['git', 'describe', '--tags', '--dirty', '--always'], + stdout=subprocess.PIPE) + if git_describe.returncode != 0: + raise CannotDiscoverVersion('git execution failed.') + version = git_describe.stdout.decode('latin-1').strip() + + dirty = version.endswith('-dirty') + + # Make version PEP 440 compliant + if dirty: + version = version.replace('-dirty', '') + version = version.strip('v') # Remove leading 'v' if it exists + version = version.replace('-', '.dev', 1) + version = version.replace('-', '+', 1) + if dirty: + version += '.dirty' + + return version + + +try: + version = get_version_from_git() +except CannotDiscoverVersion: + version = get_version_from_pkg_info() + +# +# Print version to screen +# + +print(version) diff --git a/libcleri b/libcleri index 968759b..9bbc90f 160000 --- a/libcleri +++ b/libcleri @@ -1 +1 @@ -Subproject commit 968759b0ffd8a41b87f9f47422fec9c97cc26ab8 +Subproject commit 9bbc90f6d6c150959343ce858e0a45bfb618030d diff --git a/libextxyz/Makefile b/libextxyz/Makefile index 0d98955..6db3ea4 100644 --- a/libextxyz/Makefile +++ b/libextxyz/Makefile @@ -22,13 +22,15 @@ else dlext ?= so endif +.PHONY: default all libcleri install_libcleri install clean + default: libextxyz.${dlext} all: libcleri extxyz_kv_grammar.c extxyz_kv_grammar.h libextxyz.${dlext} libcleri: if [ -z ${LIBCLERI_PATH} ]; then echo "LIBCLERI_PATH must be defined" 1>&2; exit 1; fi - ${MAKE} -C ${LIBCLERI_PATH} -f makefile + ${MAKE} -C ${LIBCLERI_PATH} -f makefile libcleri.a install_libcleri: libcleri ${MAKE} -C ${LIBCLERI_PATH} -f makefile install INSTALL_PATH=${prefix} @@ -53,11 +55,13 @@ install: libextxyz.${dlext} test_fortran_main.o: fextxyz.o extxyz.o extxyz_kv_grammar.o -fextxyz: test_fortran_main.o fextxyz.o extxyz.o extxyz_kv_grammar.o - ${F90} -g $^ -o $@ ${LIBCLERI_PATH}/libcleri.a ${LDFLAGS} ${QUIP_LDFLAGS} +FOBJS = test_fortran_main.o fextxyz.o extxyz.o extxyz_kv_grammar.o +fextxyz: libcleri ${FOBJS} + ${F90} -g ${FOBJS} -o $@ ${LIBCLERI_PATH}/libcleri.a ${LDFLAGS} ${QUIP_LDFLAGS} -cextxyz: test_C_main.o extxyz.o extxyz_kv_grammar.o - ${F90} -g $^ -o $@ ${LIBCLERI_PATH}/libcleri.a ${LDFLAGS} +COBJS = test_C_main.o extxyz.o extxyz_kv_grammar.o +cextxyz: libcleri ${COBJS} + ${F90} -g ${COBJS} -o $@ ${LIBCLERI_PATH}/libcleri.a ${LDFLAGS} clean: - rm -rf libextxyz.${dlext} *.o \ No newline at end of file + rm -rf libextxyz.${dlext} *.o diff --git a/libextxyz/meson.build b/libextxyz/meson.build new file mode 100644 index 0000000..ec1e209 --- /dev/null +++ b/libextxyz/meson.build @@ -0,0 +1,16 @@ +python3 = find_program('python3') + +run_command( + python3, + '..' / 'python' / 'extxyz' / 'extxyz_kv_grammar.py' +) + +# Build and install the extension module +module = python.extension_module( + '_extxyz', # Name of the module + ['extxyz.c', 'extxyz_kv_grammar.c'], + install: true, # Install it + subdir: 'extxyz', + gnu_symbol_visibility: 'default', # keep symbols public + dependencies: [cleri, pcre2] +) \ No newline at end of file diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..7b221e4 --- /dev/null +++ b/meson.build @@ -0,0 +1,55 @@ +# https://mesonbuild.com/ +project( + 'extxyz', # Project name + 'c', + version: run_command('python3', 'discover_version.py', check: true).stdout().strip(), # Project version +) + +# https://mesonbuild.com/Python-module.html +pymod = import('python') +python = pymod.find_installation('python3', + required: true, + pure: false +) + +host_system = host_machine.system() +cc = meson.get_compiler('c') + +# adapted from https://gitlab.gnome.org/GNOME/glib/-/blob/cd9a5c173a154e326a3ebaa28cfe41a7444625c5/meson.build#L2020-L2052 +pcre2 = dependency('libpcre2-8', version: '>= 10.23', required : false) +if not pcre2.found() + if cc.get_id() == 'msvc' or cc.get_id() == 'clang-cl' + # MSVC: Search for the PCRE2 library by the configuration, which corresponds + # to the output of CMake builds of PCRE2. Note that debugoptimized + # is really a Release build with .PDB files. + if vs_crt == 'debug' + pcre2 = cc.find_library('pcre2d-8', required : false) + else + pcre2 = cc.find_library('pcre2-8', required : false) + endif + endif +endif + +# Try again with the fallback +if not pcre2.found() + pcre2 = dependency('libpcre2-8', required : true, fallback : ['pcre2', 'libpcre2_8']) + use_pcre2_static_flag = true +elif host_system == 'windows' + pcre2_static = cc.links('''#define PCRE2_STATIC + #define PCRE2_CODE_UNIT_WIDTH 8 + #include + int main() { + void *p = NULL; + pcre2_code_free(p); + return 0; + }''', + dependencies: pcre2, + name : 'Windows system PCRE2 is a static build') + use_pcre2_static_flag = pcre2_static +else + use_pcre2_static_flag = false +endif + +subdir('libcleri') +subdir('libextxyz') +subdir('python/extxyz') \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a55b09a..18d90eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,31 @@ [build-system] -# Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel", "pyleri>=1.3.3"] -build-backend = "setuptools.build_meta" +requires = ["meson>=1.0.0", "meson-python>=0.13.0", "ninja", "pyleri>=1.3.3", "oldest-supported-numpy"] +build-backend = "mesonpy" + +[project] +name = "extxyz" +description = "Extended XYZ file format tools" +readme = "README.md" +license = { file = "LICENSE" } +authors = [ + { name = "James Kermode", email = "james.kermode@gmail.com"}, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python" +] +requires-python = ">=3.7.0" +dynamic = [ "version" ] +dependencies = [ + "numpy>=1.13.0", 'pyleri>=1.3.3', 'ase>=3.17' +] + +[project.optional-dependencies] +test = [ + "pytest", + +] + +[project.urls] +documentation = "http://libatoms.github.io/extxyz/" +repository = "https://github.com/libAtoms/extxyz" diff --git a/python/extxyz/_version.py.in b/python/extxyz/_version.py.in new file mode 100644 index 0000000..ed0561c --- /dev/null +++ b/python/extxyz/_version.py.in @@ -0,0 +1,27 @@ +# +# Copyright 2017 Lars Pastewka (U. Freiburg) +# +# matscipy - Materials science with Python at the atomic-scale +# https://github.com/libAtoms/matscipy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +__version__ = '@VERSION@' diff --git a/python/extxyz/cextxyz.py b/python/extxyz/cextxyz.py index ae94866..4af3293 100644 --- a/python/extxyz/cextxyz.py +++ b/python/extxyz/cextxyz.py @@ -1,6 +1,7 @@ import os import ctypes from ctypes.util import find_library +from distutils import sysconfig import copy @@ -42,9 +43,8 @@ class Dict_entry_struct(ctypes.Structure): Dict_entry_ptr = ctypes.POINTER(Dict_entry_struct) -# _extxyz.so is actually created as a python extension, but custom builder is -# used to make the name just _extxyz.so, rather than _extxyz.cpython--.so -extxyz_so = os.path.join(os.path.abspath(os.path.dirname(__file__)), '_extxyz.so') +suffix = sysconfig.get_config_var('EXT_SUFFIX') +extxyz_so = os.path.join(os.path.abspath(os.path.dirname(__file__)), f'_extxyz{suffix}') extxyz = ctypes.CDLL(extxyz_so) extxyz.compile_extxyz_kv_grammar.restype = cleri_grammar_t_ptr diff --git a/python/extxyz/meson.build b/python/extxyz/meson.build new file mode 100644 index 0000000..8fc4607 --- /dev/null +++ b/python/extxyz/meson.build @@ -0,0 +1,25 @@ +conf_data = configuration_data() +conf_data.set('VERSION', meson.project_version()) +version_file = configure_file( + input: '_version.py.in', + output: '_version.py', + configuration: conf_data +) + +# Pure Python sources +python_sources = [ + '__init__.py', + '__main__.py', + version_file, + 'cli.py', + 'extxyz.py', + 'cextxyz.py', + 'extxyz_kv_grammar.py', + 'utils.py' +] + +# Install pure Python +python.install_sources( + python_sources, + subdir: 'extxyz' +) \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 2eea0f9..0000000 --- a/setup.py +++ /dev/null @@ -1,139 +0,0 @@ -import sys -import tempfile -import atexit -import shutil -import sysconfig -import pathlib -import os -import subprocess -from setuptools import setup, Extension -# as per https://stackoverflow.com/questions/19569557/pip-not-picking-up-a-custom-install-cmdclass -from setuptools.command.install import install as setuptools__install -from setuptools.command.develop import develop as setuptools__develop -from setuptools.command.egg_info import egg_info as setuptools__egg_info -from setuptools.command.build_ext import build_ext as setuptools__build_ext - -def build_grammar(): - # check if we need to run the grammar definition to regenerate .c and .h - py_grammar_file = pathlib.Path('./grammar/extxyz_kv_grammar.py') - c_grammar_file = pathlib.Path('./libext/extxyz_kv_grammar.c') - - if not c_grammar_file.exists() or (py_grammar_file.stat().st_mtime > c_grammar_file.stat().st_mtime): - sys.path.insert(0, './python/extxyz') - import extxyz_kv_grammar - del sys.path[0] - extxyz_kv_grammar.write_grammar('./libextxyz') - -def which(program): - import os - - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -def build_pcre2(): - pcre2_config = which('pcre2-config') - print(f'which(pcre2-config) = {pcre2_config}') - if pcre2_config is None: - pcre2_version = '10.42' - download_url = f"https://github.com/PCRE2Project/pcre2/releases/download/pcre2-{pcre2_version}/pcre2-{pcre2_version}.tar.gz" - print(f'pcre2-config not found so downloading and installing PCRE2-{pcre2_version} from {download_url}') - - tempdir = tempfile.mkdtemp() - # atexit.register(lambda: shutil.rmtree(tempdir)) # cleanup tempdir when Python exits - build_dir = os.path.abspath(f"{tempdir}/pcre2-{pcre2_version}/build") - pcre2_config = os.path.join(build_dir, 'bin', 'pcre2-config') - - orig_dir = os.getcwd() - os.chdir(tempdir) - try: - subprocess.call(["curl", "-L", download_url, "-o", "pcre2.tar.gz"]) - subprocess.call(["tar", "xvzf", "pcre2.tar.gz"]) - subprocess.call(["./configure", f"--prefix={build_dir}"], cwd=f"pcre2-{pcre2_version}") - subprocess.call("make", cwd=f"pcre2-{pcre2_version}") - subprocess.call(["make", "install"], cwd=f"pcre2-{pcre2_version}") - finally: - os.chdir(orig_dir) - - pcre2_cflags = subprocess.check_output([f'{pcre2_config}', '--cflags'], encoding='utf-8').strip().split() - pcre2_include_dirs = [i.replace('-I', '', 1) for i in pcre2_cflags if i.startswith('-I')] - # should we also capture other flags to pass to extra_compile_flags? - - pcre2_libs = subprocess.check_output([f'{pcre2_config}', '--libs8'], encoding='utf-8').strip().split() - pcre2_library_dirs = [l.replace('-L', '', 1) for l in pcre2_libs if l.startswith('-L')] - pcre2_libraries = [l.replace('-l', '', 1) for l in pcre2_libs if l.startswith('-l')] - - return pcre2_cflags, pcre2_include_dirs, pcre2_library_dirs, pcre2_libraries - - -def build_libcleri(pcre2_cflags): - with open('libcleri/Release/makefile', 'r') as f_in, open('libcleri/Release/makefile.extxyz', 'w') as f_out: - contents = f_in.read() - contents += """ - -libcleri.a: $(OBJS) $(USER_OBJS) -\tar rcs libcleri.a $(OBJS) $(USER_OBJS) -""" - f_out.write(contents) - env = os.environ.copy() - env['CFLAGS'] = ' '.join(pcre2_cflags) - subprocess.call(['make', '-C', 'libcleri/Release', '-f', 'makefile.extxyz', 'libcleri.a'], env=env) - -class install(setuptools__install): - def run(self): - build_libcleri(pcre2_cflags) - setuptools__install.run(self) - - -class develop(setuptools__develop): - def run(self): - build_libcleri(pcre2_cflags) - setuptools__develop.run(self) - - -class egg_info(setuptools__egg_info): - def run(self): - build_libcleri(pcre2_cflags) - setuptools__egg_info.run(self) - -# https://stackoverflow.com/questions/60284403/change-output-filename-in-setup-py-distutils-extension -class NoSuffixBuilder(setuptools__build_ext): - def get_ext_filename(self, ext_name): - filename = super().get_ext_filename(ext_name) - suffix = sysconfig.get_config_var('EXT_SUFFIX') - ext = os.path.splitext(filename)[1] - return filename.replace(suffix, "") + ext - -pcre2_cflags, pcre2_include_dirs, pcre2_library_dirs, pcre2_libraries = build_pcre2() - -_extxyz_ext = Extension('extxyz._extxyz', sources=['libextxyz/extxyz_kv_grammar.c', 'libextxyz/extxyz.c'], - include_dirs=['libcleri/inc', 'extxyz'] + pcre2_include_dirs, - library_dirs=pcre2_library_dirs, libraries=pcre2_libraries, - extra_compile_args=['-fPIC'], extra_objects=['libcleri/Release/libcleri.a']) - -build_grammar() - -setup( - name='extxyz', - version='0.1.3', - author='various', - packages=['extxyz'], - package_dir={'': 'python'}, - cmdclass={'install': install, 'develop': develop, 'egg_info': egg_info, 'build_ext': NoSuffixBuilder}, - include_package_data=True, - install_requires=['numpy>=1.13', 'pyleri>=1.3.3', 'ase>=3.17'], - ext_modules=[_extxyz_ext], - entry_points={'console_scripts': ['extxyz=extxyz.cli:main']} -) -