Merge pull request #14 from UIUCLibrary/dev

Dev

.gitignore

@@ -0,0 +1,73 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+.idea/sonarlint
+
+# Sensitive or high-churn files:
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+venv
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+uiucprescon.build.egg-info/
+.mypy_cache/
+.pytest_cache/
+dist/
+buildme/
+.conan.db
+conan/
+
+reports/
+
+/.idea/inspectionProfiles/
+/.idea/uiucprescon_build.iml
+/.idea/misc.xml
+/.idea/aws.xml
+/.idea/modules.xml
+
+# old requirements files
+requirements-*.txt.old

Jenkinsfile

@@ -1,3 +1,88 @@
+def getMacToxTestsParallel(args = [:]){
+    def nodeLabel = args['label']
+    args.remove('label')
+
+    def envNamePrefix = args['envNamePrefix']
+    args.remove('envNamePrefix')
+
+    def retries = args.get('retry', 1)
+    if(args.containsKey('retry')){
+        args.remove('retry')
+    }
+    if(args.size() > 0){
+        error "getMacToxTestsParallel has invalid arguments ${args.keySet()}"
+    }
+
+    //    =============================================
+    def envs = [:]
+    node(nodeLabel){
+        try{
+            checkout scm
+            sh(
+                script: '''python3 -m venv venv --upgrade-deps
+                    venv/bin/pip install tox
+                    '''
+            )
+            def toxEnvs = sh(
+                    label: 'Getting Tox Environments',
+                    returnStdout: true,
+                    script: 'venv/bin/tox list -d --no-desc'
+                ).trim().split('\n')
+            toxEnvs.each({env ->
+                def requiredPythonVersion = sh(
+                    label: "Getting required python version for Tox Environment: ${env}",
+                    script: "venv/bin/tox config -e  ${env} -k py_dot_ver  | grep  'py_dot_ver =' | sed -E 's/py_dot_ver = ([0-9].[0-9]+)/\\1/g'",
+                    returnStdout: true
+                ).trim()
+                envs[env] = requiredPythonVersion
+            })
+
+
+        } finally {
+            sh 'rm -rf venv'
+        }
+    }
+    echo "Found tox environments for ${envs.keySet().join(', ')}"
+    def jobs = envs.collectEntries({ toxEnv, requiredPythonVersion ->
+        def jenkinsStageName = "${envNamePrefix} ${toxEnv}"
+        def jobNodeLabel = "${nodeLabel} && python${requiredPythonVersion}"
+        if(nodesByLabel(jobNodeLabel).size() > 0){
+            [jenkinsStageName,{
+                retry(retries){
+                    node(jobNodeLabel){
+                        try{
+                            checkout scm
+                            sh(
+                                script: '''python3 -m venv venv --upgrade-deps
+                                    venv/bin/pip install tox
+                                    '''
+                            )
+                            sh(
+                                label: 'Getting Tox Environments',
+                                script: "venv/bin/tox --list-dependencies --workdir=.tox run -e ${toxEnv}"
+                                )
+
+                        } finally {
+                            cleanWs(
+                                notFailBuild: true,
+                                deleteDirs: true,
+                                patterns: [
+                                    [pattern: '**/__pycache__/', type: 'INCLUDE'],
+                                    [pattern: 'venv/', type: 'INCLUDE'],
+                                    [pattern: '.tox/', type: 'INCLUDE'],
+                                ]
+                            )
+                        }
+                    }
+                }
+
+            }]
+        } else {
+            echo "Unable to add ${toxEnv} because no nodes with required labels: ${jobNodeLabel}"
+        }
+    })
+    return jobs
+}
 pipeline {
     agent none
     options {
@@ -58,6 +143,23 @@ pipeline {
                         }
                         stage('Run Checks'){
                             parallel{
+                                stage('pyDocStyle'){
+                                    steps{
+                                        catchError(buildResult: 'SUCCESS', message: 'Did not pass all pyDocStyle tests', stageResult: 'UNSTABLE') {
+                                            sh(
+                                                label: 'Run pydocstyle',
+                                                script: '''mkdir -p reports
+                                                           pydocstyle uiucprescon > reports/pydocstyle-report.txt
+                                                           '''
+                                            )
+                                        }
+                                    }
+                                    post {
+                                        always{
+                                            recordIssues(tools: [pyDocStyle(pattern: 'reports/pydocstyle-report.txt')])
+                                        }
+                                    }
+                                }
                                 stage('PyTest'){
                                     steps{
                                         catchError(buildResult: 'UNSTABLE', message: 'Did not pass all pytest tests', stageResult: 'UNSTABLE') {
@@ -131,7 +233,6 @@ pipeline {
                                     sh(label: 'combining coverage data',
                                        script: '''coverage combine
                                                   coverage xml -o ./reports/coverage-python.xml
-                                                  sed -i 's/uiucprescon\\/build\\///' reports/coverage-python.xml
                                                   '''
                                     )
                                     archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/coverage-python.xml'
@@ -171,24 +272,46 @@ pipeline {
                                                         ''
                                                         )
                             def linuxJobs = [:]
+
                             def windowsJobs = [:]
-                            linuxJobs = tox.getToxTestsParallel(
-                                envNamePrefix: 'Tox Linux',
-                                label: 'linux && docker',
-                                dockerfile: 'ci/docker/linux/tox/Dockerfile',
-                                dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL',
-                                dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip",
-                                retry: 2
-                            )
-                            windowsJobs = tox.getToxTestsParallel(
-                                envNamePrefix: 'Tox Windows',
-                                label: 'windows && docker && x86',
-                                dockerfile: 'ci/docker/windows/tox/Dockerfile',
-                                dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE',
-                                dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip",
-                                retry: 2
-                            )
-                            parallel(linuxJobs + windowsJobs)
+                            def macJobs = [:]
+                            parallel(
+                                'Tox Information Gathering For: Linux': {
+                                    if(nodesByLabel("linux && docker").size() > 0){
+                                        linuxJobs = tox.getToxTestsParallel(
+                                            envNamePrefix: 'Tox Linux',
+                                            label: 'linux && docker',
+                                            dockerfile: 'ci/docker/linux/tox/Dockerfile',
+                                            dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL',
+                                            dockerRunArgs: "-v pipcache_uiucprescon_build:/.cache/pip",
+                                            retry: 2
+                                        )
+                                    } else {
+                                        echo 'No nodes with the following labels: linux && docker labels'
+                                    }
+                                },
+                                'Tox Information Gathering For: Windows': {
+                                    if(nodesByLabel('windows && docker').size() > 0){
+                                        windowsJobs = tox.getToxTestsParallel(
+                                            envNamePrefix: 'Tox Windows',
+                                            label: 'windows && docker',
+                                            dockerfile: 'ci/docker/windows/tox/Dockerfile',
+                                            dockerArgs: '--build-arg PIP_EXTRA_INDEX_URL --build-arg PIP_INDEX_URL --build-arg CHOCOLATEY_SOURCE',
+                                            dockerRunArgs: "-v pipcache_uiucprescon_build:c:/users/containeradministrator/appdata/local/pip",
+                                            retry: 2
+                                        )
+                                    } else {
+                                        echo 'No nodes with the following labels: windows && docker && x86'
+                                    }
+                                },
+                                'Tox Information Gathering For: mac': {
+                                    if(nodesByLabel('mac && python3').size() > 0){
+                                        macJobs = macJobs + getMacToxTestsParallel(label: 'mac && python3', envNamePrefix: 'mac')
+                                    }
+                                },
+                             )
+
+                            parallel(linuxJobs + windowsJobs + macJobs)
                         }
                     }
 //                     post{

ci/docker/linux/tox/Dockerfile

@@ -1,3 +1,4 @@
+ARG PIPX_HOME=/pipx
 ARG CONAN_USER_HOME=/conan
 ARG PIP_DOWNLOAD_CACHE=/.cache/pip
 FROM ubuntu:22.04
@@ -30,5 +31,16 @@ RUN conan config init && \
     python3 /tmp/update_conan_compiler.py ${CONAN_USER_HOME}/.conan/settings.yml gcc $(cc -dumpfullversion -dumpversion | grep -oE  "^([0-9]+(\.)?)([0-9]+?)")
 ENV CONAN_USER_HOME=${CONAN_USER_HOME}
 
+
+ARG PIPX_HOME
+ENV PIPX_HOME=${PIPX_HOME}
+ENV PIPX_BIN_DIR=${PIPX_HOME}/bin
+RUN python3 -m pip install --no-cache-dir pipx && \
+    pipx ensurepath && \
+    mkdir -p $PIPX_HOME  && chmod -R 777 $PIPX_HOME
+
+RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install 'devpi-client<7.0'
+
+
 WORKDIR /src
 CMD tox --workdir /tmp/tox --recreate

ci/docker/windows/tox/Dockerfile

@@ -1,4 +1,5 @@
 # escape=`
+ARG PIPX_HOME=c:\pipx
 ARG VS_INSTALL_PATH="C:\BuildTools\"
 ARG PIP_EXTRA_INDEX_URL
 ARG PIP_INDEX_URL
@@ -121,6 +122,16 @@ COPY requirements/ c:/python_requirements/requirements
 RUN python -m pip install pip --upgrade ; `
     pip install wheel ; `
     pip install -r c:\python_requirements\requirements-ci.txt
+
+ARG PIPX_HOME
+ENV PIPX_HOME=${PIPX_HOME}
+ENV PIPX_BIN_DIR=${PIPX_HOME}\bin
+
+
+RUN py -3 -m pip install --no-cache-dir pipx ; `
+    py -3 -m pipx ensurepath
+RUN py -3 -m pipx install 'devpi-client<7.0'
+
 ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV PIP_INDEX_URL=${PIP_INDEX_URL}
 WORKDIR C:/src

pyproject.toml

@@ -17,17 +17,21 @@ dependencies = [
     'wheel',
     "cmake",
     'pybind11~=2.10.1',
-    "conan>=1.53,!=1.55.0,<2.0",
+    "conan",
     'toml'
 ]
 license = {text = "University of Illinois/NCSA Open Source License"}
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 [tool.setuptools.dynamic]
 version = {attr = "uiucprescon.build.VERSION"}
 
 [tool.setuptools]
-packages = ['uiucprescon.build']
+packages = [
+    'uiucprescon.build',
+    'uiucprescon.build.conan',
+]
+
 
 [tool.coverage.run]
 relative_files = false
@@ -46,3 +50,16 @@ include_namespace_packages= true
 [tool.coverage.paths]
 
 #[tool.coverage.report]
+
+
+[tool.pylint.'MESSAGES CONTROL']
+# docstrings linting should be handled by pydocstyle instead of pylint
+#disable = """
+#    missing-module-docstring,
+#    missing-function-docstring
+#"""
+disable = [
+    "missing-module-docstring",
+    "missing-class-docstring",
+    "missing-function-docstring"
+]

requirements-ci.txt

@@ -1,4 +1,5 @@
 -r requirements/requirements.txt
 -r requirements/requirements-dev.txt
+-r requirements/requirements-tox.txt
 -r requirements/requirements-ci.txt
 -r requirements/requirements-ci-freeze.txt