From 422beda4d05c6954d10b60aa15ee9dc7ec8e6754 Mon Sep 17 00:00:00 2001 From: Ben Webb Date: Tue, 5 Mar 2024 15:31:04 -0800 Subject: [PATCH] Squashed 'tools/dev_tools/' changes from c4337f5b..3ad5e0d8 3ad5e0d8 Drop testing with Python 2.7, add 3.12 17206759 Switch to argparse 9778d818 Switch to argparse 122d6950 Switch to argparse 69c73496 Use codecov token to avoid rate limits 5ae13679 Stick with older codecov action for now 43582234 Use latest codecov GitHub Action 7e8e4043 Use current year for copyright of autogen files 8164076f Run flake8 separately, not as part of pytest 84af9307 flake8 fix fb3f9d03 Test with Python 3.11 626e51c1 Use Python 3 by default git-subtree-dir: tools/dev_tools git-subtree-split: 3ad5e0d84c668cd06deebfe414d8e16820116335 --- tools/dev_tools/.github/workflows/build.yml | 9 ++- tools/dev_tools/check_standards.py | 13 +-- tools/dev_tools/cleanup_code.py | 90 +++++++++++---------- tools/dev_tools/git/bootstrap_setup_git.py | 2 +- tools/dev_tools/git/setup_git.py | 18 ++--- tools/dev_tools/make_all_header.py | 6 +- tools/dev_tools/pytest.ini | 3 - tools/dev_tools/python_tools/__init__.py | 4 +- tools/dev_tools/test/test_header.py | 6 +- 9 files changed, 80 insertions(+), 71 deletions(-) delete mode 100644 tools/dev_tools/pytest.ini diff --git a/tools/dev_tools/.github/workflows/build.yml b/tools/dev_tools/.github/workflows/build.yml index d7190075..e32144d4 100644 --- a/tools/dev_tools/.github/workflows/build.yml +++ b/tools/dev_tools/.github/workflows/build.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: [2.7, 3.7, 3.8, 3.9, "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] runs-on: ${{ matrix.os }} steps: @@ -20,8 +20,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install 'pytest-flake8<1.1' pytest-cov + pip install pytest flake8 pytest-cov - name: Test run: | - py.test --cov=. --cov-branch --cov-report=xml -v --flake8 . + py.test --cov=. --cov-branch --cov-report=xml -v . + flake8 --ignore=E402,W503 --exclude python_tools/reindent.py - uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/tools/dev_tools/check_standards.py b/tools/dev_tools/check_standards.py index ddda6c0e..036b3a02 100755 --- a/tools/dev_tools/check_standards.py +++ b/tools/dev_tools/check_standards.py @@ -18,7 +18,7 @@ print("Pygments (http://pygments.org/) library: please install.") print() -from optparse import OptionParser +from argparse import ArgumentParser def _check_do_not_commit(line, filename, num, errors): @@ -187,15 +187,18 @@ def get_all_files(): def main(): - parser = OptionParser() - options, args = parser.parse_args() + parser = ArgumentParser() + parser.add_argument("patterns", metavar="PATTERN", nargs="*", + help="File patterns to check; if unspecified, " + "check all files") + args = parser.parse_args() errors = [] - if len(args) == 0: + if len(args.patterns) == 0: modfiles = get_all_files() print("usage:", sys.argv[0], "file_patterns") else: - modfiles = args + modfiles = args.patterns for pattern in modfiles: expanded = glob.glob(pattern) # rint pattern, expanded diff --git a/tools/dev_tools/cleanup_code.py b/tools/dev_tools/cleanup_code.py index 57098b8f..dd42118b 100755 --- a/tools/dev_tools/cleanup_code.py +++ b/tools/dev_tools/cleanup_code.py @@ -4,7 +4,7 @@ files.""" from __future__ import print_function -from optparse import OptionParser +from argparse import ArgumentParser import subprocess import os import sys @@ -24,8 +24,8 @@ from python_tools.reindent import Reindenter -parser = OptionParser(usage="%prog [options] [FILENAME ...]", - description="""Reformat the given C++ and Python files +parser = ArgumentParser( + description="""Reformat the given C++ and Python files (using the clang-format tool if available and reindent.py, respectively). If the --all option is given, reformat all such files under the current directory. @@ -34,46 +34,48 @@ by giving the -a option. autopep8 is much more aggressive than reindent.py and will fix other issues, such as use of old-style Python syntax. """) -parser.add_option("-c", "--clang-format", dest="clang_format", - default="auto", metavar="EXE", - help="The clang-format command.") -parser.add_option("-a", dest="use_ap", action="store_true", default=False, - help="Use autopep8 rather than reindent.py for " - "Python files.") -parser.add_option("--all", dest="all_files", action="store_true", - default=False, - help="Reformat all files under current directory") -parser.add_option("--autopep8", dest="autopep8", - default="auto", metavar="EXE", - help="The autopep8 command.") -parser.add_option("-e", "--exclude", dest="exclude", - default="eigen3:config_templates", metavar="DIRS", - help="Colon-separated list of dirnames to ignore.") -parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - default=False, - help="Print extra info.") -(options, args) = parser.parse_args() -if not args and not options.all_files: +parser.add_argument("-c", "--clang-format", dest="clang_format", + default="auto", metavar="EXE", + help="The clang-format command.") +parser.add_argument("-a", dest="use_ap", action="store_true", default=False, + help="Use autopep8 rather than reindent.py for " + "Python files.") +parser.add_argument("--all", dest="all_files", action="store_true", + default=False, + help="Reformat all files under current directory") +parser.add_argument("--autopep8", dest="autopep8", + default="auto", metavar="EXE", + help="The autopep8 command.") +parser.add_argument("-e", "--exclude", dest="exclude", + default="eigen3:config_templates", metavar="DIRS", + help="Colon-separated list of dirnames to ignore.") +parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + default=False, + help="Print extra info.") +parser.add_argument("files", metavar="FILENAME", nargs="*", + help="C++ and Python files to reformat.") +args = parser.parse_args() +if not args.files and not args.all_files: parser.error("No files selected") # clang-format-3.4", # autopep8 # search for executables -if options.clang_format == "auto": - options.clang_format = None +if args.clang_format == "auto": + args.clang_format = None for name in ["clang-format-3.4", "clang-format"]: if which(name): - options.clang_format = name + args.clang_format = name break -if options.autopep8 == "auto": - options.autopep8 = None +if args.autopep8 == "auto": + args.autopep8 = None for name in ["autopep8"]: if which(name): - options.autopep8 = name + args.autopep8 = name break -exclude = options.exclude.split(":") +exclude = args.exclude.split(":") error = None @@ -137,10 +139,10 @@ def _do_get_files(glb, cur): def _get_files(glb): match = [] - if len(args) == 0: + if len(args.files) == 0: match = _do_get_files(glb, ".") else: - for a in args: + for a in args.files: if os.path.isdir(a): match += _do_get_files(glb, a) elif a.endswith(glb): @@ -163,8 +165,8 @@ def clean_cpp(path): # skip code that isn't ours if "dependency" in path or "/eigen3/" in path: return - if options.clang_format: - contents = _run([options.clang_format, "--style=Google", path]) + if args.clang_format: + contents = _run([args.clang_format, "--style=Google", path]) else: contents = open(path, "r").read() contents = contents.replace("% template", "%template") @@ -172,8 +174,8 @@ def clean_cpp(path): def clean_py(path): - if options.use_ap and options.autopep8: - contents = _run([options.autopep8, "--aggressive", "--aggressive", + if args.use_ap and args.autopep8: + contents = _run([args.autopep8, "--aggressive", "--aggressive", path]) else: r = Reindenter(open(path)) @@ -185,25 +187,25 @@ def clean_py(path): def main(): - if options.verbose: - if options.autopep8 is None: + if args.verbose: + if args.autopep8 is None: print("autopep8 not found") else: - print("autopep8 is `%s`" % options.autopep8) - if options.clang_format is None: + print("autopep8 is `%s`" % args.autopep8) + if args.clang_format is None: print("clang-format not found") else: - print("clang-format is `%s`" % options.clang_format) + print("clang-format is `%s`" % args.clang_format) tp = ThreadPool() - if args: - for f in args: + if args.files: + for f in args.files: if f.endswith(".py"): tp.add_task(clean_py, f) elif f.endswith(".h") or f.endswith(".cpp"): tp.add_task(clean_cpp, f) - elif options.all_files: + elif args.all_files: for f in _get_files(".py"): tp.add_task(clean_py, f) for f in _get_files(".h") + _get_files(".cpp"): diff --git a/tools/dev_tools/git/bootstrap_setup_git.py b/tools/dev_tools/git/bootstrap_setup_git.py index 1f1a1fa9..cd553ebc 100755 --- a/tools/dev_tools/git/bootstrap_setup_git.py +++ b/tools/dev_tools/git/bootstrap_setup_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Update submodules and then call the main setup_git.py. This should be copied to the main directory of your project and named setup_git.py.""" diff --git a/tools/dev_tools/git/setup_git.py b/tools/dev_tools/git/setup_git.py index 0fbbbea0..6821d3ae 100755 --- a/tools/dev_tools/git/setup_git.py +++ b/tools/dev_tools/git/setup_git.py @@ -1,24 +1,24 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import os.path import subprocess import glob -from optparse import OptionParser +from argparse import ArgumentParser import shutil dev_tools_path = os.path.join("tools", "dev_tools") -opt = OptionParser() -opt.add_option("-g", "--global", - action="store_true", dest="glob", default=False, - help="Set global git settings instead of repo " - "settings [default]") +opt = ArgumentParser() +opt.add_argument("-g", "--global", + action="store_true", dest="glob", default=False, + help="Set global git settings instead of repo " + "settings [default]") -(options, args) = opt.parse_args() +args = opt.parse_args() -if options.glob: +if args.glob: git_config = "git config --global --replace-all" config_contents = "" else: diff --git a/tools/dev_tools/make_all_header.py b/tools/dev_tools/make_all_header.py index ec9af48a..b10deee0 100755 --- a/tools/dev_tools/make_all_header.py +++ b/tools/dev_tools/make_all_header.py @@ -6,6 +6,7 @@ import sys import glob +import datetime import os sys.path.append(os.path.split(sys.argv[0])[0]) @@ -27,13 +28,14 @@ def _add_includes(headers, output): includepath = sys.argv[1][sys.argv[1].find("include") + len("include") + 1:] +year = datetime.datetime.now().year output = ["""/** * \\file %s * \\brief Include all non-deprecated headers in %s. * - * Copyright 2007-2022 IMP Inventors. All rights reserved. + * Copyright 2007-%d IMP Inventors. All rights reserved. */ -""" % (includepath, includepath[:-2].replace('/', '.'))] +""" % (includepath, includepath[:-2].replace('/', '.'), year)] guard = includepath.replace( "/", "_").replace("\\", diff --git a/tools/dev_tools/pytest.ini b/tools/dev_tools/pytest.ini deleted file mode 100644 index aec0cb16..00000000 --- a/tools/dev_tools/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -flake8-ignore = E402 W503 -addopts = --ignore=python_tools/reindent.py diff --git a/tools/dev_tools/python_tools/__init__.py b/tools/dev_tools/python_tools/__init__.py index 17fd6027..aecd175f 100644 --- a/tools/dev_tools/python_tools/__init__.py +++ b/tools/dev_tools/python_tools/__init__.py @@ -174,8 +174,8 @@ def get_modules(source): def split(string, sep=":"): - return([x.replace("@", ":") - for x in string.replace("\\:", "@").split(sep) if x != ""]) + return ([x.replace("@", ":") + for x in string.replace("\\:", "@").split(sep) if x != ""]) def get_project_info(path): diff --git a/tools/dev_tools/test/test_header.py b/tools/dev_tools/test/test_header.py index a00c32bd..ef3f7b06 100644 --- a/tools/dev_tools/test/test_header.py +++ b/tools/dev_tools/test/test_header.py @@ -1,6 +1,7 @@ import unittest import subprocess import os +import datetime import utils TOPDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -26,13 +27,14 @@ def test_header(self): 'include/test/subdir'], cwd=tmpdir) stdout, stderr = p.communicate() self.assertEqual(p.returncode, 0) + year = datetime.datetime.now().year self.assertEqual(utils.read_file(os.path.join(tmpdir, 'include/test.h')), """/** * \\file test.h * \\brief Include all non-deprecated headers in test. * - * Copyright 2007-2022 IMP Inventors. All rights reserved. + * Copyright 2007-%d IMP Inventors. All rights reserved. */ #ifndef TEST_H @@ -43,7 +45,7 @@ def test_header(self): #include #endif #endif /* TEST_H */ -""") +""" % year) if __name__ == '__main__':