From b9dc351ade983eab55ef64a7856c20e5af4335dc Mon Sep 17 00:00:00 2001 From: Jose Luis Blanco-Claraco Date: Tue, 22 Aug 2023 23:46:43 +0200 Subject: [PATCH] Add GitHub actions CI --- .github/check_style.sh | 81 +++ .github/python_clang_format_reqs.txt | 3 + .github/workflows/build-ros.yml | 76 +++ .github/workflows/check-clang-format.yml | 40 ++ README.md | 2 + scripts/clang_git_format/.gitignore | 40 ++ scripts/clang_git_format/LICENSE | 202 ++++++ scripts/clang_git_format/README.md | 116 ++++ .../clang_git_format/__init__.py | 10 + .../clang_git_format/clang_format.py | 159 +++++ .../clang_git_format/config.py | 2 + .../clang_git_format/custom_exceptions.py | 38 ++ .../clang_git_format/clang_git_format/repo.py | 343 +++++++++++ .../clang_git_format/utils.py | 80 +++ scripts/clang_git_format/format_code.py | 578 ++++++++++++++++++ 15 files changed, 1770 insertions(+) create mode 100755 .github/check_style.sh create mode 100644 .github/python_clang_format_reqs.txt create mode 100644 .github/workflows/build-ros.yml create mode 100644 .github/workflows/check-clang-format.yml create mode 100644 scripts/clang_git_format/.gitignore create mode 100644 scripts/clang_git_format/LICENSE create mode 100644 scripts/clang_git_format/README.md create mode 100644 scripts/clang_git_format/clang_git_format/__init__.py create mode 100644 scripts/clang_git_format/clang_git_format/clang_format.py create mode 100644 scripts/clang_git_format/clang_git_format/config.py create mode 100644 scripts/clang_git_format/clang_git_format/custom_exceptions.py create mode 100644 scripts/clang_git_format/clang_git_format/repo.py create mode 100644 scripts/clang_git_format/clang_git_format/utils.py create mode 100755 scripts/clang_git_format/format_code.py diff --git a/.github/check_style.sh b/.github/check_style.sh new file mode 100755 index 00000000..c4fb847d --- /dev/null +++ b/.github/check_style.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Given a set of commits current script checks the style based on clang_format +# on the list of changed files that exist in the commits. +# +# Script should be called from a travis.yml file. + +# Functions +###################### + +# This is run by travis automatically from the root of the project + +set +e +set +x + +DIRS_IN="modules" +DIRS_OUT="demos docs modules/mp2p_icp/3rdparty" +LANGS=cpp +FORMAT_CODE_BIN="scripts/clang_git_format/format_code.py" + +function lint() { + +# Get list of changed files for lint to run on. Grep only .h, .cpp files and +# mind the DIRS_IN, DIRS_OUT vars + +# Following command fails if you "git commit --amend" - Works correctly on +# standard commits +echo "commit_range: $TRAVIS_COMMIT_RANGE" +changed_files=$(git diff --name-only $TRAVIS_COMMIT_RANGE) +printf "Summary of changed files:\n\n${changed_files}\n\n" + +# include +regexp_in= +for i in $DIRS_IN; do + if [ -n "${regexp_in}" ]; then + regexp_in="${regexp_in}\|${i}" + else + regexp_in="${i}" + fi +done + +# exclude +regexp_out= +for i in $DIRS_OUT; do + if [ -n "${regexp_out}" ]; then + regexp_out="${regexp_out}\|${i}" + else + regexp_out="${i}" + fi +done + +echo "regexp_in: ${regexp_in}" +echo "regexp_out: ${regexp_out}" + +valid_files=$(echo ${changed_files} \ + | xargs -d' ' -n 1 \ + | grep -i -e "${regexp_in}" \ + | grep -v "${regexp_out}" \ + | grep -ie ".*\.h$\|.*\.cpp") + + +printf "Valid files for lint:\n\t${valid_files}\n" +if [ -n "${valid_files}" ]; then + ${FORMAT_CODE_BIN} -g . --lint_files ${valid_files} +else + true +fi +exit $? + +} + +function lint_all() { + +${FORMAT_CODE_BIN} -g . --lang ${LANGS} \ + -o ${DIRS_OUT} -i ${DIRS_IN} \ + -l + +exit $? +} + +lint_all diff --git a/.github/python_clang_format_reqs.txt b/.github/python_clang_format_reqs.txt new file mode 100644 index 00000000..302af969 --- /dev/null +++ b/.github/python_clang_format_reqs.txt @@ -0,0 +1,3 @@ +# Requisites for the clang-format linter script +argparse +colorlog diff --git a/.github/workflows/build-ros.yml b/.github/workflows/build-ros.yml new file mode 100644 index 00000000..24c29452 --- /dev/null +++ b/.github/workflows/build-ros.yml @@ -0,0 +1,76 @@ +# Based on GTSAM file (by @ProfFan) +name: CI ROS + +on: [push, pull_request] + +jobs: + test_docker: # On Linux, iterates on all ROS 1 and ROS 2 distributions. + runs-on: ubuntu-latest + env: + DEBIAN_FRONTEND: noninteractive + strategy: + matrix: + # Github Actions requires a single row to be added to the build matrix. + # See https://help.github.com/en/articles/workflow-syntax-for-github-actions. + ros_distribution: + - noetic + - humble + - iron + - rolling + + # Define the Docker image(s) associated with each ROS distribution. + # The include syntax allows additional variables to be defined, like + # docker_image in this case. See documentation: + # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-configurations-in-a-matrix-build + # + # Platforms are defined in REP 3 and REP 2000: + # https://ros.org/reps/rep-0003.html + # https://ros.org/reps/rep-2000.html + include: + # Noetic Ninjemys (May 2020 - May 2025) + - docker_image: ubuntu:focal + ros_distribution: noetic + ros_version: 1 + + # Humble Hawksbill (May 2022 - May 2027) + - docker_image: ubuntu:jammy + ros_distribution: humble + ros_version: 2 + + # Iron Irwini (May 2023 - November 2024) + - docker_image: ubuntu:jammy + ros_distribution: iron + ros_version: 2 + + # Rolling Ridley (No End-Of-Life) + - docker_image: ubuntu:jammy + ros_distribution: rolling + ros_version: 2 + + container: + image: ${{ matrix.docker_image }} + + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Checkout submodules + run: | + apt-get -y update + apt-get -y install git + git submodule update --init --recursive + + - name: setup ROS environment + uses: ros-tooling/setup-ros@v0.7 + with: + required-ros-distributions: ${{ matrix.ros_distribution }} + + - name: Install rosdep dependencies + run: | + # See bug/issue: https://github.com/orgs/community/discussions/47863 + rosdep install --from-paths . --ignore-src -r -y + + - name: Build with colcon + run: | + source /opt/ros/*/setup.bash + colcon build --symlink-install diff --git a/.github/workflows/check-clang-format.yml b/.github/workflows/check-clang-format.yml new file mode 100644 index 00000000..8f024af3 --- /dev/null +++ b/.github/workflows/check-clang-format.yml @@ -0,0 +1,40 @@ +name: CI Check clang-format + +on: [push, pull_request] + +jobs: + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + # Github Actions requires a single row to be added to the build matrix. + # See https://help.github.com/en/articles/workflow-syntax-for-github-actions. + name: [ + clang-format-check + ] + + include: + - name: clang-format-check + os: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Git submodule + run: | + git submodule sync + git submodule update --init --recursive + + - name: Install Dependencies + run: | + sudo apt install clang-format-11 -yq + pip3 install --user -r .github/python_clang_format_reqs.txt + + - name: Check code style + run: | + echo "TASK=lint_all" >> $GITHUB_ENV + bash .github/check_style.sh diff --git a/README.md b/README.md index dbce93d7..a53b4777 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![CI Check clang-format](https://github.com/MOLAorg/mola/actions/workflows/check-clang-format.yml/badge.svg)](https://github.com/MOLAorg/mola/actions/workflows/check-clang-format.yml) +[![CI ROS](https://github.com/MOLAorg/mola/actions/workflows/build-ros.yml/badge.svg)](https://github.com/MOLAorg/mola/actions/workflows/build-ros.yml) [![CircleCI](https://img.shields.io/circleci/build/gh/MOLAorg/mola/master.svg)](https://circleci.com/gh/MOLAorg/mola) [![Docs](https://img.shields.io/badge/docs-latest-brightgreen.svg)](https://docs.mola-slam.org/latest/) diff --git a/scripts/clang_git_format/.gitignore b/scripts/clang_git_format/.gitignore new file mode 100644 index 00000000..7b8e9d3f --- /dev/null +++ b/scripts/clang_git_format/.gitignore @@ -0,0 +1,40 @@ +# Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so +*.pyc + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite + +# OS generated files # +###################### +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +build/ diff --git a/scripts/clang_git_format/LICENSE b/scripts/clang_git_format/LICENSE new file mode 100644 index 00000000..03c7bfc3 --- /dev/null +++ b/scripts/clang_git_format/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/scripts/clang_git_format/README.md b/scripts/clang_git_format/README.md new file mode 100644 index 00000000..374034e3 --- /dev/null +++ b/scripts/clang_git_format/README.md @@ -0,0 +1,116 @@ +# clang_git_format - Python wrapper for bulk reformatting of Git repos + +## General Information + +Current repository offers an automated solution to the reformatting of entire +projects managed by Git. It was inspired by [the series of +articles](https://engineering.mongodb.com/post/succeeding-with-clangformat-part-1-pitfalls-and-planning/) +of how MongoDB used clang-format to reformat their codebase. As a backbone for +this package, the MongoDB +[clang_format.py](https://github.com/mongodb/mongo/blob/master/buildscripts/clang_format.py) +has been used. Most notably package has the following capabilities: + +- Make sure a predetermined (`see clang_git_format/config.py` ) version of + clang-format script is to be used. If one is not installed, the + `ClangFormat` object is responsible of downloading and setting one up. +- Run clang-format using the `~/.clang-format` configuration file. A sample + file is provided along with the python code. Symlink it to your home + directory if you want to use it. + +- Format a code Git repo. User can optionally provide the programming + language(s) that the repository holds so that clang-format runs on the files + of that language only. By default C++ is assumed. Run + `clang_git_format --help for a list of provided languages`. + +- In case of unmerged code in a user (stray) branch users can also use the + `clang_git_format --reformat_branch `to sync his + code with the formatting changes of the master branch. For more information + on how this is done, read [this + article](https://engineering.mongodb.com/post/succeeding-with-clangformat-part-3-persisting-the-change). + +- Validate that a specific set of files complies to the clang-format rules that + is set. This can be handy in cases when one wants to integrate clang-format + into a continuous integration (CI) system to verify that all incoming + pull-requests / commits comply to the repo's coding style. + +This package differs to the initial MongoDB script in the following points: + +- It's refactored into a sane python package and breaks definition of classes, + utility methods into separate python files for readability. + +- Uses the powerful + [argparse](https://docs.python.org/dev/library/argparse.html) module instead + of the deprecated optparse. + +- It's more generic (doesn't depend on the MongoDB repo configuration), can be + used with an arbitrary Git repository and is not tied to the repo language + (C++, Javascript). + + +## Usage Instructions + +A typical usage of the script would be the following + +A list of command-line options is the following. Run `--help` yourself for an +up-to-date list of options: + + +% TODO - update this + +``` + +$ ./format_code.py --help + +usage: format_code.py [-h] [-c CLANG_FORMAT] -g GIT_REPO [-a LANG [LANG ...]] + [-x REGEX] [-i DIRS_IN [DIRS_IN ...]] + [-o DIRS_OUT [DIRS_OUT ...]] + (-l | -L | -p LINT_PATCHES [LINT_PATCHES ...] | -b REFORMAT_BRANCH REFORMAT_BRANCH | -f | -F) + +Apply clang-format to a whole Git repository. Execute this script and provide +it with the path to the git repository to operate in. WARNING: You have to run +it from the root of the repo if you want to apply its actions to all the +files. + +optional arguments: + -h, --help show this help message and exit + -c CLANG_FORMAT, --clang_format CLANG_FORMAT + Path to the clang-format command. + -g GIT_REPO, --git_repo GIT_REPO + Relative path to the root of the git repo that is to + be formatted/linted. + -a LANG [LANG ...], --lang LANG [LANG ...] + Languages used in the repository. This is used to + determinethe files which clang format runs for. + Default langs: {'const': None, 'help': 'Languages used + in the repository. This is used to determinethe files + which clang format runs for. Default langs: %s.', + 'option_strings': ['-a', '--lang'], 'dest': 'lang', + 'required': False, 'nargs': '+', 'choices': None, + 'default': ['cpp'], 'prog': 'format_code.py', + 'container': , 'type': 'str', 'metavar': None}. + -x REGEX, --regex REGEX + Custom regular expression to apply to the files that + are to be fed to clang-format. + -i DIRS_IN [DIRS_IN ...], --dirs_in DIRS_IN [DIRS_IN ...] + Sequence of directories. If given clang-format is + going to run for source exclusively in these + directories. + -o DIRS_OUT [DIRS_OUT ...], --dirs_out DIRS_OUT [DIRS_OUT ...] + Sequence of directories. If given clang-format is + going to ignore source files in these directories + -l, --lint Check if clang-format reports no diffs (clean state). + Execute only on files managed by git + -L, --lint_all Check if clang-format reports no diffs (clean state). + Checked files may or may not be managed by git + -p LINT_PATCHES [LINT_PATCHES ...], --lint_patches LINT_PATCHES [LINT_PATCHES ...] + Check if clang-format reports no diffs (clean state). + Check a list of patches, given sequentially after this + flag + -b REFORMAT_BRANCH REFORMAT_BRANCH, --reformat_branch REFORMAT_BRANCH REFORMAT_BRANCH + Reformat a branch given the and commits. + -f, --format Run clang-format against files managed by git + -F, --format_all Run clang-format against files that may or may not be + managed by the current git repository``` + +``` diff --git a/scripts/clang_git_format/clang_git_format/__init__.py b/scripts/clang_git_format/clang_git_format/__init__.py new file mode 100644 index 00000000..9862ed30 --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/__init__.py @@ -0,0 +1,10 @@ +from .clang_format import ClangFormat +from .repo import Repo +from .custom_exceptions import CommitIDTooShort + + +__all__ = [ + ClangFormat, + Repo, + CommitIDTooShort, +] diff --git a/scripts/clang_git_format/clang_git_format/clang_format.py b/scripts/clang_git_format/clang_git_format/clang_format.py new file mode 100644 index 00000000..584a7d86 --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/clang_format.py @@ -0,0 +1,159 @@ +import difflib +from distutils import spawn +import glob +import os +import sys +import threading +import subprocess +import logging + +from .config import (PROGNAME) + +from .utils import ( + callo, + ) + +logger = logging.getLogger("clang-format") + + +class ClangFormat: + """Find clang-format and linting/formating individual files. + """ + + def __init__(self, clang_path, cache_dir): + """Initialization method. + """ + self.clang_path = None + self.clang_format_progname_ext = "" + + if sys.platform == "win32": + self.clang_format_progname_ext += ".exe" + + # Check the clang-format the user specified + if clang_path is not None: + if os.path.isfile(clang_path): + self.clang_path = clang_path + + # Check the users' PATH environment variable now + if self.clang_path is None: + # Check for various versions staring with binaries with version + # specific suffixes in the user's path + programs = [ + PROGNAME + ] + + if sys.platform == "win32": + for i in range(len(programs)): + programs[i] += '.exe' + + for program in programs: + self.clang_path = spawn.find_executable(program) + + if self.clang_path: + if not self._validate_version(): + self.clang_path = None + else: + break + + # If Windows, try to grab it from Program Files + # Check both native Program Files and WOW64 version + if sys.platform == "win32": + programfiles = [ + os.environ["ProgramFiles"], + os.environ["ProgramFiles(x86)"], + ] + + for programfile in programfiles: + win32bin = os.path.join(programfile, + "LLVM\\bin\\clang-format.exe") + if os.path.exists(win32bin): + self.clang_path = win32bin + break + + # Have not found it yet, download it from the web + if self.clang_path is None: + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + + self.clang_path = os.path.join(cache_dir, + PROGNAME + + self.clang_format_progname_ext) + + # Download a new version if the cache is empty or stale + if not os.path.isfile(self.clang_path) \ + or not self._validate_version(): + + logger.error("Haven't found a valid %s version in PATH. ", + PROGNAME) + sys.exit(1) + + # Validate we have the correct version + # We only can fail here if the user specified a clang-format binary and + # it is the wrong version + if not self._validate_version(): + logger.error("Exiting because of previous warning.") + sys.exit(1) + + self.print_lock = threading.Lock() + + def _validate_version(self): + """Validate clang-format is the expected version + """ + cf_version = callo([self.clang_path, "--version"]) + logger.warning("Using clang-format: %s", str(cf_version)) + + return True + + def _lint(self, file_name, print_diff): + """Check the specified file has the correct format + """ + fo = open(file_name, 'rb') + original_file = fo.read().decode('utf-8') + + # Get formatted file as clang-format would format the file + formatted_file = callo( + [self.clang_path, "--style=file", file_name]).decode('utf-8') + + if original_file != formatted_file: + if print_diff: + original_lines = original_file.splitlines(keepends=True) + formatted_lines = formatted_file.splitlines(keepends=True) + result = difflib.unified_diff(original_lines, formatted_lines) + + # Take a lock to ensure diffs do not get mixed when printed to + # the screen + with self.print_lock: + logger.error("Found diff for %s", file_name) + logger.info("To fix formatting errors, run %s " + "--style=file -i %s", self.clang_path, + file_name) + + sys.stderr.writelines(result) + + return False + + return True + + def lint(self, file_name): + """Check the specified file has the correct format + """ + return self._lint(file_name, print_diff=True) + + def format_func(self, file_name): + """Update the format of the specified file + """ + if self._lint(file_name, print_diff=False): + return True + + # Update the file with clang-format + formatted = not subprocess.call( + [self.clang_path, "--style=file", "-i", file_name]) + + # Version 3.8 generates files like foo.cpp~RF83372177.TMP when it + # formats foo.cpp on Windows, we must clean these up + if sys.platform == "win32": + glob_pattern = file_name + "*.TMP" + for fglob in glob.glob(glob_pattern): + os.unlink(fglob) + + return formatted diff --git a/scripts/clang_git_format/clang_git_format/config.py b/scripts/clang_git_format/clang_git_format/config.py new file mode 100644 index 00000000..c834781f --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/config.py @@ -0,0 +1,2 @@ + +PROGNAME = "clang-format-11" diff --git a/scripts/clang_git_format/clang_git_format/custom_exceptions.py b/scripts/clang_git_format/clang_git_format/custom_exceptions.py new file mode 100644 index 00000000..a92824a7 --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/custom_exceptions.py @@ -0,0 +1,38 @@ +# Copied from python 2.7 version of subprocess.py +# Exception classes used by this module. + + +class CalledProcessError(Exception): + """This exception is raised when a process run by check_call() or + check_output() returns a non-zero exit status. The exit status will be + stored in the returncode attribute; check_output() will also store the + output in the output attribute. + + """ + + def __init__(self, returncode, cmd, output=None): + self.returncode = returncode + self.cmd = cmd + self.output = output + + def __str__(self): + return ("Command '%s' returned non-zero exit status %d with output %s" + % (self.cmd, self.returncode, self.output)) + + +class CustomError(BaseException): + """Class for implementing custom exceptions""" + pass + + +class CommitIDTooShort(CustomError): + def __init__(self, commit_id, min_len): + self.commit_id = commit_id + self.min_len = min_len + + + def __str__(self): + return ("At least %d characters of the commit ID " + "hash should be provided. Current commit ID: %s" + % (self.min_len, self.commit_id)) + diff --git a/scripts/clang_git_format/clang_git_format/repo.py b/scripts/clang_git_format/clang_git_format/repo.py new file mode 100644 index 00000000..823604d2 --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/repo.py @@ -0,0 +1,343 @@ +from .utils import callo +import os +import subprocess +import re + +import logging +logger = logging.getLogger("clang-format") + + +class Repo(object): + """Class encapsulates all knowledge about a git repository, and its metadata + to run clang-format. + """ + + def __init__(self, path, custom_regex="", dirs_in=[], dirs_out=[]): + """Initialization method. + + :path str Relative path to the Root of the repository + + WARNING: After initialization of the Repo, users should set the + languages the repo contains (langs_used). This is used to run + clang-format only on files designated by the langage + """ + logger.info("Initializing repo...") + + self.path = path + self.custom_regex = "(" + custom_regex + ")" + + # Programming languages that the files in this repo are written in. + # Variable is used to decide what files is clang-format ultimately is + # going to operate on + self.langs_to_file_endings = { + "cpp": ["h", "hxx", "cpp", "cc", "cxx"], + "c": ["h", "c"], + "objc": ["h", "mm"], + "java": ["class", "java"], + "javascript": ["js"], + None: [], + } + + assert isinstance(dirs_in, list) and\ + "dirs_in should be a list of directories. Instead got {}"\ + .format(dirs_in) + self.dirs_in = dirs_in + assert isinstance(dirs_out, list) and\ + "dirs_out should be a list of directories. Instead got {}"\ + .format(dirs_out) + self.dirs_out = dirs_out + + # default language is cpp + self._langs_used = ["cpp"] + + self.root = self._get_root() + + @property + def langs_used(self): + return self._langs_used + + @langs_used.setter + def langs_used(self, langs_in): + """Set the programming languages that the repo contains files of.""" + + if not langs_in: + return + + assert isinstance(langs_in, list) and \ + ("The languages of the repo should be provided in a list of " + "strings.\nExiting...") + + if set([i for i in langs_in + if i in self.langs_to_file_endings.keys()]) != set(langs_in): + logger.fatal("The following languages are available to use: %s", + self.langs_to_file_endings.keys()) + exit(1) + + self._langs_used = langs_in + + @langs_used.getter + def langs_used(self): + return self._langs_used + + def _callgito(self, args): + """Call git for this repository, and return the captured output + """ + # These two flags are the equivalent of -C in newer versions of Git but + # we use these to support versions pre 1.8.5 but it depends on the + # command and what the current directory is + return callo([ + 'git', '--git-dir', + os.path.join(self.path, ".git"), '--work-tree', self.path + ] + args) + + def _callgit(self, args): + """Call git for this repository without capturing output + This is designed to be used when git returns non-zero exit codes. + """ + # These two flags are the equivalent of -C in newer versions of Git but + # we use these to support versions pre 1.8.5 but it depends on the + # command and what the current directory is + return subprocess.call([ + 'git', '--git-dir', + os.path.join(self.path, ".git"), '--work-tree', self.path + ] + args) + + def _get_local_dir(self, path): + """Get a directory path relative to the git root directory + """ + if os.path.isabs(path): + return os.path.relpath(path, self.root) + return path + + def get_candidates(self, candidates): + """Get the set of candidate files to check by querying the repository + + Returns the full path to the file for clang-format to consume. + """ + if candidates is not None and len(candidates) > 0: + candidates = [self._get_local_dir(f) for f in candidates] + valid_files = list( + set(candidates).intersection(self.get_candidate_files())) + else: + valid_files = list(self.get_candidate_files()) + + # Get the full file name here + valid_files = [ + os.path.normpath(os.path.join(self.root, f)) for f in valid_files + ] + + return valid_files + + def get_root(self): + """Get the root directory for this repository + """ + return self.root + + def _get_root(self): + """Gets the root directory for this repository from git + """ + gito = self._callgito(['rev-parse', '--show-toplevel']) + + return gito.rstrip() + + def _git_ls_files(self, cmd): + """Run git-ls-files and filter the list of files to a valid candidate + list + + This constitutes a backbone method for fetching the list of files on + which clang-format operates on. + """ + gito = self._callgito(cmd) + + # This allows us to pick all the interesting files + # in the mongo and mongo-enterprise repos + file_list = [line.rstrip() for line in gito.splitlines()] + final_list = self.filter_files_by_dir(file_list) + + + files_regexp = self.get_files_regexp() + final_list = [l for l in final_list if files_regexp.search(l.decode('utf8'))] + logger.warn("Executing clang-format on %d files" % len(final_list)) + + return final_list + + def filter_files_by_dir(self, file_list): + """Filter the given list of files based on the list of specified + directories. + """ + + # If dirs_in is given use only those files that have that directory in + # their body + valid_files_in = [] + if self.dirs_in: + for line in file_list: + if any([self._dir_filter(d.encode(), line, do_include=True) + for d in self.dirs_in]): + valid_files_in.append(line) + continue + else: + valid_files_in = file_list + + # If dirs_out is given use only those files that have that directory in + # their body + valid_files_out = [] + if self.dirs_out: + for line in valid_files_in: + if all([self._dir_filter(d.encode(), line, do_include=False) + for d in self.dirs_out]): + valid_files_out.append(line) + continue + else: + valid_files_out = valid_files_in + + return valid_files_out + + + def get_files_regexp(self): + """Return the regular expression that is used to filter the files for + which clang format is actually going to run. + + This takes in account + - Language suffixes that are to be considered + - User-provided custom regexp + """ + + files_match_str = "" + for lang in self.langs_used: + lang_exts = self.langs_to_file_endings[lang] + for ext in lang_exts + [ext.upper() for ext in lang_exts]: + files_match_str += ext + "|" + + + files_match_str = "(" + files_match_str + ")" + files_regexp = re.compile( + '{}\\.{}$'.format(self.custom_regex, files_match_str)) + logger.warn("Regexp to find source files: %s" % files_regexp.pattern) + + return files_regexp + + + def get_candidate_files(self): + """Query git to get a list of all files in the repo to consider for + analysis + """ + return self._git_ls_files(["ls-files", "--cached"]) + + def get_working_tree_candidate_files(self): + """Query git to get a list of all files in the working tree to consider + for analysis. Files may not be managed by Git + """ + files = self._git_ls_files(["ls-files", "--cached", "--others"]) + return files + + def get_working_tree_candidates(self): + """Get the set of candidate files to check by querying the repository + + Returns the full path to the file for clang-format to consume. + """ + valid_files = list(self.get_working_tree_candidate_files()) + + # Get the full file name here + valid_files = [ + os.path.normpath(os.path.join(self.root, f)) for f in valid_files + ] + + return valid_files + + def is_detached(self): + """Is the current working tree in a detached HEAD state? + """ + # symbolic-ref returns 1 if the repo is in a detached HEAD state + return self._callgit(["symbolic-ref", "--quiet", "HEAD"]) + + def is_ancestor(self, parent, child): + """Is the specified parent hash an ancestor of child hash? + """ + # merge base returns 0 if parent is an ancestor of child + return not self._callgit(["merge-base", "--is-ancestor", parent, child]) + + def is_commit(self, sha1): + """Is the specified hash a valid git commit? + """ + # cat-file -e returns 0 if it is a valid hash + return not self._callgit(["cat-file", "-e", "%s^{commit}" % sha1]) + + def is_working_tree_dirty(self): + """Does the current working tree have changes? + """ + # diff returns 1 if the working tree has local changes + return self._callgit(["diff", "--quiet"]) + + def does_branch_exist(self, branch): + """Does the branch exist? + """ + # rev-parse returns 0 if the branch exists + return not self._callgit(["rev-parse", "--verify", branch]) + + def get_merge_base(self, commit): + """Get the merge base between 'commit' and HEAD""" + return self._callgito(["merge-base", "HEAD", commit]).rstrip() + + def get_branch_name(self): + """Get the current branch name, short form + This returns "master", not "refs/head/master" + Will not work if the current branch is detached + """ + branch = self.rev_parse(["--abbrev-ref", "HEAD"]) + if branch == "HEAD": + raise ValueError("Branch is currently detached") + + return branch + + def add(self, command): + """git add wrapper + """ + return self._callgito(["add"] + command) + + def checkout(self, command): + """git checkout wrapper + """ + return self._callgito(["checkout"] + command) + + def commit(self, command): + """git commit wrapper + """ + return self._callgito(["commit"] + command) + + def diff(self, command): + """git diff wrapper + """ + return self._callgito(["diff"] + command) + + def log(self, command): + """git log wrapper + """ + return self._callgito(["log"] + command) + + def rev_parse(self, command): + """git rev-parse wrapper + """ + return self._callgito(["rev-parse"] + command).rstrip() + + def rm(self, command): + """git rm wrapper + """ + return self._callgito(["rm"] + command) + + def show(self, command): + """git show wrapper + """ + return self._callgito(["show"] + command) + + @staticmethod + def _dir_filter(d, line, do_include=True): + """Return True if line includes/doesn't include the given directory + d. + """ + ret = False + if do_include: + ret = d in line + else: + ret = d not in line + + return ret diff --git a/scripts/clang_git_format/clang_git_format/utils.py b/scripts/clang_git_format/clang_git_format/utils.py new file mode 100644 index 00000000..222f15b4 --- /dev/null +++ b/scripts/clang_git_format/clang_git_format/utils.py @@ -0,0 +1,80 @@ +"""File containing utility functions""" + +import os +import sys +import subprocess +from .custom_exceptions import CalledProcessError +import tarfile + +import logging +logger = logging.getLogger("clang-format") + + +def get_base_dir(): + """Get the base directory for the Git Repo. + + This script assumes that it is running in buildscripts/, and uses + that to find the base directory. + """ + try: + return subprocess.check_output( + ['git', 'rev-parse', '--show-toplevel']).rstrip() + except subprocess.CalledProcessError: + # We are not in a valid git directory. Use the script path instead. + return os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + +# Copied from python 2.7 version of subprocess.py +def _check_output(*popenargs, **kwargs): + r"""Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> _check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd, output) + return output + + +def callo(args): + """Call a program, and capture its output + """ + return _check_output(args) + + +def extract_clang_format(tar_path): + """ Extract just the clang-format binary. + On OSX, we shell out to tar because tarfile doesn't support xz compression + """ + if sys.platform == 'darwin': + subprocess.call(['tar', '-xzf', tar_path, '*clang-format*']) + # Otherwise we use tarfile because some versions of tar don't support + # wildcards without a special flag + else: + tarfp = tarfile.open(tar_path) + for name in tarfp.getnames(): + if name.endswith('clang-format'): + tarfp.extract(name) + tarfp.close() diff --git a/scripts/clang_git_format/format_code.py b/scripts/clang_git_format/format_code.py new file mode 100755 index 00000000..e484b26d --- /dev/null +++ b/scripts/clang_git_format/format_code.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +""" +A script that provides: +1. Ability to grab binaries where possible from LLVM. +2. Ability to download binaries from MongoDB cache for clang-format. +3. Validates clang-format is the right version. +4. Supports checking which files are to be checked. +5. Supports validating and updating a set of files to the right coding style. +""" +from __future__ import print_function +# absolute_import + +import queue +import re +import os +import sys +import threading +import time +import argparse +from multiprocessing import cpu_count + +from clang_git_format import ClangFormat +from clang_git_format import Repo +from clang_git_format import CommitIDTooShort + +# setup the logging +import logging +from colorlog import ColoredFormatter + +LOG_LEVEL = logging.DEBUG +LOGFORMAT = ("%(log_color)s%(levelname)-5s%(reset)s " + "| %(log_color)s%(message)s%(reset)s") + +logging.root.setLevel(LOG_LEVEL) +formatter = ColoredFormatter(LOGFORMAT) +stream = logging.StreamHandler() +stream.setLevel(LOG_LEVEL) +stream.setFormatter(formatter) + +logger = logging.getLogger("clang-format") +logger.setLevel(LOG_LEVEL) +logger.addHandler(stream) + + +def parallel_process(items, func): + """Run a set of work items to completion + """ + try: + cpus = cpu_count() + + except NotImplementedError: + cpus = 1 + + task_queue = queue.Queue() + + # Use a list so that worker function will capture this variable + pp_event = threading.Event() + pp_result = [True] + pp_lock = threading.Lock() + + def worker(): + """Worker thread to process work items in parallel. """ + while not pp_event.is_set(): + + try: + item = task_queue.get_nowait() + except queue.Empty: + # if the queue is empty, exit the worker thread + pp_event.set() + return + + try: + logger.debug("Operating on file: %s" % item) + ret = func(item) + finally: + # Tell the queue we finished with the item + task_queue.task_done() + + # Return early if we fail, and signal we are done + if not ret: + logger.error("clang-format exited with an error!") + with pp_lock: + pp_result[0] = False + + pp_event.set() + return + + # Enqueue all the work we want to process + for item in items: + task_queue.put(item) + + # Process all the work + threads = [] + for cpu in range(cpus): + thread = threading.Thread(target=worker) + + thread.daemon = True + thread.start() + threads.append(thread) + + # Wait for the threads to finish + # Loop with a timeout so that we can process Ctrl-C interrupts + # Note: On Python 2.6 wait always returns None so we check is_set also, + # This works because we only set the event once, and never reset it + while not pp_event.wait(1) and not pp_event.is_set(): + time.sleep(1) + + for thread in threads: + thread.join() + + return pp_result[0] + + +class ClangRepoFormatter(object): + """Class that handles the overall execution.""" + + def __init__(self): + super(ClangRepoFormatter, self).__init__() + + def get_repo(self): + """Return Repo object for the git repository to be formatted.""" + return self.git_repo + + def run(self): + """Main entry point """ + logger.info("Initializing script...") + + parser = argparse.ArgumentParser() + parser.description = ("Apply clang-format to a whole Git repository. " + "Execute this script and provide it with the " + "path to the git repository " + "to operate in. " + "WARNING: You have to run it from the root of " + "the repo if you want to apply its actions " + "to all the files.") + + parser.add_argument( + "-c", + "--clang_format", + default=None, + type=str, + help="Path to the clang-format command.") + + parser.add_argument( + '-g', + '--git_repo', + type=str, + required=True, + help=("Relative path to the root of the git repo that " + "is to be formatted/linted.")) + + default_langs = ["cpp"] + parser.add_argument( + '-a', + '--lang', + type=str, + nargs='+', + default=default_langs, + help=("Languages used in the repository. This is used to determine" + "the files which clang format runs for. Default langs: %s." + .format(default_langs))) + parser.add_argument( + '-x', + '--regex', + type=str, + default="", + required=False, + help=("Custom regular expression to apply to the files that are " + "to be fed to clang-format.")) + parser.add_argument( + '-i', + '--dirs_in', + type=str, + nargs="+", + default=[], + required=False, + help=("Sequence of directories. If given clang-format is going to " + "run for source exclusively in these directories.")) + parser.add_argument( + '-o', + '--dirs_out', + type=str, + default=[], + nargs="+", + required=False, + help=("Sequence of directories. If given clang-format is going to " + "ignore source files in these directories")) + + + + # Mutually exclusive, internet-related arguments + commands_group = parser.add_mutually_exclusive_group(required=True) + + commands_group.add_argument( + "-d", + "--lint_files", + default=[], + nargs='+', + help=("Check if clang-format reports no diffs (clean state). " + "Execute only on files given sequentially after this flag.")) + commands_group.add_argument( + "-l", + "--lint", + action="store_true", + help=("Check if clang-format reports no diffs (clean state). " + "Execute only on files managed by git")) + commands_group.add_argument( + "-L", + "--lint_all", + action="store_true", + help=("Check if clang-format reports no diffs (clean state). " + "Checked files may or may not be managed by git")) + commands_group.add_argument( + "-p", + "--lint_patches", + default=[], + nargs='+', + help=("Check if clang-format reports no diffs (clean state). " + "Check a list of patches, given sequentially after this " + "flag")) + commands_group.add_argument( + "-b", + "--reformat_branch", + default=[], + nargs=3, + help=("Reformat CURRENT branch given the and" + " commits." + " Finally rebase it on top of branch")) + commands_group.add_argument( + "-f", + "--format", + action="store_true", + help=("Run clang-format against files managed by git")) + commands_group.add_argument( + "-F", + "--format_all", + action="store_true", + help=("Run clang-format against files that may or may not " + "be managed by the current git repository")) + + parser_args = vars(parser.parse_args()) + + # clang-format executabel wrapper + self.clang_format = ClangFormat(parser_args["clang_format"], + self._get_build_dir()) + + # git repo + self.git_repo = Repo( + parser_args["git_repo"], custom_regex=parser_args["regex"], + dirs_in=parser_args["dirs_in"], + dirs_out=parser_args["dirs_out"], + ) + self.git_repo.langs_used = parser_args['lang'] + + # determine your action based on the user input + if parser_args["lint"]: + self.lint() + elif parser_args["lint_all"]: + self.lint_all() + elif parser_args["lint_files"]: + file_list = parser_args["lint_files"] + self._lint_files(file_list) + elif len(parser_args["lint_patches"]): + file_list = parser_args["lint_patches"] + self.lint_patches(file_list) + elif len(parser_args["reformat_branch"]): + start = parser_args["reformat_branch"][0] + end = parser_args["reformat_branch"][1] + main_branch = parser_args["reformat_branch"][2] + self.reformat_branch(start, end, main_branch) + elif parser_args["format"]: + self.format_func() + elif parser_args["format_all"]: + self.format_func_all() + else: + logger.fatal("Unexpected error in the parsing of command line " + "arguments! Exiting.") + + def get_list_from_lines(self, lines): + """"Convert a string containing a series of lines into a list of strings + """ + return [line.rstrip() for line in lines.splitlines()] + + def get_files_to_check_working_tree(self): + """Get a list of files to check from the working tree. + This will pick up files not managed by git. + """ + repo = self.get_repo() + valid_files = repo.get_working_tree_candidates() + + return valid_files + + def get_files_to_check(self): + """Get a list of files that need to be checked + based on which files are managed by git. + """ + repo = self.get_repo() + valid_files = repo.get_candidates(None) + return valid_files + + def get_files_to_check_from_patch(self, patches): + """Take a list of patch files generated by git diff, and scan them for a + list of files. + + :param patches list of patches to check + """ + candidates = [] + + # Get a list of candidate_files + check = re.compile( + r"^diff --git a\/([a-z\/\.\-_0-9]+) b\/[a-z\/\.\-_0-9]+") + + lines = [] + for patch in patches: + with open(patch, "rb") as infile: + lines += infile.readlines() + + candidates = [ + check.match(line).group(1) for line in lines if check.match(line) + ] + + repo = self.get_repo() + valid_files = repo.get_candidates(candidates) + return valid_files + + def _get_build_dir(self): + """Get the location of a build directory in case we need to download + clang-format + + """ + return os.path.join(os.path.curdir, "build") + + def _lint_files(self, files): + """Lint a list of files with clang-format + """ + lint_clean = parallel_process([os.path.abspath(f) + for f in files], self.clang_format.lint) + + if not lint_clean: + logger.error("Code Style does not match coding style") + sys.exit(1) + + def lint_patches(self, infile): + """Lint patch command entry point + """ + files = self.get_files_to_check_from_patch(infile) + + # Patch may have files that we do not want to check which is fine + if files: + self._lint_files(files) + + def lint(self): + """Lint files command entry point + """ + files = self.get_files_to_check() + self._lint_files(files) + + return True + + def lint_all(self): + """Lint files command entry point based on working tree (some files may + not be managed by git + """ + files = self.get_files_to_check_working_tree() + self._lint_files(files) + + return True + + def _format_files(self, files): + """Format a list of files with clang-format + """ + format_clean = parallel_process([os.path.abspath(f) for f in files], + self.clang_format.format_func) + + if not format_clean: + logger.error("failed to format files") + sys.exit(1) + + def format_func(self): + """Format files command entry point + """ + files = self.get_files_to_check() + + self._format_files(files) + + def format_func_all(self): + """Format files command entry point + """ + files = self.get_files_to_check_working_tree() + self._format_files(files) + + def reformat_branch(self, commit_prior_reformat, commit_after_reformat, + main_branch): + """Reformat a branch made before a clang-format run + + :param str commit_prior_reformat The base commit ID connecting the main + branch with the branch to be merged + :param str commit_prior_reformat The last commit ID to be reformatted + and merged into the main branch + """ + + min_commit_id_len = 5 + + # verify given commits lenght + if (len(commit_prior_reformat) < min_commit_id_len): + raise CommitIDTooShort(commit_prior_reformat, + min_commit_id_len) + if (len(commit_after_reformat) < min_commit_id_len): + raise CommitIDTooShort(commit_after_reformat, + min_commit_id_len) + + old_pwd = os.getcwd() + os.chdir(self.get_repo().path) + + repo = self.get_repo() + + ###################################################################3 + # Validate that the current state is OK + + # Validate that user passes valid commits + if not repo.is_commit(commit_prior_reformat): + raise ValueError( + "Commit Prior to Reformat '%s' is not " + "a valid commit in this repo" % commit_prior_reformat) + if not repo.is_commit(commit_after_reformat): + raise ValueError( + "Commit After Reformat '%s' is not a valid commit in this repo" + % commit_after_reformat) +# if not repo.is_ancestor(commit_prior_reformat, + # commit_after_reformat): + # raise ValueError( + # ("Commit Prior to Reformat '%s' is not a valid ancestor " + # "of Commit After" + " Reformat '%s' in this repo") % + # (commit_prior_reformat, commit_after_reformat)) + + # Validate the user is on a local branch that has the right merge base + if repo.is_detached(): + raise ValueError("You must not run this script in a detached " + "HEAD state") + + # Validate the user has no pending changes + if repo.is_working_tree_dirty(): + raise ValueError("Your working tree has pending changes. " + "You must have a clean working tree before " + "proceeding.") + + # validate that the parent of the stranded commit is the merge base + merge_base = repo.get_merge_base(commit_prior_reformat) + if (not merge_base[0:min_commit_id_len] == + commit_prior_reformat[0:min_commit_id_len]): + raise ValueError("Please **rebase** your work to '%s' and resolve all " + "conflicts before running this script" % + (commit_prior_reformat)) + + merge_base = repo.get_merge_base(main_branch) + + # if (not merge_base[0:min_commit_id_len] == + # commit_prior_reformat[0:min_commit_id_len]): + # raise ValueError(("The base commit of the merge (%s) and the " + # "start_commit (%s) issued don't match." + # % (merge_base, commit_prior_reformat))) + + # End of validations + ###################################################################3 + + # Everything looks good so lets start going through all the commits + branch_name = repo.get_branch_name() + new_branch = "%s-reformatted" % branch_name + + # Make sure that this is a new branch + if repo.does_branch_exist(new_branch): + raise ValueError("The branch '%s' already exists. " + "Please delete the " + "branch '%s', or rename the current branch." % + (new_branch, new_branch)) + + commits = self.get_list_from_lines( + repo.log([ + "--reverse", "--pretty=format:%H", + "%s..HEAD" % commit_prior_reformat + ])) + + previous_commit_base = commit_after_reformat + files_to_check = self.get_files_to_check() + + # Go through all the commits the user made on the local branch and + # migrate to a new branch that is based on post_reformat commits instead + for commit_hash in commits: + repo.checkout(["--quiet", commit_hash]) + + deleted_files = [] + + # Format each of the files by checking out just a single commit from + # the user's branch + commit_files = self.get_list_from_lines( + repo.diff(["HEAD~", "--name-only"])) + + for commit_file in commit_files: + + # Format each file needed if it was not deleted + if not os.path.exists(commit_file): + logger.warning("Skipping file '%s' since it has been " + "deleted in commit '%s'" + % (commit_file, commit_hash)) + deleted_files.append(commit_file) + continue + + if commit_file in files_to_check: + self.clang_format.format_func(commit_file) + else: + logger.info("Skipping file '%s' since it is not a " + "file clang_format should format" % commit_file) + + # Check if anything needed reformatting, and if so amend the commit + if not repo.is_working_tree_dirty(): + print("Commit %s needed no reformatting" % commit_hash) + else: + repo.commit(["--all", "--amend", "--no-edit"]) + + # Rebase our new commit on top the post-reformat commit + previous_commit = repo.rev_parse(["HEAD"]) + + # Checkout the new branch with the reformatted commits + # Note: we will not name as a branch until we are done with all + # commits on the local branch + repo.checkout(["--quiet", previous_commit_base]) + + # Copy each file from the reformatted commit on top of the post + # reformat + diff_files = self.get_list_from_lines( + repo.diff([ + "%s~..%s" % (previous_commit, previous_commit), + "--name-only" + ])) + + for diff_file in diff_files: + # If the file was deleted in the commit we are reformatting, we + # need to delete it again + if diff_file in deleted_files: + repo.rm([diff_file]) + continue + + # The file has been added or modified, continue as normal + file_contents = repo.show( + ["%s:%s" % (previous_commit, diff_file)]) + + root_dir = os.path.dirname(diff_file) + if root_dir and not os.path.exists(root_dir): + os.makedirs(root_dir) + + with open(diff_file, "w+") as new_file: + new_file.write(file_contents) + + repo.add([diff_file]) + + # Create a new commit onto clang-formatted branch + repo.commit(["--reuse-message=%s" % previous_commit]) + + previous_commit_base = repo.rev_parse(["HEAD"]) + + + # Create a new branch to mark the hashes we have been using + repo.checkout(["-b", new_branch]) + + # change back to previous directory + os.chdir(old_pwd) + + logger.info("reformat_branch is done running.\n" + "A copy of your branch has been made named '%s'," + " and formatted with clang-format.\n" + " The original branch has been left unchanged.\n" + " Your new branch should already be rebased on top of %s." + % (new_branch)) + + +if __name__ == "__main__": + crf = ClangRepoFormatter() + crf.run()