Merge branch 'master' into publish_all

.github/workflows/ci.yml

@@ -6,13 +6,19 @@ on: [push, pull_request]
 jobs:
   bionic-ci:
     runs-on: ubuntu-18.04
-    name: Bionic Aptly CI -
+    name: Bionic Aptly ${{ matrix.job_type }} -
           debug ${{ matrix.show_debug_cmds }}
     strategy:
       matrix:
         show_debug_cmds:
           - true
           - false
+        job_type:
+          - 'CI'
+          - 'CONFIG_TESTER'
+        exclude:
+          - job_type: 'CONFIG_TESTER'
+            show_debug_cmds: false
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -52,9 +58,27 @@ jobs:
                 %no-protection
             EOF
             gpg --gen-key --batch genkey
-      - name: Run tests
+
+      # CI TESTING
+      - name: Run CI tests
+        if: ${{ matrix.job_type == 'CI' }}
         run: |
             python3 scripts/aptly/aptly_importer_TEST.py
         env:
           _DEBUG_MSGS_REPREPRO_UPDATER_TEST_SUITE_: ${{ matrix.show_debug_cmds }}
           _ALLOW_DESTRUCTIVE_TESTS_REPREPRO_UPDATER_TEST_SUITE_: true
+      # CONFIG FILE TESTER
+      - id: files
+        if: ${{ matrix.job_type == 'CONFIG_TESTER' }}
+        uses: jitterbit/get-changed-files@v1
+      - name: Run testing on changed config files
+        if: ${{ matrix.job_type == 'CONFIG_TESTER' }}
+        run: |
+          for changed_file in ${{ steps.files.outputs.all }}; do
+            if [[ ${changed_file} != ${changed_file/config\/*.yaml} ]]; then
+              echo "+ Detected config file: ${changed_file}."
+              python3 scripts/aptly/aptly_importer.py --ignore-signatures --only-mirror-creation ${changed_file}
+            else
+              echo "- Ignoring non config file ${changed_file}."
+            fi
+          done

config/libopensplice67_bionic_xenial.yaml

@@ -1,3 +1,4 @@
+---
 name: libopensplice67_bionic_xenial
 method: http://packages.osrfoundation.org/gazebo/ubuntu-prerelease
 suites: [bionic, xenial]

config/opensplice69.osrfoundation.yaml

@@ -1,3 +1,4 @@
+---
 name: opensplice69
 method: http://packages.osrfoundation.org/gazebo/ubuntu-stable
 suites: [xenial, bionic]

config/pyside2.prerelease.yaml

@@ -1,3 +1,4 @@
+---
 name: pyside2
 method: http://ppa.launchpad.net/tully.foote/pyside2-reproduction/ubuntu/
 suites: [xenial]
@@ -46,9 +47,9 @@ Package (= python-pyside2.qtwidgets) |\
 Package (= python-pyside2.qtwidgets-dbg) |\
 Package (= python-pyside2.qtx11extras) |\
 Package (= python-pyside2.qtx11extras-dbg) |\
-Package (= python-pyside2.qtxml) \
+Package (= python-pyside2.qtxml) |\
 Package (= python-pyside2.qtxml-dbg)), \
-$Version (% 2.0.0+dev1-1~202102130210~rev1858~pkg5~git131fdfd1~ubuntu16.04.1) |\
+$Version (% 2.0.0+dev1-1~202102130210~rev1858~pkg5~git131fdfd1~ubuntu16.04.1)) |\
 ((Package (= shiboken2) |\
 Package (= libshiboken-py3-2.0) |\
 Package (= libshiboken2-dev) |\

scripts/aptly/aptly_importer.py

@@ -183,7 +183,7 @@ def __error(self, msg):
         exit(2)
 
     def __load_config_file(self, config_file_path):
-        fn = Path(__file__).parent / config_file_path
+        fn = Path(config_file_path).absolute()
         try:
             with open(str(fn), 'r') as stream:
                 config = yaml.safe_load(stream)
@@ -195,11 +195,18 @@ def __load_config_file(self, config_file_path):
 
 
 class UpdaterManager():
-    def __init__(self, input_file, debug=False, aptly_config_file=None, simulate_repo_import=False, snapshot_and_publish=False):
+    def __init__(self, input_file, debug=False,
+                 aptly_config_file=None, 
+                 ignore_mirror_signature=False,
+                 only_mirror_creation=False,
+                 simulate_repo_import=False,
+                 snapshot_and_publish=False):
         self.aptly = Aptly(debug,
                            config_file=aptly_config_file)
         self.config = UpdaterConfiguration(input_file)
         self.debug = debug
+        self.ignore_mirror_signature = ignore_mirror_signature
+        self.only_mirror_creation = only_mirror_creation
         self.simulate_repo_import = simulate_repo_import
         self.snapshot_and_publish = snapshot_and_publish
         self.snapshot_timestamp = None
@@ -211,14 +218,22 @@ def __create_aptly_mirror(self, distribution):
         if self.aptly.exists(Aptly.ArtifactType.MIRROR, mirror_name):
             self.__log_ok('Removing existing mirror')
             self.aptly.run(['mirror', 'drop', mirror_name])
-        self.aptly.run(['mirror', 'create', '-with-sources',
-                        f"-architectures={','.join(self.config.architectures)}",
-                        f"-filter={self.config.filter_formula}",
-                        mirror_name,
-                        self.config.method,
-                        distribution,
-                        self.config.component])
-        self.aptly.run(['mirror', 'update', mirror_name])
+        create_cmd = ['mirror', 'create', '-with-sources',
+                      f"-architectures={','.join(self.config.architectures)}",
+                      f"-filter={self.config.filter_formula}"]
+        if self.ignore_mirror_signature:
+            create_cmd += ['-ignore-signatures']
+        create_cmd += [mirror_name,
+                       self.config.method,
+                       distribution,
+                       self.config.component]
+        self.aptly.run(create_cmd)
+        update_cmd = ['mirror', 'update']
+        if self.ignore_mirror_signature:
+            update_cmd += ['-ignore-signatures']
+        update_cmd += [mirror_name]
+        self.aptly.run(update_cmd)
+
         self.__log_ok(f"mirror {mirror_name} created")
 
     def __create_aptly_snapshot_from_repo(self, distribution):
@@ -331,7 +346,9 @@ def run(self):
                                                          self.__get_mirror_name(dist)):
                 self.__remove_all_generated_mirrors()
                 self.__error(f'{self.__get_mirror_name(dist)} does not have a source package. Removing generated mirrors')
-            self.__log_ok('There is a source pakage in the mirror')
+            self.__log_ok('All source packages exist in the mirror')
+            if self.only_mirror_creation:
+                return True
             # 2. Import from mirrors to local repositories
             self.__import__aptly_mirror_to_repo(dist)
             if self.simulate_repo_import:
@@ -351,11 +368,19 @@ def run(self):
 
 def main():
     """
-    Usage: python3 aptly_importer.py [--simulate-repo-import] <config_file>
+    Usage: python3 aptly_importer.py [parameters] <config_file>
     """
-    usage = "usage: %prog [--simulate-repo-import] config_file"
+    usage = "usage: python3 aptly_importer.py [parameters] <config_file>"
     parser = argparse.ArgumentParser(usage)
     parser.add_argument('config_file', type=str, default=None)
+    parser.add_argument("--ignore-signatures",
+                        help="Ignore mirror signatures when importing",
+                        action="store_true")
+    parser.add_argument("--only-mirror-creation",
+                        help="Perform only the mirror creation and update. Useful for"
+                             "checking, it does not modify anything in the system",
+                        action="store_true")
+
     parser.add_argument('--simulate-repo-import',
                         action='store_true',
                         help='Perform a dry-run until the point of simulation mirrors import to repositories')
@@ -366,7 +391,11 @@ def main():
     if not path.exists(args.config_file):
         parser.error("Missing input file from %s" % args.config_file)
 
-    manager = UpdaterManager(args.config_file, simulate_repo_import=args.simulate_repo_import, snapshot_and_publish=args.snapshot_and_publish)
+    manager = UpdaterManager(input_file=args.config_file,
+                             ignore_mirror_signature=args.ignore_signatures,
+                             only_mirror_creation=args.only_mirror_creation,
+                             simulate_repo_import=args.simulate_repo_import,
+                             snapshot_and_publish=args.snapshot_and_publish)
     manager.run()
 
 

scripts/aptly/aptly_importer_TEST.py

@@ -1,10 +1,13 @@
 import aptly_importer
 import os
+from pathlib import Path
 import sys
 import tempfile
 import unittest
 
 
+TEST_CONFIG_DIR = Path(__file__).parent.absolute() / 'test'
+
 class TestYamlConfiguration(unittest.TestCase):
     def test_non_existsing_file(self):
         with self.assertRaises(SystemExit):
@@ -15,7 +18,7 @@ def test_missing_field(self):
             aptly_importer.UpdaterConfiguration('test/missing_architectures_field.yaml')
 
     def test_ok(self):
-        self.assertTrue(aptly_importer.UpdaterConfiguration('test/example.yaml'))
+        self.assertTrue(aptly_importer.UpdaterConfiguration(TEST_CONFIG_DIR / 'example.yaml'))
 
 
 class TestAptly(unittest.TestCase):
@@ -126,7 +129,7 @@ def __setup__(self, distros_expected, config_file, snapshot_and_publish=True):
                                          for distro in self.expected_distros]
         self.expected_repos_by_distro_test_name =\
             {f"{distro}": f"ros_bootstrap-{distro}" for distro in self.expected_distros}
-        self.manager = aptly_importer.UpdaterManager(config_file,
+        self.manager = aptly_importer.UpdaterManager(TEST_CONFIG_DIR / config_file,
                                                      debug=self.debug_msgs,
                                                      aptly_config_file=self.aptly_config_file,
                                                      snapshot_and_publish=snapshot_and_publish)
@@ -139,24 +142,24 @@ def test_no_publish_basic_example_creation_from_scratch(self):
         self.__assert_expected_repos_mirrors()
 
     def test_basic_example_creation_from_scratch(self):
-        self.__setup__(['focal', 'groovy'], 'test/example.yaml')
+        self.__setup__(['focal', 'groovy'], 'example.yaml')
         self.assertTrue(self.manager.run())
         self.__assert_expected_repos_mirrors()
 
     def test_basic_example_creation_existing_mirror(self):
-        self.__setup__(['focal', 'groovy'], 'test/example.yaml')
+        self.__setup__(['focal', 'groovy'], 'example.yaml')
         [self.__add_mirror(name) for name in self.expected_mirrors_test_name]
         self.assertTrue(self.manager.run())
         self.__assert_expected_repos_mirrors()
 
     def test_basic_example_creation_existing_repo(self):
-        self.__setup__(['focal', 'groovy'], 'test/example.yaml')
+        self.__setup__(['focal', 'groovy'], 'example.yaml')
         [self.__add_repo(name) for name in self.expected_repos_test_name]
         self.assertTrue(self.manager.run())
         self.__assert_expected_repos_mirrors()
 
     def test_example_no_sources(self):
-        self.__setup__(['xenial'], 'test/example_no_source_package.yaml')
+        self.__setup__(['xenial'], 'example_no_source_package.yaml')
         with self.assertRaises(SystemExit):
             self.manager.run()
         self.__assert_no_mirrors()