Change behaviour to publish all existing repositories

The intended behaviour of the publish is being changed from publishing
repositories affected by the distributions present in the config file to
publish existing snapshots in aptly.

This make easier to rollback back changes since it will make possible to
revert all at once using the latest time-stamp instead of detecting
which ones were affected by the last update.

scripts/aptly/aptly_importer.py

@@ -221,24 +221,48 @@ def __create_aptly_mirror(self, distribution):
         self.aptly.run(['mirror', 'update', mirror_name])
         self.__log_ok(f"mirror {mirror_name} created")
 
-    def __create_aptly_snapshot(self, distribution):
-        self.__log('Creating an aptly snapshot from local aptly repository')
+    def __create_aptly_snapshot_from_repo(self, distribution):
         self.aptly.run(['snapshot', 'create', self.__get_snapshot_name(distribution),
                         'from', 'repo', self.__get_repo_name(distribution)])
         self.__log_ok(f"snapshot {self.__get_snapshot_name(distribution)} created from repo {self.__get_repo_name(distribution)}")
 
+    def __create_aptly_snapshot_from_all_repos(self):
+        for repo in self.__get_all_repos():
+            self.__create_aptly_snapshot_from_repo(self.__get_distro_from_repo_name(repo))
+
     def __error(self, msg):
         print(f"Update Manager error: {msg} \n", file=stderr)
         exit(1)
 
+    def __get_all_repos(self):
+        cmd = ['repo', 'list']
+        run_result = self.aptly.run(cmd, return_all_info=True)
+        if run_result.returncode != 0:
+            self.__error(run_result.stderr.decode('utf-8'))
+
+        all_repos = []
+        for line in run_result.stdout.splitlines():
+            if f"[{self.__get_repo_name_prefix()}-" in line.decode():
+                m = re.findall(r"\[(.*)\] \(packages", line.decode())
+                all_repos.append(m[0])
+        return all_repos
+
+    def __get_distro_from_repo_name(self, repo_name):
+        distro_regexp = re.findall(r'%s-(.*)' % self.__get_repo_name_prefix(), repo_name)
+        assert(len(distro_regexp) == 1)
+        return distro_regexp[0]
+
     def __get_endpoint_name(self, distribution):
         return f"filesystem:live:ros_bootstrap"
 
     def __get_mirror_name(self, distribution):
         return f"{self.config.name}-{distribution}"
 
     def __get_repo_name(self, distribution):
-        return f"ros_bootstrap-{distribution}"
+        return f"{self.__get_repo_name_prefix()}-{distribution}"
+
+    def __get_repo_name_prefix(self):
+        return 'ros_bootstrap'
 
     def __get_snapshot_name(self, distribution):
         if not self.snapshot_timestamp:
@@ -271,8 +295,11 @@ def __log(self, msg):
     def __log_ok(self, msg):
         self.__log(f"   [ok] {msg}")
 
+    def __publish_all_new_snapshot(self):
+        for repo in self.__get_all_repos():
+            self.__publish_new_snapshot(self.__get_distro_from_repo_name(repo))
+
     def __publish_new_snapshot(self, dist):
-        self.__log('Publish the new snapshot')
         if (self.aptly.exists_publication(dist, self.__get_endpoint_name(dist))):
             self.aptly.run(['publish', 'switch',
                             dist,
@@ -310,11 +337,14 @@ def run(self):
             if self.simulate_repo_import:
                 self.__log_ok(f"Simulation of the import actions from mirrors to repos finished")
                 exit(0)
-            if self.snapshot_and_publish:
-                # 3. Create snapshots from repositories
-                self.__create_aptly_snapshot(dist)
-                # 4. Publish new snapshots
-                self.__publish_new_snapshot(dist)
+
+        if self.snapshot_and_publish:
+            # 3. Create snapshots from all existing repositories
+            self.__log('Creating snapshots from all repositories')
+            self.__create_aptly_snapshot_from_all_repos()
+            # 4. Publish new snapshots
+            self.__log('Publishing all new snapshots')
+            self.__publish_all_new_snapshot()
         self.__log(f"\n == [ END OF PROCESSING {self.config.name} ] ==\n")
         return True
 

scripts/aptly/aptly_importer_TEST.py

@@ -4,6 +4,7 @@
 import tempfile
 import unittest
 
+
 class TestYamlConfiguration(unittest.TestCase):
     def test_non_existsing_file(self):
         with self.assertRaises(SystemExit):
@@ -64,7 +65,11 @@ def __add_mirror(self, mirror_name):
         self.assertTrue(self.aptly.run(['mirror', 'create', mirror_name, 'http://packages.osrfoundation.org/gazebo/ubuntu-stable', 'focal']))
 
     def __add_repo(self, repo_name):
-        self.assertTrue(self.aptly.run(['repo', 'create', repo_name]))
+        self.assertTrue(self.aptly.run([
+            'repo',
+            'create',
+            f"-architectures={','.join(self.manager.config.architectures)}",
+            repo_name]))
 
     def __assert_no_mirrors(self):
         for name in self.expected_mirrors_test_name:
@@ -88,6 +93,8 @@ def __clean_up_aptly_test_artifacts(self):
         [self.__remove_repo(name) for name in self.expected_repos_test_name]
         [self.__remove_mirror(name) for name in self.expected_mirrors_test_name]
         [self.__remove_snapshots_from_mirror(name) for name in self.expected_mirrors_test_name]
+        # extra artifacts in some tests
+        self.__remove_repo('ros_bootstrap-xenial')
         self.aptly.run(['db', 'cleanup'])
 
     def __remove_mirror(self, mirror_name):
@@ -110,7 +117,7 @@ def __remove_snapshots_from_mirror(self, mirror_name):
         for snap in self.aptly.get_snapshots_from_mirror(mirror_name):
             self.aptly.run(['snapshot', 'drop', snap])
 
-    def __setup__(self, distros_expected, config_file):
+    def __setup__(self, distros_expected, config_file, snapshot_and_publish=True):
         self.expected_distros = distros_expected
         self.expected_endpoint_name = 'filesystem:live:ros_bootstrap'
         self.expected_mirrors_test_name = [f"_reprepro_updater_test_suite_-{distro}"
@@ -122,10 +129,15 @@ def __setup__(self, distros_expected, config_file):
         self.manager = aptly_importer.UpdaterManager(config_file,
                                                      debug=self.debug_msgs,
                                                      aptly_config_file=self.aptly_config_file,
-                                                     snapshot_and_publish=True)
+                                                     snapshot_and_publish=snapshot_and_publish)
         # clean up testing artifacts if they previously exists
         self.__clean_up_aptly_test_artifacts()
 
+    def test_no_publish_basic_example_creation_from_scratch(self):
+        self.__setup__(['focal', 'groovy'], 'test/example.yaml', snapshot_and_publish=False)
+        self.assertTrue(self.manager.run())
+        self.__assert_expected_repos_mirrors()
+
     def test_basic_example_creation_from_scratch(self):
         self.__setup__(['focal', 'groovy'], 'test/example.yaml')
         self.assertTrue(self.manager.run())
@@ -149,6 +161,25 @@ def test_example_no_sources(self):
             self.manager.run()
         self.__assert_no_mirrors()
 
+    def test_publish_existing_repos_in_config(self):
+        self.__setup__(['focal', 'groovy'], 'test/example.yaml')
+        # add existing repositories
+        self.__add_repo('ros_bootstrap-focal')
+        self.assertTrue(self.manager.run())
+        self.__assert_expected_repos_mirrors()
+
+    def test_publish_existing_repos_not_in_config(self):
+        self.__setup__(['focal', 'groovy'], 'test/example.yaml')
+        # add existing repositories
+        self.__add_repo('ros_bootstrap-xenial')
+        [self.__add_repo(name) for name in self.expected_repos_test_name]
+        # publishing neeeds custom architectures parameter to work nicely on created xenial
+        # repo. We expect failure to indicate the xenial is in the mix which is the expected
+        # behaviour. To solve the problem extra support needs to be added to aptly_imported
+        # to declare architectures
+        with self.assertRaises(SystemExit):
+            self.assertTrue(self.manager.run())
+
 
 class TestReprepro2AptlyFilter(unittest.TestCase):
     def setUp(self):