diff --git a/scripts/aptly/aptly_importer.py b/scripts/aptly/aptly_importer.py index 255bab2d..a18fb49a 100644 --- a/scripts/aptly/aptly_importer.py +++ b/scripts/aptly/aptly_importer.py @@ -221,16 +221,37 @@ def __create_aptly_mirror(self, distribution): self.aptly.run(['mirror', 'update', mirror_name]) self.__log_ok(f"mirror {mirror_name} created") - def __create_aptly_snapshot(self, distribution): - self.__log('Creating an aptly snapshot from local aptly repository') + def __create_aptly_snapshot_from_repo(self, distribution): self.aptly.run(['snapshot', 'create', self.__get_snapshot_name(distribution), 'from', 'repo', self.__get_repo_name(distribution)]) self.__log_ok(f"snapshot {self.__get_snapshot_name(distribution)} created from repo {self.__get_repo_name(distribution)}") + def __create_aptly_snapshot_from_all_repos(self): + for repo in self.__get_all_repos(): + self.__create_aptly_snapshot_from_repo(self.__get_distro_from_repo_name(repo)) + def __error(self, msg): print(f"Update Manager error: {msg} \n", file=stderr) exit(1) + def __get_all_repos(self): + cmd = ['repo', 'list'] + run_result = self.aptly.run(cmd, return_all_info=True) + if run_result.returncode != 0: + self.__error(run_result.stderr.decode('utf-8')) + + all_repos = [] + for line in run_result.stdout.splitlines(): + if f"[{self.__get_repo_name_prefix()}-" in line.decode(): + m = re.findall(r"\[(.*)\] \(packages", line.decode()) + all_repos.append(m[0]) + return all_repos + + def __get_distro_from_repo_name(self, repo_name): + distro_regexp = re.findall(r'%s-(.*)' % self.__get_repo_name_prefix(), repo_name) + assert(len(distro_regexp) == 1) + return distro_regexp[0] + def __get_endpoint_name(self, distribution): return f"filesystem:live:ros_bootstrap" @@ -238,7 +259,10 @@ def __get_mirror_name(self, distribution): return f"{self.config.name}-{distribution}" def __get_repo_name(self, distribution): - return f"ros_bootstrap-{distribution}" + return f"{self.__get_repo_name_prefix()}-{distribution}" + + def __get_repo_name_prefix(self): + return 'ros_bootstrap' def __get_snapshot_name(self, distribution): if not self.snapshot_timestamp: @@ -271,8 +295,11 @@ def __log(self, msg): def __log_ok(self, msg): self.__log(f" [ok] {msg}") + def __publish_all_new_snapshot(self): + for repo in self.__get_all_repos(): + self.__publish_new_snapshot(self.__get_distro_from_repo_name(repo)) + def __publish_new_snapshot(self, dist): - self.__log('Publish the new snapshot') if (self.aptly.exists_publication(dist, self.__get_endpoint_name(dist))): self.aptly.run(['publish', 'switch', dist, @@ -310,11 +337,14 @@ def run(self): if self.simulate_repo_import: self.__log_ok(f"Simulation of the import actions from mirrors to repos finished") exit(0) - if self.snapshot_and_publish: - # 3. Create snapshots from repositories - self.__create_aptly_snapshot(dist) - # 4. Publish new snapshots - self.__publish_new_snapshot(dist) + + if self.snapshot_and_publish: + # 3. Create snapshots from all existing repositories + self.__log('Creating snapshots from all repositories') + self.__create_aptly_snapshot_from_all_repos() + # 4. Publish new snapshots + self.__log('Publishing all new snapshots') + self.__publish_all_new_snapshot() self.__log(f"\n == [ END OF PROCESSING {self.config.name} ] ==\n") return True diff --git a/scripts/aptly/aptly_importer_TEST.py b/scripts/aptly/aptly_importer_TEST.py index 1aa654bb..d7eacd1a 100644 --- a/scripts/aptly/aptly_importer_TEST.py +++ b/scripts/aptly/aptly_importer_TEST.py @@ -4,6 +4,7 @@ import tempfile import unittest + class TestYamlConfiguration(unittest.TestCase): def test_non_existsing_file(self): with self.assertRaises(SystemExit): @@ -64,7 +65,11 @@ def __add_mirror(self, mirror_name): self.assertTrue(self.aptly.run(['mirror', 'create', mirror_name, 'http://packages.osrfoundation.org/gazebo/ubuntu-stable', 'focal'])) def __add_repo(self, repo_name): - self.assertTrue(self.aptly.run(['repo', 'create', repo_name])) + self.assertTrue(self.aptly.run([ + 'repo', + 'create', + f"-architectures={','.join(self.manager.config.architectures)}", + repo_name])) def __assert_no_mirrors(self): for name in self.expected_mirrors_test_name: @@ -88,6 +93,8 @@ def __clean_up_aptly_test_artifacts(self): [self.__remove_repo(name) for name in self.expected_repos_test_name] [self.__remove_mirror(name) for name in self.expected_mirrors_test_name] [self.__remove_snapshots_from_mirror(name) for name in self.expected_mirrors_test_name] + # extra artifacts in some tests + self.__remove_repo('ros_bootstrap-xenial') self.aptly.run(['db', 'cleanup']) def __remove_mirror(self, mirror_name): @@ -110,7 +117,7 @@ def __remove_snapshots_from_mirror(self, mirror_name): for snap in self.aptly.get_snapshots_from_mirror(mirror_name): self.aptly.run(['snapshot', 'drop', snap]) - def __setup__(self, distros_expected, config_file): + def __setup__(self, distros_expected, config_file, snapshot_and_publish=True): self.expected_distros = distros_expected self.expected_endpoint_name = 'filesystem:live:ros_bootstrap' self.expected_mirrors_test_name = [f"_reprepro_updater_test_suite_-{distro}" @@ -122,10 +129,15 @@ def __setup__(self, distros_expected, config_file): self.manager = aptly_importer.UpdaterManager(config_file, debug=self.debug_msgs, aptly_config_file=self.aptly_config_file, - snapshot_and_publish=True) + snapshot_and_publish=snapshot_and_publish) # clean up testing artifacts if they previously exists self.__clean_up_aptly_test_artifacts() + def test_no_publish_basic_example_creation_from_scratch(self): + self.__setup__(['focal', 'groovy'], 'test/example.yaml', snapshot_and_publish=False) + self.assertTrue(self.manager.run()) + self.__assert_expected_repos_mirrors() + def test_basic_example_creation_from_scratch(self): self.__setup__(['focal', 'groovy'], 'test/example.yaml') self.assertTrue(self.manager.run()) @@ -149,6 +161,25 @@ def test_example_no_sources(self): self.manager.run() self.__assert_no_mirrors() + def test_publish_existing_repos_in_config(self): + self.__setup__(['focal', 'groovy'], 'test/example.yaml') + # add existing repositories + self.__add_repo('ros_bootstrap-focal') + self.assertTrue(self.manager.run()) + self.__assert_expected_repos_mirrors() + + def test_publish_existing_repos_not_in_config(self): + self.__setup__(['focal', 'groovy'], 'test/example.yaml') + # add existing repositories + self.__add_repo('ros_bootstrap-xenial') + [self.__add_repo(name) for name in self.expected_repos_test_name] + # publishing neeeds custom architectures parameter to work nicely on created xenial + # repo. We expect failure to indicate the xenial is in the mix which is the expected + # behaviour. To solve the problem extra support needs to be added to aptly_imported + # to declare architectures + with self.assertRaises(SystemExit): + self.assertTrue(self.manager.run()) + class TestReprepro2AptlyFilter(unittest.TestCase): def setUp(self):