From 044648ebb14cd36a0f3b681c8c28c4d12b55986a Mon Sep 17 00:00:00 2001 From: Alcolo47 Date: Fri, 27 Sep 2019 19:05:38 +0200 Subject: [PATCH 1/2] Adding cloning.workspace_type option --- docs/source/cloning.rst | 31 +++++++++ src/cosmic_ray/cloning.py | 111 ++++++++++++++++++------------ src/cosmic_ray/config.py | 5 ++ src/cosmic_ray/execution/local.py | 8 ++- 4 files changed, 109 insertions(+), 46 deletions(-) diff --git a/docs/source/cloning.rst b/docs/source/cloning.rst index 124c8e0b..e01d9cea 100644 --- a/docs/source/cloning.rst +++ b/docs/source/cloning.rst @@ -16,6 +16,37 @@ You can configure cloning in your configuration TOML in the `cosmic-ray.cloning` section. At a minimum, you must have a `cosmic-ray.cloning.method` entry in your config. +Possible entry into `[cosmic-ray.cloning]` are: + +- `workspace_type` +- `method` +- `command` + +workspace_type +============== + +The `workspace_type` entry is the name of workspace method creation: +Possible names are: + +- `cloned`: Workspace is built by cloning your sources with the `copy` method + (see bellow). Useful if your sources can be run in-place. +- `cloned_with_virtualenv`: Workspace is built by cloning your sources like + previous method and create an local virtualenv environment locally. + Useful to deploy your sources. + +:: + + [cosmic-ray.cloning] + workspace_type = 'cloned' + +This entry is optional. For compatibility reason, the default is +`cloned_with_virtualenv`. + +After This workspace deployent, `command` will be executed (see bellow). + +copy +==== + The "copy" cloning method simple copies an entire filesystem directory tree. You can use configure it like this:: diff --git a/src/cosmic_ray/cloning.py b/src/cosmic_ray/cloning.py index 4f1bb971..27eb0478 100644 --- a/src/cosmic_ray/cloning.py +++ b/src/cosmic_ray/cloning.py @@ -1,13 +1,14 @@ """Support for making clones of projects for test isolation. """ - -import contextlib +import abc import logging import os from pathlib import Path import shutil import subprocess import tempfile +from typing import Type + import virtualenv import git @@ -17,69 +18,68 @@ log = logging.getLogger(__name__) -@contextlib.contextmanager -def cloned_workspace(clone_config, chdir=True): - """Create a cloned workspace and yield it. +class WorkspaceMeta(type): + _registred = {} - This creates a workspace for a with-block and cleans it up on exit. By - default, this will also change to the workspace's `clone_dir` for the - duration of the with-block. + def __new__(cls, *args, **kwargs): + new_class = super(WorkspaceMeta, cls).__new__(cls, *args, **kwargs) # type: Type[Workspace] + cls._registred[new_class.name] = new_class + return new_class - Args: - clone_config: The execution engine configuration to use for the workspace. - chdir: Whether to change to the workspace's `clone_dir` before entering the with-block. + @classmethod + def get_workspace_class(cls, name): + return cls._registred[name] - Yields: The `CloneWorkspace` instance created for the context. - """ - workspace = ClonedWorkspace(clone_config) - original_dir = os.getcwd() - if chdir: - os.chdir(workspace.clone_dir) - try: - yield workspace - finally: - os.chdir(original_dir) - workspace.cleanup() +class Workspace(metaclass=WorkspaceMeta): + name = None + @classmethod + def get_workspace(cls, name, clone_config) -> 'Workspace': + return type(cls).get_workspace_class(name)(clone_config) -class ClonedWorkspace: - """Clone a project and install it into a temporary virtual environment. + @abc.abstractmethod + def cleanup(self): + pass - Note that this actually *activates* the virtual environment, so don't construct one - of these unless you want that to happen in your process. + @property + @abc.abstractmethod + def clone_dir(self): + pass + + +class ClonedWorkspace(Workspace): + """Clone a project into a temporary directory. """ + name = 'cloned' def __init__(self, clone_config): self._tempdir = tempfile.TemporaryDirectory() - log.info('New project clone in %s', self._tempdir.name) + self._prepare_directory(clone_config) + self._load_environment() + self._run_commands(clone_config.get('commands', ())) + def _prepare_directory(self, clone_config): + log.info('New project clone in %s', self._tempdir.name) self._clone_dir = str(Path(self._tempdir.name) / 'repo') - if clone_config['method'] == 'git': + method = clone_config['method'] + if method == 'git': _clone_with_git( clone_config.get('repo-uri', '.'), self._clone_dir) - elif clone_config['method'] == 'copy': + + elif method == 'copy': _clone_with_copy( os.getcwd(), self._clone_dir) - # pylint: disable=fixme - # TODO: We should allow user to specify which version of Python to use. - # How? The EnvBuilder could be passed a path to a python interpreter - # which is used in the call to pip. This path would need to come from - # the config. + else: + raise Exception("Clone method '%s' unknown" % method) - # Install into venv - self._venv_path = Path(self._tempdir.name) / 'venv' - log.info('Creating virtual environment in %s', self._venv_path) - virtualenv.create_environment(str(self._venv_path)) - - _activate(self._venv_path) - _install_sitecustomize(self._venv_path) - - self._run_commands(clone_config.get('commands', ())) + def _load_environment(self): + os.environ['PYTHONPATH'] = \ + '%s:%s' % (self.clone_dir, os.environ.get('PYTHONPATH', '')) @property def clone_dir(self): @@ -113,6 +113,31 @@ def _run_commands(self, commands): command, exc.output) +class ClonedWorkspaceWithVirtualenv(ClonedWorkspace): + """Clone a project and install it into a temporary virtual environment. + + Note that this actually *activates* the virtual environment, so don't construct one + of these unless you want that to happen in your process. + """ + name = 'cloned_with_virtualenv' + + def _prepare_directory(self, clone_config): + # pylint: disable=fixme + # TODO: We should allow user to specify which version of Python to use. + # How? The EnvBuilder could be passed a path to a python interpreter + # which is used in the call to pip. This path would need to come from + # the config. + + # Install into venv + self._venv_path = Path(self._tempdir.name) / 'venv' + log.info('Creating virtual environment in %s', self._venv_path) + virtualenv.create_environment(str(self._venv_path)) + + def _load_environment(self): + _activate(self._venv_path) + _install_sitecustomize(self._venv_path) + + def _clone_with_git(repo_uri, dest_path): """Create a clone by cloning a git repository. diff --git a/src/cosmic_ray/config.py b/src/cosmic_ray/config.py index f8f2d39e..27beb21f 100644 --- a/src/cosmic_ray/config.py +++ b/src/cosmic_ray/config.py @@ -107,6 +107,11 @@ def cloning_config(self): "The 'cloning' section of the config." return self['cloning'] + @property + def cloning_config_workspace_type(self): + "The 'workspace' section of the cloning section" + return self.cloning_config.get('workspace_type', 'cloned_with_virtualenv') + @contextmanager def _config_stream(filename): diff --git a/src/cosmic_ray/execution/local.py b/src/cosmic_ray/execution/local.py index ec3b7491..6a511a47 100644 --- a/src/cosmic_ray/execution/local.py +++ b/src/cosmic_ray/execution/local.py @@ -41,7 +41,8 @@ import multiprocessing.util import os -from cosmic_ray.cloning import ClonedWorkspace +from cosmic_ray.cloning import ClonedWorkspaceWithVirtualenv, Workspace +from cosmic_ray.config import ConfigDict from cosmic_ray.execution.execution_engine import ExecutionEngine from cosmic_ray.worker import worker @@ -62,7 +63,7 @@ def excursion(dirname): os.chdir(orig) -def _initialize_worker(config): +def _initialize_worker(config: ConfigDict): # pylint: disable=global-statement global _workspace global _config @@ -72,7 +73,8 @@ def _initialize_worker(config): _config = config log.info('Initialize local-git worker in PID %s', os.getpid()) - _workspace = ClonedWorkspace(config.cloning_config) + _workspace = Workspace.get_workspace(config.cloning_config_workspace_type, + config.cloning_config) # Register a finalizer multiprocessing.util.Finalize(_workspace, _workspace.cleanup, exitpriority=16) From b54359fb50a6efd0c306f13619bc30af3c6629a3 Mon Sep 17 00:00:00 2001 From: Alcolo47 Date: Fri, 27 Sep 2019 19:11:07 +0200 Subject: [PATCH 2/2] updating new_config --- src/cosmic_ray/commands/new_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cosmic_ray/commands/new_config.py b/src/cosmic_ray/commands/new_config.py index 4a4cfc93..986a50d5 100644 --- a/src/cosmic_ray/commands/new_config.py +++ b/src/cosmic_ray/commands/new_config.py @@ -90,6 +90,7 @@ def new_config(): config['execution-engine']['name'] = menu.show(header="Execution engine", returns="desc") config["cloning"] = ConfigDict() + config['cloning']['workspace_type'] = 'cloned_with_virtualenv' config['cloning']['method'] = 'copy' config['cloning']['commands'] = []