From b4b99c5cc05aadd2e2c579c40f9ba8d0a3a07805 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:34:45 +0100 Subject: [PATCH] Allow the Symlinking for log/job --- changes.d/feat.6168.md | 1 + cylc/flow/cfgspec/globalcfg.py | 65 +++++---------------- cylc/flow/pathutil.py | 11 +++- tests/functional/cylc-install/01-symlinks.t | 11 +++- tests/unit/test_pathutil.py | 2 + 5 files changed, 38 insertions(+), 52 deletions(-) create mode 100644 changes.d/feat.6168.md diff --git a/changes.d/feat.6168.md b/changes.d/feat.6168.md new file mode 100644 index 00000000000..84ccc73b236 --- /dev/null +++ b/changes.d/feat.6168.md @@ -0,0 +1 @@ +Allow symlinking log/job separately from log diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index db8c7bdd5ff..ef14d0db817 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -30,6 +30,7 @@ from cylc.flow.exceptions import GlobalConfigError from cylc.flow.hostuserutil import get_user_home from cylc.flow.network.client_factory import CommsMeth +from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS from cylc.flow.parsec.config import ( ConfigNode as Conf, ParsecConfig, @@ -1148,55 +1149,21 @@ def default_for( .. versionadded:: 8.0.0 """) - Conf('log', VDR.V_STRING, None, desc=""" - Alternative location for the log dir. - - If specified the workflow log directory will be created in - ``/cylc-run//log`` and a - symbolic link will be created from - ``$HOME/cylc-run//log``. If not specified - the workflow log directory will be created in - ``$HOME/cylc-run//log``. - - .. versionadded:: 8.0.0 - """) - Conf('share', VDR.V_STRING, None, desc=""" - Alternative location for the share dir. - - If specified the workflow share directory will be - created in ``/cylc-run//share`` - and a symbolic link will be created from - ``<$HOME/cylc-run//share``. If not specified - the workflow share directory will be created in - ``$HOME/cylc-run//share``. - - .. versionadded:: 8.0.0 - """) - Conf('share/cycle', VDR.V_STRING, None, desc=""" - Alternative directory for the share/cycle dir. - - If specified the workflow share/cycle directory - will be created in - ``/cylc-run//share/cycle`` - and a symbolic link will be created from - ``$HOME/cylc-run//share/cycle``. If not - specified the workflow share/cycle directory will be - created in ``$HOME/cylc-run//share/cycle``. - - .. versionadded:: 8.0.0 - """) - Conf('work', VDR.V_STRING, None, desc=""" - Alternative directory for the work dir. - - If specified the workflow work directory will be created in - ``/cylc-run//work`` and a - symbolic link will be created from - ``$HOME/cylc-run//work``. If not specified - the workflow work directory will be created in - ``$HOME/cylc-run//work``. - - .. versionadded:: 8.0.0 - """) + for folder, versionadded in SYMLINKABLE_LOCATIONS.items(): + Conf(folder, VDR.V_STRING, None, desc=f""" + Alternative location for the log dir. + + If specified the workflow {folder} directory will + be created in + ``/cylc-run//{folder}`` + and a symbolic link will be created from + ``$HOME/cylc-run//{folder}``. + If not specified the workflow log directory will + be created in + ``$HOME/cylc-run//{folder}``. + + .. versionadded:: {versionadded} + """) with Conf('platforms', desc=''' Platforms allow you to define compute resources available at your site. diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index aa097a2e680..904b73e61b9 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -45,7 +45,14 @@ """Matches relative paths that are explicit (starts with ./)""" SHELL_ENV_VARS = re.compile(r'\$[^$/]*') - +SYMLINKABLE_LOCATIONS: Dict[str, str] = { + # Location: Version Added + 'log': '8.0.0', + 'log/job': '8.4.0', + 'share': '8.0.0', + 'share/cycle': '8.0.0', + 'work': '8.0.0' +} def expand_path(*args: Union[Path, str]) -> str: """Expand both vars and user in path and normalise it, joining any @@ -234,7 +241,7 @@ def get_dirs_to_symlink( if base_dir: dirs_to_symlink['run'] = os.path.join( base_dir, 'cylc-run', workflow_id) - for dir_ in ['log', 'share', 'share/cycle', 'work']: + for dir_ in SYMLINKABLE_LOCATIONS.keys(): link = symlink_conf[install_target].get(dir_, None) if (not link) or link == base_dir: continue diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t index 8fbcc66c1ca..2f55b01d8b0 100644 --- a/tests/functional/cylc-install/01-symlinks.t +++ b/tests/functional/cylc-install/01-symlinks.t @@ -24,7 +24,7 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' fi -set_test_number 30 +set_test_number 32 create_test_global_config "" " [install] @@ -33,6 +33,7 @@ create_test_global_config "" " run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir share = \$TMPDIR/\$USER/test_cylc_symlink/ log = \$TMPDIR/\$USER/test_cylc_symlink/ + log/job = \$TMPDIR/\$USER/test_cylc_symlink/job_log_dir share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir work = \$TMPDIR/\$USER/test_cylc_symlink/ " @@ -53,6 +54,10 @@ TEST_SYM="${TEST_NAME_BASE}-share-cycle-glblcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" +TEST_SYM="${TEST_NAME_BASE}-log-job-glblcfg" +run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job" + for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME_BASE}-${DIR}-glbcfg" run_ok "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ @@ -137,6 +142,10 @@ TEST_SYM="${TEST_NAME}-share-cycle" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/share/cycle")" \ = "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/share/cycle" +TEST_SYM="${TEST_NAME}-log-job" +run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/log/job")" \ + = "$TMPDIR/${USER}/test_cylc_symlink/job_log_dir/cylc-run/${RND_WORKFLOW_NAME}/run1/log/job" + for DIR in 'work' 'share' 'log'; do TEST_SYM="${TEST_NAME}-${DIR}" run_fail "${TEST_SYM}" test "$(readlink "${WORKFLOW_RUN_DIR}/${DIR}")" \ diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py index 6ed9e9ec120..24cc3999400 100644 --- a/tests/unit/test_pathutil.py +++ b/tests/unit/test_pathutil.py @@ -203,6 +203,7 @@ def test_make_workflow_run_tree( run = $DEE work = $DAH log = $DUH + log/job =$RAY share = $DOH share/cycle = $DAH ''', @@ -210,6 +211,7 @@ def test_make_workflow_run_tree( 'run': '$DEE/cylc-run/morpheus', 'work': '$DAH/cylc-run/morpheus/work', 'log': '$DUH/cylc-run/morpheus/log', + 'log/job': '$RAY/cylc-run/morpheus/log/job', 'share': '$DOH/cylc-run/morpheus/share', 'share/cycle': '$DAH/cylc-run/morpheus/share/cycle' },