diff --git a/.deepsource.toml b/.deepsource.toml index 0ac63d33..c9e3b509 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -4,4 +4,4 @@ version = 1 name = "python" [analyzers.meta] - runtime_version = "3.x.x" \ No newline at end of file + runtime_version = "3.x.x" diff --git a/.gitignore b/.gitignore index fb444e9d..534976a7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ logs/ *.tar.gz *.txt miniconda/ +miniforge/ quantities_map/ output/ CROWN/ diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 040b4c1c..00000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -ml_configs diff --git a/.gitmodules b/.gitmodules index fb45fdeb..b9b3db6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,4 +6,7 @@ url = git@github.com:KIT-CMS/KingMaker_sample_database.git [submodule "sm-htt-analysis"] path = sm-htt-analysis - url = git@github.com:tvoigtlaender/sm-htt-analysis.git \ No newline at end of file + url = git@github.com:tvoigtlaender/sm-htt-analysis.git +[submodule "kingmaker-images"] + path = kingmaker-images + url = git@github.com:KIT-CMS/kingmaker-images.git diff --git a/README.md b/README.md index 130cae5b..68da77c0 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ source setup.sh this should setup the environment specified in the luigi.cfg file (located at `lawluigi_configs/_luigi.cfg`), which includes all needed packages. The environment is sourced from the conda instance located at `/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/` if possible. If the relevant environment is not available this way, the environment will be set up in a local conda instance. -The environment files are located at `conda_environments/_env.cfg`. +The environment files are located at `kingmaker-images/KingMaker_envs/_env.cfg`. In addition other files are installed depending on the analysis. A list of available analyses can be found in the `setup.sh` script or by running diff --git a/conda_environments/BaseWRoot_env.yml b/conda_environments/BaseWRoot_env.yml deleted file mode 100644 index 6393ec3a..00000000 --- a/conda_environments/BaseWRoot_env.yml +++ /dev/null @@ -1,256 +0,0 @@ -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_gnu - - afterimage=1.21=ha9998ff_1003 - - argon2-cffi=21.3.0=pyhd8ed1ab_0 - - argon2-cffi-bindings=21.2.0=py39hb9d737c_2 - - asttokens=2.0.5=pyhd8ed1ab_0 - - atk-1.0=2.36.0=h3371d22_4 - - attrs=21.4.0=pyhd8ed1ab_0 - - backcall=0.2.0=pyh9f0ad1d_0 - - backports=1.0=py_2 - - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 - - beautifulsoup4=4.11.1=pyha770c72_0 - - binutils=2.36.1=hdd6e379_2 - - binutils_impl_linux-64=2.36.1=h193b22a_2 - - binutils_linux-64=2.36=hf3e587d_9 - - bleach=5.0.0=pyhd8ed1ab_0 - - boost=1.74.0=py39h5472131_5 - - boost-cpp=1.74.0=h312852a_4 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.18.1=h7f98852_0 - - c-compiler=1.3.0=h7f98852_0 - - ca-certificates=2021.10.8=ha878542_0 - - cairo=1.16.0=h6cf1ce9_1008 - - cffi=1.15.0=py39h4bc2ebd_0 - - cfitsio=3.470=hb418390_7 - - cgsi-gsoap=1.3.11=h876a3cc_2 - - colorama=0.4.4=pyh9f0ad1d_0 - - commonmark=0.9.1=py_0 - - compilers=1.3.0=ha770c72_0 - - conda-pack=0.7.0=pyh6c4a22f_0 - - cxx-compiler=1.3.0=h4bd325d_0 - - cyrus-sasl=2.1.27=h230043b_5 - - dataclasses=0.8=pyhc8e2a94_3 - - davix=0.7.6=hb44b51d_0 - - dcap=2.47.12=hd35416c_3 - - debugpy=1.6.0=py39h5a03fae_0 - - decorator=5.1.1=pyhd8ed1ab_0 - - defusedxml=0.7.1=pyhd8ed1ab_0 - - docutils=0.18.1=py39hf3d152e_1 - - entrypoints=0.4=pyhd8ed1ab_0 - - executing=0.8.3=pyhd8ed1ab_0 - - expat=2.4.8=h27087fc_0 - - fftw=3.3.10=nompi_h77c792f_102 - - flit-core=3.7.1=pyhd8ed1ab_0 - - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - - font-ttf-inconsolata=3.000=h77eed37_0 - - font-ttf-source-code-pro=2.038=h77eed37_0 - - font-ttf-ubuntu=0.83=hab24e00_0 - - fontconfig=2.14.0=h8e229c2_0 - - fonts-conda-ecosystem=1=0 - - fonts-conda-forge=1=0 - - fortran-compiler=1.3.0=h1990efc_0 - - freetype=2.10.4=h0708190_1 - - fribidi=1.0.10=h36c2ea0_0 - - ftgl=2.4.0=hbcb1f35_0 - - future=0.18.2=py39hf3d152e_5 - - gcc=9.4.0=h192d537_9 - - gcc_impl_linux-64=9.4.0=h03d3576_16 - - gcc_linux-64=9.4.0=h391b98a_9 - - gct=6.2.1550507116=h812cca2_2 - - gdk-pixbuf=2.42.6=h04a7f16_0 - - gettext=0.19.8.1=h73d1719_1008 - - gfal2=2.19.2=h13c04c2_0 - - gfal2-util=1.7.1=pyhd8ed1ab_0 - - gfortran=9.4.0=h2018a41_9 - - gfortran_impl_linux-64=9.4.0=h0003116_16 - - gfortran_linux-64=9.4.0=hf0ab688_9 - - giflib=5.2.1=h36c2ea0_2 - - gl2ps=1.4.2=h0708190_0 - - glew=2.1.0=h9c3ff4c_2 - - glib=2.68.4=h9c3ff4c_1 - - glib-tools=2.68.4=h9c3ff4c_1 - - graphite2=1.3.13=h58526e2_1001 - - graphviz=2.49.1=h85b4f2f_0 - - gsl=2.7=he838d99_0 - - gsoap=2.8.119=h90a1d37_0 - - gsoap_abi=2.8.106=1 - - gtk2=2.24.33=h539f30e_1 - - gts=0.7.6=h64030ff_2 - - gxx=9.4.0=h192d537_9 - - gxx_impl_linux-64=9.4.0=h03d3576_16 - - gxx_linux-64=9.4.0=h0316aca_9 - - harfbuzz=3.0.0=h83ec7ef_1 - - icu=68.2=h9c3ff4c_0 - - importlib-metadata=4.11.3=py39hf3d152e_1 - - importlib_resources=5.7.1=pyhd8ed1ab_0 - - ipykernel=6.13.0=py39hef51801_0 - - ipyparallel=8.2.1=pyhd8ed1ab_0 - - ipython=8.2.0=py39hf3d152e_0 - - ipython_genutils=0.2.0=py_1 - - jbig=2.1=h7f98852_2003 - - jedi=0.18.1=py39hf3d152e_1 - - jinja2=3.1.1=pyhd8ed1ab_0 - - jpeg=9e=h166bdaf_1 - - json-c=0.15=h98cffda_0 - - jsonschema=4.4.0=pyhd8ed1ab_0 - - jupyter_client=7.3.0=pyhd8ed1ab_0 - - jupyter_core=4.9.2=py39hf3d152e_0 - - jupyterlab_pygments=0.2.2=pyhd8ed1ab_0 - - kernel-headers_linux-64=2.6.32=he073ed8_15 - - keyutils=1.6.1=h166bdaf_0 - - krb5=1.19.3=h3790be6_0 - - ld_impl_linux-64=2.36.1=hea4e1c9_2 - - lerc=3.0=h9c3ff4c_0 - - libblas=3.9.0=14_linux64_openblas - - libcblas=3.9.0=14_linux64_openblas - - libcurl=7.82.0=h7bff187_0 - - libcxx=12.0.1=h4bd325d_1 - - libcxxabi=12.0.1=ha770c72_1 - - libdeflate=1.10=h7f98852_0 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-devel_linux-64=9.4.0=hd854feb_16 - - libgcc-ng=11.2.0=h1d223b6_16 - - libgd=2.3.3=h6ad9fb6_0 - - libgfortran-ng=11.2.0=h69a702a_16 - - libgfortran5=11.2.0=h5c6108e_16 - - libglib=2.68.4=h174f98d_1 - - libglu=9.0.0=he1b5a44_1001 - - libgomp=11.2.0=h1d223b6_16 - - libiconv=1.16=h516909a_0 - - liblapack=3.9.0=14_linux64_openblas - - libllvm11=11.1.0=hf817b99_3 - - libllvm9=9.0.1=default_hc23dcda_7 - - libnghttp2=1.47.0=h727a467_0 - - libnsl=2.0.0=h7f98852_0 - - libntlm=1.4=h7f98852_1002 - - libopenblas=0.3.20=pthreads_h78a6416_0 - - libpng=1.6.37=h21135ba_2 - - librsvg=2.52.2=hc3c00ef_0 - - libsanitizer=9.4.0=h79bfe98_16 - - libsodium=1.0.18=h36c2ea0_1 - - libssh2=1.10.0=ha56f1ee_2 - - libstdcxx-devel_linux-64=9.4.0=hd854feb_16 - - libstdcxx-ng=11.2.0=he4da1e4_16 - - libtiff=4.3.0=h542a066_3 - - libtool=2.4.6=h9c3ff4c_1008 - - libuuid=2.32.1=h7f98852_1000 - - libwebp=1.2.2=h3452ae3_0 - - libwebp-base=1.2.2=h7f98852_1 - - libxcb=1.13=h7f98852_1004 - - libxml2=2.9.12=h72842e0_0 - - libzlib=1.2.11=h166bdaf_1014 - - llvmlite=0.38.0=py39h7d9a04d_1 - - lockfile=0.12.2=py_1 - - luigi=3.0.3=py39hf3d152e_2 - - lz4-c=1.9.3=h9c3ff4c_1 - - markupsafe=2.1.1=py39hb9d737c_1 - - matplotlib-inline=0.1.3=pyhd8ed1ab_0 - - metakernel=0.29.0=pyhd8ed1ab_0 - - mistune=0.8.4=py39h3811e60_1005 - - nbclient=0.6.0=pyhd8ed1ab_0 - - nbconvert=6.5.0=pyhd8ed1ab_0 - - nbconvert-core=6.5.0=pyhd8ed1ab_0 - - nbconvert-pandoc=6.5.0=pyhd8ed1ab_0 - - nbformat=5.3.0=pyhd8ed1ab_0 - - ncurses=6.3=h27087fc_1 - - nest-asyncio=1.5.5=pyhd8ed1ab_0 - - nlohmann_json=3.10.5=h9c3ff4c_0 - - notebook=6.4.11=pyha770c72_0 - - numba=0.55.1=py39h56b8d98_0 - - numpy=1.21.6=py39h18676bf_0 - - openldap=2.4.59=h43732ee_0 - - openssl=1.1.1n=h166bdaf_0 - - packaging=21.3=pyhd8ed1ab_0 - - pandoc=2.18=ha770c72_0 - - pandocfilters=1.5.0=pyhd8ed1ab_0 - - pango=1.48.10=h54213e6_2 - - parso=0.8.3=pyhd8ed1ab_0 - - pcre=8.45=h9c3ff4c_0 - - perl=5.32.1=2_h7f98852_perl5 - - pexpect=4.8.0=pyh9f0ad1d_2 - - pickleshare=0.7.5=py_1003 - - pip=22.0.4=pyhd8ed1ab_0 - - pixman=0.40.0=h36c2ea0_0 - - portalocker=2.4.0=py39hf3d152e_0 - - prometheus_client=0.5.0=py_0 - - prompt-toolkit=3.0.29=pyha770c72_0 - - psutil=5.9.0=py39hb9d737c_1 - - pthread-stubs=0.4=h36c2ea0_1001 - - ptyprocess=0.7.0=pyhd3deb0d_0 - - pugixml=1.11.4=h9c3ff4c_0 - - pure_eval=0.2.2=pyhd8ed1ab_0 - - pycparser=2.21=pyhd8ed1ab_0 - - pygments=2.12.0=pyhd8ed1ab_0 - - pyparsing=3.0.8=pyhd8ed1ab_0 - - pyrsistent=0.18.1=py39hb9d737c_1 - - pythia8=8.305=py39he80948d_0 - - python=3.9.12=h9a8a25e_1_cpython - - python-daemon=2.3.0=pyhd8ed1ab_1 - - python-dateutil=2.8.2=pyhd8ed1ab_0 - - python-fastjsonschema=2.15.3=pyhd8ed1ab_0 - - python-gfal2=1.10.1=py39h4309f8f_0 - - python_abi=3.9=2_cp39 - - pyyaml=6.0=py39hb9d737c_4 - - pyzmq=22.3.0=py39headdf64_2 - - readline=8.1=h46c0cb4_0 - - rich=12.3.0=pyhd8ed1ab_0 - - root=6.24.6=py39hde0f152_0 - - root_base=6.24.6=py39he6e31e0_0 - - scitokens-cpp=0.6.3=h00015f2_2 - - send2trash=1.8.0=pyhd8ed1ab_0 - - setuptools=62.1.0=py39hf3d152e_0 - - six=1.16.0=pyh6c4a22f_0 - - soupsieve=2.3.1=pyhd8ed1ab_0 - - sqlite=3.38.2=h4ff8645_0 - - srm-ifce=1.24.5=h9da80fa_1 - - stack_data=0.2.0=pyhd8ed1ab_0 - - sysroot_linux-64=2.12=he073ed8_15 - - tbb=2021.5.0=h924138e_1 - - tenacity=6.3.1=pyhd8ed1ab_0 - - terminado=0.13.3=py39hf3d152e_1 - - tinycss2=1.1.1=pyhd8ed1ab_0 - - tk=8.6.12=h27826a3_0 - - toml=0.10.2=pyhd8ed1ab_0 - - tornado=6.1=py39hb9d737c_3 - - tqdm=4.64.0=pyhd8ed1ab_0 - - traitlets=5.1.1=pyhd8ed1ab_0 - - typing_extensions=4.2.0=pyha770c72_1 - - tzdata=2022a=h191b570_0 - - vdt=0.4.3=h9c3ff4c_0 - - vector-classes=1.4.2=h4bd325d_1 - - voms=2.1.0rc0=h46dc511_6 - - wcwidth=0.2.5=pyh9f0ad1d_2 - - webencodings=0.5.1=py_1 - - wheel=0.37.1=pyhd8ed1ab_0 - - xorg-fixesproto=5.0=h7f98852_1002 - - xorg-kbproto=1.0.7=h7f98852_1002 - - xorg-libice=1.0.10=h7f98852_0 - - xorg-libsm=1.2.3=hd9c2040_1000 - - xorg-libx11=1.6.12=h36c2ea0_0 - - xorg-libxau=1.0.9=h7f98852_0 - - xorg-libxcursor=1.2.0=h516909a_0 - - xorg-libxdmcp=1.1.3=h7f98852_0 - - xorg-libxext=1.3.4=h516909a_0 - - xorg-libxfixes=5.0.3=h516909a_1004 - - xorg-libxft=2.3.4=hc534e41_1 - - xorg-libxpm=3.5.13=h516909a_0 - - xorg-libxrender=0.9.10=h516909a_1002 - - xorg-libxt=1.1.5=h516909a_1003 - - xorg-renderproto=0.11.1=h7f98852_1002 - - xorg-xextproto=7.3.0=h7f98852_1002 - - xorg-xproto=7.0.31=h7f98852_1007 - - xrootd=5.4.2=py39hbfe9b54_0 - - xxhash=0.8.0=h7f98852_3 - - xz=5.2.5=h516909a_1 - - yaml=0.2.5=h7f98852_2 - - zeromq=4.3.4=h9c3ff4c_1 - - zipp=3.8.0=pyhd8ed1ab_0 - - zlib=1.2.11=h166bdaf_1014 - - zstd=1.5.2=ha95c52a_0 diff --git a/conda_environments/KingMaker_env.yml b/conda_environments/KingMaker_env.yml deleted file mode 100644 index 4617a780..00000000 --- a/conda_environments/KingMaker_env.yml +++ /dev/null @@ -1,106 +0,0 @@ -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_gnu - - boost=1.74.0=py39h5472131_5 - - boost-cpp=1.74.0=h75c5d50_8 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.18.1=h7f98852_0 - - ca-certificates=2022.12.7=ha878542_0 - - cgsi-gsoap=1.3.11=h876a3cc_2 - - conda-pack=0.7.1=pyhd8ed1ab_0 - - curl=7.86.0=h7bff187_1 - - cyrus-sasl=2.1.27=h230043b_5 - - davix=0.7.6=hb44b51d_0 - - dcap=2.47.12=hd35416c_3 - - expat=2.5.0=hcb278e6_1 - - gct=6.2.1550507116=h812cca2_2 - - gettext=0.21.1=h27087fc_0 - - gfal2=2.19.2=h13c04c2_0 - - gfal2-util=1.8.0=pyhd8ed1ab_0 - - git=2.39.1=pl5321ha3eba64_0 - - git-lfs=3.3.0=ha770c72_0 - - glib=2.68.4=h9c3ff4c_1 - - glib-tools=2.68.4=h9c3ff4c_1 - - gsoap=2.8.123=h598caef_0 - - gsoap_abi=2.8.106=1 - - icu=70.1=h27087fc_0 - - json-c=0.15=h98cffda_0 - - keyutils=1.6.1=h166bdaf_0 - - krb5=1.19.3=h3790be6_0 - - ld_impl_linux-64=2.40=h41732ed_0 - - libblas=3.9.0=16_linux64_openblas - - libcblas=3.9.0=16_linux64_openblas - - libcurl=7.86.0=h7bff187_1 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libexpat=2.5.0=hcb278e6_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-ng=12.2.0=h65d4601_19 - - libgfortran-ng=12.2.0=h69a702a_19 - - libgfortran5=12.2.0=h337968e_19 - - libglib=2.68.4=h174f98d_1 - - libgomp=12.2.0=h65d4601_19 - - libiconv=1.17=h166bdaf_0 - - liblapack=3.9.0=16_linux64_openblas - - libnghttp2=1.51.0=hdcd2b5c_0 - - libnsl=2.0.0=h7f98852_0 - - libntlm=1.4=h7f98852_1002 - - libopenblas=0.3.21=pthreads_h78a6416_3 - - libsqlite=3.40.0=h753d276_0 - - libssh2=1.10.0=haa6b8db_3 - - libstdcxx-ng=12.2.0=h46fd767_19 - - libtool=2.4.7=h27087fc_0 - - libuuid=2.38.1=h0b41bf4_0 - - libxml2=2.10.3=hca2bb57_4 - - libzlib=1.2.13=h166bdaf_4 - - ncurses=6.3=h27087fc_1 - - numpy=1.24.2=py39h7360e5f_0 - - openldap=2.4.59=h43732ee_0 - - openssl=1.1.1t=h0b41bf4_0 - - pcre=8.45=h9c3ff4c_0 - - pcre2=10.40=hc3806b6_0 - - perl=5.32.1=2_h7f98852_perl5 - - pip=23.0.1=pyhd8ed1ab_0 - - pugixml=1.11.4=h9c3ff4c_0 - - python=3.9.15=h47a2c10_0_cpython - - python-gfal2=1.10.1=py39h4309f8f_0 - - python_abi=3.9=3_cp39 - - readline=8.2=h8228510_1 - - scitokens-cpp=0.7.1=h0255ffa_0 - - setuptools=67.6.1=pyhd8ed1ab_0 - - sqlite=3.40.0=h4ff8645_0 - - srm-ifce=1.24.5=h9da80fa_1 - - tk=8.6.12=h27826a3_0 - - tzdata=2023c=h71feb2d_0 - - voms=2.1.0rc2=h4fe81f9_7 - - wheel=0.40.0=pyhd8ed1ab_0 - - xrootd=5.3.4=py39hf708bbc_0 - - xz=5.2.6=h166bdaf_0 - - yaml=0.2.5=h7f98852_2 - - zlib=1.2.13=h166bdaf_4 - - zstd=1.5.2=h3eb15da_6 - - pip: - - awkward==2.1.2 - - awkward-cpp==13 - - docutils==0.19 - - lockfile==0.12.2 - - luigi==3.0.3 - - markdown-it-py==2.2.0 - - mdurl==0.1.2 - - packaging==23.0 - - prompt-toolkit==3.0.38 - - pygments==2.15.0 - - python-daemon==3.0.1 - - python-dateutil==2.8.2 - - pyyaml==6.0 - - questionary==1.10.0 - - rich==13.3.3 - - six==1.16.0 - - tenacity==6.3.1 - - tornado==6.2 - - typing-extensions==4.5.0 - - uproot==5.0.6 - - wcwidth==0.2.6 diff --git a/conda_environments/ML_LAW_env.yml b/conda_environments/ML_LAW_env.yml deleted file mode 100644 index 14b96a67..00000000 --- a/conda_environments/ML_LAW_env.yml +++ /dev/null @@ -1,148 +0,0 @@ -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=1_gnu - - boost=1.74.0=py39h5472131_5 - - boost-cpp=1.74.0=h75c5d50_8 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.18.1=h7f98852_0 - - ca-certificates=2021.10.8=ha878542_0 - - cgsi-gsoap=1.3.11=h876a3cc_2 - - colorama=0.4.4=pyh9f0ad1d_0 - - commonmark=0.9.1=py_0 - - conda-pack=0.7.0=pyh6c4a22f_0 - - cyrus-sasl=2.1.27=h230043b_5 - - dataclasses=0.8=pyhc8e2a94_3 - - davix=0.7.6=hb44b51d_0 - - dcap=2.47.12=hd35416c_3 - - expat=2.4.8=h27087fc_0 - - future=0.18.2=py39hf3d152e_5 - - gct=6.2.1550507116=h812cca2_2 - - gettext=0.19.8.1=h73d1719_1008 - - gfal2=2.19.2=h13c04c2_0 - - gfal2-util=1.6.0=pyhd8ed1ab_0 - - glib=2.68.4=h9c3ff4c_1 - - glib-tools=2.68.4=h9c3ff4c_1 - - gsoap=2.8.119=h90a1d37_0 - - gsoap_abi=2.8.106=1 - - icu=70.1=h27087fc_0 - - json-c=0.15=h98cffda_0 - - keyutils=1.6.1=h166bdaf_0 - - krb5=1.19.3=h3790be6_0 - - ld_impl_linux-64=2.36.1=hea4e1c9_2 - - libblas=3.9.0=14_linux64_openblas - - libcblas=3.9.0=14_linux64_openblas - - libcurl=7.82.0=h7bff187_0 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-ng=11.2.0=h1d223b6_14 - - libgfortran-ng=11.2.0=h69a702a_14 - - libgfortran5=11.2.0=h5c6108e_14 - - libglib=2.68.4=h174f98d_1 - - libgomp=11.2.0=h1d223b6_14 - - libiconv=1.16=h516909a_0 - - liblapack=3.9.0=14_linux64_openblas - - libnghttp2=1.47.0=h727a467_0 - - libnsl=2.0.0=h7f98852_0 - - libntlm=1.4=h7f98852_1002 - - libopenblas=0.3.20=pthreads_h78a6416_0 - - libssh2=1.10.0=ha56f1ee_2 - - libstdcxx-ng=11.2.0=he4da1e4_14 - - libtool=2.4.6=h9c3ff4c_1008 - - libuuid=2.32.1=h7f98852_1000 - - libxml2=2.9.12=h22db469_2 - - libzlib=1.2.11=h166bdaf_1014 - - lz4-c=1.9.3=h9c3ff4c_1 - - ncurses=6.3=h9c3ff4c_0 - - numpy=1.22.3=py39h18676bf_1 - - openldap=2.4.59=h43732ee_0 - - openssl=1.1.1n=h166bdaf_0 - - pcre=8.45=h9c3ff4c_0 - - perl=5.32.1=2_h7f98852_perl5 - - pip=22.0.4=pyhd8ed1ab_0 - - pugixml=1.11.4=h9c3ff4c_0 - - pygments=2.11.2=pyhd8ed1ab_0 - - python=3.9.12=h9a8a25e_1_cpython - - python-gfal2=1.10.1=py39h4309f8f_0 - - python_abi=3.9=2_cp39 - - readline=8.1=h46c0cb4_0 - - rich=10.16.2=pyhd8ed1ab_0 - - scitokens-cpp=0.6.3=h00015f2_2 - - setuptools=62.0.0=py39hf3d152e_0 - - sqlite=3.37.1=h4ff8645_0 - - srm-ifce=1.24.5=h9da80fa_1 - - tk=8.6.12=h27826a3_0 - - typing_extensions=4.1.1=pyha770c72_0 - - tzdata=2022a=h191b570_0 - - voms=2.1.0rc0=h46dc511_6 - - wheel=0.37.1=pyhd8ed1ab_0 - - xrootd=5.4.1=py39hbfe9b54_0 - - xz=5.2.5=h516909a_1 - - yaml=0.2.5=h7f98852_2 - - zlib=1.2.11=h166bdaf_1014 - - zstd=1.5.2=ha95c52a_0 - - pip: - - absl-py==1.0.0 - - astunparse==1.6.3 - - cachetools==5.0.0 - - certifi==2021.10.8 - - charset-normalizer==2.0.12 - - cycler==0.11.0 - - docutils==0.18.1 - - flatbuffers==2.0 - - fonttools==4.33.3 - - gast==0.5.3 - - google-auth==2.6.2 - - google-auth-oauthlib==0.4.6 - - google-pasta==0.2.0 - - grpcio==1.44.0 - - h5py==3.1.0 - - idna==3.3 - - importlib-metadata==4.11.3 - - joblib==1.1.0 - - keras==2.8.0 - - keras-preprocessing==1.1.2 - - kiwisolver==1.4.2 - - libclang==13.0.0 - - lockfile==0.12.2 - - luigi==3.0.3 - - markdown==3.3.6 - - matplotlib==3.5.2 - - oauthlib==3.2.0 - - opt-einsum==3.3.0 - - packaging==21.3 - - pandas==1.1.5 - - pillow==9.1.1 - - protobuf==3.20.0 - - psutil==5.9.0 - - pyasn1==0.4.8 - - pyasn1-modules==0.2.8 - - pyparsing==3.0.9 - - python-daemon==2.3.0 - - python-dateutil==2.8.2 - - pytz==2022.1 - - pyyaml==6.0 - - requests==2.27.1 - - requests-oauthlib==1.3.1 - - rsa==4.8 - - scikit-learn==0.24.2 - - scipy==1.8.0 - - six==1.15.0 - - tenacity==6.3.1 - - tensorboard==2.8.0 - - tensorboard-data-server==0.6.1 - - tensorboard-plugin-wit==1.8.1 - - tensorflow==2.8.0 - - tensorflow-io-gcs-filesystem==0.24.0 - - termcolor==1.1.0 - - tf-estimator-nightly==2.8.0.dev2021122109 - - threadpoolctl==3.1.0 - - tornado==6.1 - - uproot==4.2.2 - - urllib3==1.26.9 - - werkzeug==2.1.1 - - wrapt==1.14.0 - - zipp==3.8.0 diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 7c658274..00000000 --- a/environment.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: KingMaker -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=1_gnu - - boost=1.74.0=py39h5472131_5 - - boost-cpp=1.74.0=h6cacc03_7 - - bzip2=1.0.8=h7f98852_4 - - c-ares=1.18.1=h7f98852_0 - - ca-certificates=2021.10.8=ha878542_0 - - cgsi-gsoap=1.3.11=h876a3cc_2 - - colorama=0.4.4=pyh9f0ad1d_0 - - commonmark=0.9.1=py_0 - - conda-pack=0.7.0=pyh6c4a22f_0 - - cyrus-sasl=2.1.27=h230043b_5 - - dataclasses=0.8=pyhc8e2a94_3 - - davix=0.7.6=hb44b51d_0 - - dcap=2.47.12=hd35416c_3 - - docutils=0.18.1=py39hf3d152e_0 - - expat=2.4.4=h9c3ff4c_0 - - future=0.18.2=py39hf3d152e_4 - - gct=6.2.1550507116=h812cca2_2 - - gettext=0.19.8.1=h73d1719_1008 - - gfal2=2.19.2=h13c04c2_0 - - gfal2-util=1.6.0=pyhd8ed1ab_0 - - glib=2.68.4=h9c3ff4c_1 - - glib-tools=2.68.4=h9c3ff4c_1 - - gsoap=2.8.119=h90a1d37_0 - - gsoap_abi=2.8.106=1 - - icu=69.1=h9c3ff4c_0 - - json-c=0.15=h98cffda_0 - - krb5=1.19.2=hcc1bbae_3 - - ld_impl_linux-64=2.36.1=hea4e1c9_2 - - libblas=3.9.0=13_linux64_openblas - - libcblas=3.9.0=13_linux64_openblas - - libcurl=7.81.0=h2574ce0_0 - - libedit=3.1.20191231=he28a2e2_2 - - libev=4.33=h516909a_1 - - libffi=3.4.2=h7f98852_5 - - libgcc-ng=11.2.0=h1d223b6_12 - - libgfortran-ng=11.2.0=h69a702a_12 - - libgfortran5=11.2.0=h5c6108e_12 - - libglib=2.68.4=h174f98d_1 - - libgomp=11.2.0=h1d223b6_12 - - libiconv=1.16=h516909a_0 - - liblapack=3.9.0=13_linux64_openblas - - libnghttp2=1.46.0=h812cca2_0 - - libnsl=2.0.0=h7f98852_0 - - libntlm=1.4=h7f98852_1002 - - libopenblas=0.3.18=pthreads_h8fe5266_0 - - libssh2=1.10.0=ha56f1ee_2 - - libstdcxx-ng=11.2.0=he4da1e4_12 - - libtool=2.4.6=h9c3ff4c_1008 - - libuuid=2.32.1=h7f98852_1000 - - libxml2=2.9.12=h885dcf4_1 - - libzlib=1.2.11=h36c2ea0_1013 - - lockfile=0.12.2=py_1 - - luigi=3.0.3=py39hf3d152e_1 - - lz4-c=1.9.3=h9c3ff4c_1 - - ncurses=6.3=h9c3ff4c_0 - - numpy=1.22.2=py39h91f2184_0 - - openldap=2.4.59=h43732ee_0 - - openssl=1.1.1l=h7f98852_0 - - pcre=8.45=h9c3ff4c_0 - - perl=5.32.1=1_h7f98852_perl5 - - pip=22.0.3=pyhd8ed1ab_0 - - prometheus_client=0.5.0=py_0 - - pugixml=1.11.4=h9c3ff4c_0 - - pygments=2.11.2=pyhd8ed1ab_0 - - python=3.9.10=h85951f9_2_cpython - - python-daemon=2.3.0=pyhd8ed1ab_1 - - python-dateutil=2.8.2=pyhd8ed1ab_0 - - python-gfal2=1.10.1=py39h4309f8f_0 - - python_abi=3.9=2_cp39 - - pyyaml=6.0=py39h3811e60_3 - - readline=8.1=h46c0cb4_0 - - rich=10.16.2=pyhd8ed1ab_0 - - scitokens-cpp=0.6.3=h00015f2_2 - - setuptools=60.8.1=py39hf3d152e_0 - - six=1.16.0=pyh6c4a22f_0 - - sqlite=3.37.0=h9cd32fc_0 - - srm-ifce=1.24.5=h9da80fa_1 - - tenacity=6.3.1=pyhd8ed1ab_0 - - tk=8.6.11=h27826a3_1 - - toml=0.10.2=pyhd8ed1ab_0 - - tornado=6.1=py39h3811e60_2 - - typing_extensions=4.0.1=pyha770c72_0 - - tzdata=2021e=he74cb21_0 - - voms=2.1.0rc0=h82ee70a_4 - - wheel=0.37.1=pyhd8ed1ab_0 - - xrootd=5.4.0=py39hf708bbc_0 - - xz=5.2.5=h516909a_1 - - yaml=0.2.5=h7f98852_2 - - zlib=1.2.11=h36c2ea0_1013 - - zstd=1.5.2=ha95c52a_0 diff --git a/kingmaker-images b/kingmaker-images new file mode 160000 index 00000000..8188fce1 --- /dev/null +++ b/kingmaker-images @@ -0,0 +1 @@ +Subproject commit 8188fce137bb1f42694dbc09c91289350adf6d94 diff --git a/law b/law index d067e023..7ff75132 160000 --- a/law +++ b/law @@ -1 +1 @@ -Subproject commit d067e023b89896208b2c424a2beaf110cd8618ee +Subproject commit 7ff7513272943d856a59472d3fd3a6dc4411bc0b diff --git a/lawluigi_configs/GPU_example_luigi.cfg b/lawluigi_configs/GPU_example_luigi.cfg index dcc8de49..5e75eb50 100644 --- a/lawluigi_configs/GPU_example_luigi.cfg +++ b/lawluigi_configs/GPU_example_luigi.cfg @@ -10,7 +10,7 @@ max_reschedules = 3 [DEFAULT] name = GPU_example -ENV_NAME = ML_LAW +ENV_NAME = Base ; grid storage protocol and path usable from submitting machine and worker nodes of cluster ; job in- and output will be stored in $wlcg_path under subdirectory of analysis $name @@ -21,11 +21,9 @@ htcondor_remote_job = True htcondor_request_cpus = 1 htcondor_request_gpus = 1 ; for all cores in total -htcondor_universe = docker +htcondor_universe = docker ;image without GPU libraries # htcondor_docker_image = mschnepf/slc7-condocker:latest -;image with GPU libraries -htcondor_docker_image = tvoigtlaender/slc7-condocker-cuda-11.5-cudnn8:base ; create log files in htcondor jobs transfer_logs = True ; set local scheduler @@ -41,9 +39,9 @@ bootstrap_file = setup_law_remote.sh [SaveToRemote] [RunRemote] +ENV_NAME = ML_LAW htcondor_walltime = 360 htcondor_request_memory = 1000 htcondor_requirements = ( TARGET.CloudSite =?= "topas" ) -# && (Machine =?= "f03-001-179-e.gridka.de") htcondor_request_disk = 1000000 [ReadFromRemote] diff --git a/lawluigi_configs/ML_train_luigi.cfg b/lawluigi_configs/ML_train_luigi.cfg index 5b0f154d..a0924215 100644 --- a/lawluigi_configs/ML_train_luigi.cfg +++ b/lawluigi_configs/ML_train_luigi.cfg @@ -13,7 +13,7 @@ retry_count: 0 [DEFAULT] name = ML_train -ENV_NAME = BaseWRoot +ENV_NAME = Base ; grid storage protocol and path usable from submitting machine and worker nodes of cluster ; job in- and output will be stored in $wlcg_path under subdirectory of analysis $name wlcg_path = root://cmsdcache-kit-disk.gridka.de//store/user/${USER}/LAW_storage @@ -23,8 +23,8 @@ htcondor_remote_job = True htcondor_request_cpus = 1 ; for all cores in total htcondor_universe = docker -;image with GPU libraries -# htcondor_docker_image = tvoigtlaender/slc7-condocker-cuda-11.5-cudnn8:base +;image without GPU libraries +# htcondor_docker_image = mschnepf/slc7-condocker:latest ; create log files in htcondor jobs transfer_logs = True ; set local scheduler @@ -43,7 +43,7 @@ bootstrap_file = setup_law_remote.sh htcondor_request_cpus = 1 htcondor_walltime = 36000 htcondor_request_memory = 4000 -htcondor_request_disk = 1000000 +htcondor_request_disk = 2000000 htcondor_requirements = (TARGET.ProvidesEKPResources==True) && (TARGET.ProvidesIO==True) additional_files = ["ml_configs", "sm-htt-analysis"] @@ -52,7 +52,6 @@ additional_files = ["ml_configs", "sm-htt-analysis"] ENV_NAME = ML_LAW htcondor_request_cpus = 2 htcondor_request_gpus = 1 -htcondor_docker_image = tvoigtlaender/slc7-condocker-cuda-11.5-cudnn8:base htcondor_walltime = 3600 htcondor_request_memory = 4000 htcondor_requirements = ( TARGET.CloudSite =?= "topas" ) @@ -64,7 +63,6 @@ additional_files = ["ml_configs", "sm-htt-analysis"] ENV_NAME = ML_LAW htcondor_request_cpus = 1 htcondor_request_gpus = 1 -htcondor_docker_image = tvoigtlaender/slc7-condocker-cuda-11.5-cudnn8:base htcondor_walltime = 3600 htcondor_request_memory = 10000 htcondor_request_disk = 10000000 diff --git a/processor/framework.py b/processor/framework.py index 3a22cbac..d6a299f7 100644 --- a/processor/framework.py +++ b/processor/framework.py @@ -2,21 +2,28 @@ import luigi import law import select -from law.util import interruptable_popen, readable_popen -from subprocess import PIPE, Popen +import subprocess +from law.util import interruptable_popen from rich.console import Console -from law.util import merge_dicts, DotDict +from law.util import merge_dicts from datetime import datetime from law.contrib.htcondor.job import HTCondorJobManager from tempfile import mkdtemp from getpass import getuser -from law.target.collection import flatten_collections from law.config import Config -import subprocess + +try: + from luigi.parameter import UnconsumedParameterWarning + import warnings + + # Ignore warnings about unused parameters that are set in the default config but not used by all tasks + warnings.simplefilter("ignore", UnconsumedParameterWarning) +except: + pass law.contrib.load("wlcg") law.contrib.load("htcondor") -# try to get the terminal width, if this fails, we are in a remote job, set it to 140 +# try to get the terminal width, if this fails, we are probably in a remote job, set it to 140 try: current_width = os.get_terminal_size().columns except OSError: @@ -41,24 +48,27 @@ class Task(law.Task): local_user = getuser() wlcg_path = luigi.Parameter(description="Base-path to remote file location.") - local_output_path = luigi.Parameter(description="Base-path to local file location.") - output_destination = luigi.Parameter(description="Whether to use local storage.") + local_output_path = luigi.Parameter( + description="Base-path to local file location.", + default=os.getenv("ANALYSIS_DATA_PATH"), + ) + is_local_output = luigi.BoolParameter( + description="Whether to use local storage. False by default." + ) # Behaviour of production_tag: # If a tag is give it will be used for the respective task. - # If no tag is given a timestamp abse on startup_time is used. - # This timestamp is the same for all tasks with no set production_tag. + # If no tag is given a timestamp based on startup_time is used. + # This timestamp is the same for all tasks in a workflow run with no set production_tag. production_tag = luigi.Parameter( default=f"default/{startup_time}", description="Tag to differentiate workflow runs. Set to a timestamp as default.", ) output_collection_cls = law.NestedSiblingFileCollection - @property - def is_local_output(self): - return self.output_destination == "local" - - # Path of local targets. Composed from the analysis path set during the setup.sh, + # Path of local targets. + # Composed from the analysis path set during the setup.sh + # or the local_output_path if is_local_output is set, # the production_tag, the name of the task and an additional path if provided. def local_path(self, *path): return os.path.join( @@ -131,8 +141,8 @@ def set_environment(self, sourcescript, silent=False): code, out, error = interruptable_popen( source_command_string, shell=True, - stdout=PIPE, - stderr=PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, # rich_console=console ) if code != 0: @@ -173,8 +183,8 @@ def run_command( code, out, error = interruptable_popen( " ".join(command), shell=True, - stdout=PIPE, - stderr=PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=run_env, cwd=run_location, ) @@ -214,11 +224,11 @@ def run_command_readable(self, command=[], sourcescript=[], run_location=None): console.rule() console.log(logstring) try: - p = Popen( + p = subprocess.Popen( " ".join(command), shell=True, - stdout=PIPE, - stderr=PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=run_env, cwd=run_location, encoding="utf-8", @@ -263,7 +273,8 @@ class HTCondorWorkflow(Task, law.htcondor.HTCondorWorkflow): description="Runtime to be set in HTCondor job submission." ) htcondor_request_cpus = luigi.Parameter( - description="Number of CPU cores to be requested in HTCondor job submission." + description="Number of CPU cores to be requested in HTCondor job submission.", + default="1", ) htcondor_request_gpus = luigi.Parameter( default="0", @@ -289,36 +300,51 @@ class HTCondorWorkflow(Task, law.htcondor.HTCondorWorkflow): default=[], description="Additional files to be included in the job tarball. Will be unpacked in the run directory", ) + remote_source_script = luigi.Parameter( + description="Script to source environment in remote jobs. Leave empty if not needed. Defaults to use with docker images", + default="source /opt/conda/bin/activate env", + ) # Use proxy file located in $X509_USER_PROXY or /tmp/x509up_u$(id) if empty htcondor_user_proxy = law.wlcg.get_vomsproxy_file() def get_submission_os(self): - # function to check, if running on centos7, centos8 or rhel9 + # function to check, if running on centos7, rhel9 or Ubuntu22 + # Other OS are not permitted # based on this, the correct docker image is chosen, overwriting the htcondor_docker_image parameter # check if lsb_release is installed, if not, use the information from /etc/os-release + # Please note that this selection can be somewhat unstable. Modify if neccessary. try: distro = ( - subprocess.check_output("lsb_release -i | cut -f2", shell=True) + subprocess.check_output( + "lsb_release -i | cut -f2", stderr=subprocess.STDOUT + ) .decode() + .replace("Linux", "") + .replace("linux", "") .strip() ) os_version = ( - subprocess.check_output("lsb_release -r | cut -f2", shell=True) + subprocess.check_output( + "lsb_release -r | cut -f2", stderr=subprocess.STDOUT + ) .decode() .strip() ) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): distro = ( subprocess.check_output( - "cat /etc/os-release | grep '^NAME=' | cut -f2 -d=''", shell=True + "cat /etc/os-release | grep '^NAME=' | cut -f2 -d='' | tr -d '\"'", + shell=True, ) .decode() + .replace("Linux", "") + .replace("linux", "") .strip() ) os_version = ( subprocess.check_output( - "cat /etc/os-release | grep '^VERSION_ID=' | cut -f2 -d=''", + "cat /etc/os-release | grep '^VERSION_ID=' | cut -f2 -d='' | tr -d '\"'", shell=True, ) .decode() @@ -330,21 +356,18 @@ def get_submission_os(self): if distro == "CentOS": if os_version[0] == "7": image_name = "centos7" - elif distro == "RedHatEnterprise" or distro == "AlmaLinux": - if os_version[0] == "8": - image_name = "centos8" - elif os_version[0] == "9": + elif distro in ("RedHatEnterprise", "Alma"): + if os_version[0] == "9": image_name = "rhel9" elif distro == "Ubuntu": - if os_version[0:2] == "20": - image_name = "ubuntu2004" - elif os_version[0:2] == "22": + if os_version[0:2] == "22": image_name = "ubuntu2204" else: raise Exception( - f"Unknown OS {distro} {os_version}, CROWN will not run without changes" + f"Unknown OS {distro} {os_version}, KingMaker will not run without changes" ) - image = f"ghcr.io/kit-cms/kingmaker-images-{image_name}-{str(self.ENV_NAME).lower()}:main" + image_hash = os.getenv("IMAGE_HASH") + image = f"ghcr.io/kit-cms/kingmaker-images-{image_name}-{str(self.ENV_NAME).lower()}:main_{image_hash}" # print(f"Running on {distro} {os_version}, using image {image}") return image @@ -371,7 +394,6 @@ def htcondor_output_directory(self): def htcondor_create_job_file_factory(self): factory = super(HTCondorWorkflow, self).htcondor_create_job_file_factory() - factory.is_tmp = False # Print location of job dir console.log(f"HTCondor job directory is: {factory.dir}") return factory @@ -451,8 +473,6 @@ def htcondor_job_config(self, config, job_num, branches): ) if not tarball.exists(): # Make new tarball - prevdir = os.getcwd() - os.system("cd $ANALYSIS_PATH") # get absolute path to tarball dir tarball_dir = os.path.abspath(f"tarballs/{self.production_tag}") tarball_local = law.LocalFileTarget( @@ -484,8 +504,8 @@ def htcondor_job_config(self, config, job_num, branches): ] + list(self.additional_files) code, out, error = interruptable_popen( command, - stdout=PIPE, - stderr=PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, # rich_console=console ) if code != 0: @@ -501,41 +521,15 @@ def htcondor_job_config(self, config, job_num, branches): tarball.parent.touch() tarball.copy_from_local(src=tarball_local.path) console.rule("Framework tarball uploaded!") - os.chdir(prevdir) - # Check if env of this task was found in cvmfs - env_list = os.getenv("ENV_NAMES_LIST").split(";") - env_list = list(dict.fromkeys(env_list[:-1])) - env_dict = dict(env.split(",") for env in env_list) - if env_dict[self.ENV_NAME] == "False": - # IMPORTANT: environments have to be named differently with each change - # as caching prevents a clean overwrite of existing files - if self.is_local_output: - tarball_env = law.LocalFileTarget( - path=f"env_tarballs/{self.ENV_NAME}.tar.gz", - fs=law.LocalFileSystem( - None, - base=f"{os.path.expandvars(self.local_output_path)}", - ), - ) - else: - tarball_env = law.wlcg.WLCGFileTarget( - path=f"env_tarballs/{self.ENV_NAME}.tar.gz" - ) - - if not tarball_env.exists(): - tarball_env.parent.touch() - tarball_env.copy_from_local( - src=os.path.abspath(f"tarballs/conda_envs/{self.ENV_NAME}.tar.gz") - ) config.render_variables["USER"] = self.local_user config.render_variables["ANA_NAME"] = os.getenv("ANA_NAME") config.render_variables["ENV_NAME"] = self.ENV_NAME config.render_variables["TAG"] = self.production_tag - config.render_variables["USE_CVMFS"] = env_dict[self.ENV_NAME] config.render_variables["NTHREADS"] = self.htcondor_request_cpus config.render_variables["LUIGIPORT"] = os.getenv("LUIGIPORT") + config.render_variables["SOURCE_SCRIPT"] = self.remote_source_script - config.render_variables["OUTPUT_DESTINATION"] = self.output_destination + config.render_variables["IS_LOCAL_OUTPUT"] = str(self.is_local_output) if not self.is_local_output: config.render_variables["TARBALL_PATH"] = ( os.path.expandvars(self.wlcg_path) + tarball.path @@ -544,16 +538,6 @@ def htcondor_job_config(self, config, job_num, branches): config.render_variables["TARBALL_PATH"] = ( os.path.expandvars(self.local_output_path) + tarball.path ) - # Include path to env tarball if env not in cvmfs - if env_dict[self.ENV_NAME] == "False": - if not self.is_local_output: - config.render_variables["TARBALL_ENV_PATH"] = ( - os.path.expandvars(self.wlcg_path) + tarball_env.path - ) - else: - config.render_variables["TARBALL_ENV_PATH"] = ( - os.path.expandvars(self.local_output_path) + tarball_env.path - ) config.render_variables["LOCAL_TIMESTAMP"] = startup_time config.render_variables["LOCAL_PWD"] = startup_dir # only needed for $ANA_NAME=ML_train see setup.sh line 158 diff --git a/processor/setup_law_remote.sh b/processor/setup_law_remote.sh index 39532666..6333fc07 100644 --- a/processor/setup_law_remote.sh +++ b/processor/setup_law_remote.sh @@ -28,10 +28,10 @@ action() { echo " | XRD_PARALLELEVTLOOP = ${XRD_PARALLELEVTLOOP}" echo "------------------------------------------" - source /opt/conda/etc/profile.d/conda.sh - conda activate env + echo "Setting up environment via {{SOURCE_SCRIPT}}." + {{SOURCE_SCRIPT}} - if [ "{{OUTPUT_DESTINATION}}" = "local" ] + if [ "{{IS_LOCAL_OUTPUT}}" = "True" ] then echo "cp {{TARBALL_PATH}} ${SPAWNPOINT}" cp {{TARBALL_PATH}} ${SPAWNPOINT} diff --git a/processor/tasks/MLTraining.py b/processor/tasks/MLTraining.py index 6f15371d..b3e11646 100644 --- a/processor/tasks/MLTraining.py +++ b/processor/tasks/MLTraining.py @@ -62,16 +62,11 @@ def htcondor_job_config(self, config, job_num, branches): config = super(CreateTrainingDataShard, self).htcondor_job_config( config, job_num, branches ) - name_list = [ - "_".join(info + (fold,)) - for info in self.datashard_information - for fold in ["0", "1"] - ] task_name = self.__class__.__name__ - branch_names = [] - for branch in branches: - branch_names.append(name_list[branch]) - branch_str = "|".join(branch_names) + flattened_branches = sum( + branches, [] + ) # Quick and dirty way to flatten a nested list + branch_str = f"{min(flattened_branches)}to{max(flattened_branches)}" config.custom_content.append(("JobBatchName", f"{task_name}-{branch_str}")) return config @@ -177,16 +172,11 @@ class RunTraining(MLBase): # Add branch specific names to the HTCondor jobs def htcondor_job_config(self, config, job_num, branches): config = super(RunTraining, self).htcondor_job_config(config, job_num, branches) - name_list = [ - "_".join([info[0], fold]) - for info in self.training_information - for fold in ["0", "1"] - ] task_name = self.__class__.__name__ - branch_names = [] - for branch in branches: - branch_names.append(name_list[branch]) - branch_str = "|".join(branch_names) + flattened_branches = sum( + branches, [] + ) # Quick and dirty way to flatten a nested list + branch_str = f"{min(flattened_branches)}to{max(flattened_branches)}" config.custom_content.append(("JobBatchName", f"{task_name}-{branch_str}")) return config @@ -358,9 +348,6 @@ def run(self): "--output-dir {}".format(out_dir), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) ## Convert model to lwtnn format @@ -374,9 +361,6 @@ def run(self): "--in-out-dir {}".format(out_dir), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) self.run_command( @@ -391,9 +375,6 @@ def run(self): "> {dir}/fold{fold}_lwtnn.json".format(dir=out_dir, fold=fold), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) # Copy locally created files to remote storage @@ -427,12 +408,11 @@ class RunTesting(MLBase): # Add branch specific names to the HTCondor jobs def htcondor_job_config(self, config, job_num, branches): config = super(RunTesting, self).htcondor_job_config(config, job_num, branches) - name_list = [info[0] for info in self.training_information] task_name = self.__class__.__name__ - branch_names = [] - for branch in branches: - branch_names.append(name_list[branch]) - branch_str = "|".join(branch_names) + flattened_branches = sum( + branches, [] + ) # Quick and dirty way to flatten a nested list + branch_str = f"{min(flattened_branches)}to{max(flattened_branches)}" config.custom_content.append(("JobBatchName", f"{task_name}-{branch_str}")) return config @@ -612,9 +592,6 @@ def run(self): "--output-dir {}".format(store_dir), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) ## Create 1D taylor coefficient plots @@ -629,9 +606,6 @@ def run(self): "--output-dir {}".format(store_dir), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) ## Create taylor ranking plots @@ -646,9 +620,6 @@ def run(self): "--output-dir {}".format(store_dir), ], run_location=run_loc, - sourcescript=[ - "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ML_LAW" - ], ) ## Tar plots together diff --git a/processor/tasks/scripts/compile_crown.sh b/processor/tasks/scripts/compile_crown.sh index bfdba23d..dee52d20 100644 --- a/processor/tasks/scripts/compile_crown.sh +++ b/processor/tasks/scripts/compile_crown.sh @@ -14,11 +14,14 @@ EXECUTALBE_THREADS=${11} set -o pipefail set -e source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS -# remove conda /cvmfs/etp.kit.edu from $PATH so cmakes uses the LCG stack python and not the conda one -PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') -CONDA_PYTHON_EXE="" -CONDA_EXE="" -CONDA_PREFIX="" +# remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one +if [[ ! -z "${CONDA_PREFIX}" ]]; then + PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") + # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + CONDA_PYTHON_EXE="" + CONDA_EXE="" + CONDA_PREFIX="" +fi # use a fourth of the machine for compiling THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) THREADS=$(( THREADS_AVAILABLE / 4 )) diff --git a/processor/tasks/scripts/compile_crown_friends.sh b/processor/tasks/scripts/compile_crown_friends.sh index 472dae22..68ed35c2 100644 --- a/processor/tasks/scripts/compile_crown_friends.sh +++ b/processor/tasks/scripts/compile_crown_friends.sh @@ -14,11 +14,14 @@ QUANTITIESMAP=${11} set -o pipefail set -e source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS -# remove conda /cvmfs/etp.kit.edu from $PATH so cmakes uses the LCG stack python and not the conda one -PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') -CONDA_PYTHON_EXE="" -CONDA_EXE="" -CONDA_PREFIX="" +# remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one +if [[ ! -z "${CONDA_PREFIX}" ]]; then + PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") + # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + CONDA_PYTHON_EXE="" + CONDA_EXE="" + CONDA_PREFIX="" +fi # use a fourth of the machine for compiling THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) # THREADS=$(( THREADS_AVAILABLE / 4 )) diff --git a/processor/tasks/scripts/compile_crown_lib.sh b/processor/tasks/scripts/compile_crown_lib.sh index 9435cbc2..aac7acfd 100644 --- a/processor/tasks/scripts/compile_crown_lib.sh +++ b/processor/tasks/scripts/compile_crown_lib.sh @@ -10,11 +10,14 @@ echo "Build dir: $BUILDDIR" set -o pipefail set -e source $ANALYSIS_PATH/CROWN/init.sh $ANALYSIS -# remove conda /cvmfs/etp.kit.edu from $PATH so cmakes uses the LCG stack python and not the conda one -PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') -CONDA_PYTHON_EXE="" -CONDA_EXE="" -CONDA_PREFIX="" +# remove conda prefix from $PATH so cmakes uses the LCG stack python and not the conda one +if [[ ! -z "${CONDA_PREFIX}" ]]; then + PATH=$(echo $PATH | sed "s@$CONDA_PREFIX@@g") + # PATH=$(echo $PATH | sed 's%/cvmfs/etp.kit.edu/[^:]*:%%g') + CONDA_PYTHON_EXE="" + CONDA_EXE="" + CONDA_PREFIX="" +fi # use a fourth of the machine for compiling THREADS_AVAILABLE=$(grep -c ^processor /proc/cpuinfo) THREADS=$(( THREADS_AVAILABLE / 4 )) diff --git a/sample_database b/sample_database index 64419ecf..a19687cd 160000 --- a/sample_database +++ b/sample_database @@ -1 +1 @@ -Subproject commit 64419ecf1fd073cf0a9cbe8a4b1b42fb146de2cb +Subproject commit a19687cdb01a6f8ca54233b351814af58f1ad9b9 diff --git a/scripts/ParseNeededEnv.py b/scripts/ParseNeededEnv.py index fa42a863..1133d369 100644 --- a/scripts/ParseNeededEnv.py +++ b/scripts/ParseNeededEnv.py @@ -1,6 +1,7 @@ import configparser from sys import argv import os +import sys config = configparser.ConfigParser() @@ -12,35 +13,31 @@ "Please provided a luigi config file to search for the necessary environments." ) print("Example: 'python ParseNeededEnv.py '") - exit(1) + sys.exit(1) # Check if file exists at that location if not os.path.isfile(cfg_path): - print("There was no file found at {}".format(cfg_path)) - exit(1) + print(f"There was no file found at {cfg_path}") + sys.exit(1) # Try to parse config file try: config.read(cfg_path) except (configparser.ParsingError, configparser.MissingSectionHeaderError) as error: print( - "{}@File at {} could not be parsed. Is it a valid luigi config file?".format( - error, cfg_path - ) + f"{error}@File at {cfg_path} could not be parsed. Is it a valid luigi config file?" ) - exit(1) + sys.exit(1) # Try to get starting env from 'ENV_NAME' of 'DEFAULT' section try: base_env = config["DEFAULT"]["ENV_NAME"].strip() except KeyError as error: print( - "Config file at {} does not provide an 'ENV_NAME' in it's 'DEFAULT' section.".format( - cfg_path - ), + f"Config file at {cfg_path} does not provide an 'ENV_NAME' in it's 'DEFAULT' section.", "Without this, the starting env cannot be set.", ) - exit(1) + sys.exit(1) all_envs = [base_env] # Add all other envs mentioned in the 'ENV_NAME' of the sections to the list diff --git a/scripts/os-version.sh b/scripts/os-version.sh index cd79c868..d789c570 100644 --- a/scripts/os-version.sh +++ b/scripts/os-version.sh @@ -12,4 +12,4 @@ fi distro=${distro//[[:space:]]/} distro="${distro//Linux/}" distro="${distro//linux/}" -echo "Running Kingmaker on $distro Version $os_version" +#echo "Trying to run Kingmaker on $distro Version $os_version" diff --git a/setup.sh b/setup.sh index ba3528ca..416d4a21 100755 --- a/setup.sh +++ b/setup.sh @@ -1,35 +1,32 @@ ############################################################################################ -# This script setups all dependencies necessary for making law executable # +# This script sets up all dependencies necessary for running KingMaker # ############################################################################################ + +_addpy() { + [ ! -z "$1" ] && export PYTHONPATH="$1:${PYTHONPATH}" +} + +_addbin() { + [ ! -z "$1" ] && export PATH="$1:${PATH}" +} + action() { # Check if law was already set up in this shell if ( [[ ! -z ${LAW_IS_SET_UP} ]] && [[ ! "$@" =~ "-f" ]] ); then - echo "LAW was already set up in this shell. Please, use a new one." + echo "KingMaker was already set up in this shell. Please, use a new one." return 1 fi - # Check if current machine is an etp portal machine. - PORTAL_LIST=("bms1.etp.kit.edu" "bms2.etp.kit.edu" "bms3.etp.kit.edu" "portal1.etp.kit.edu" "bms1-centos7.etp.kit.edu" "bms2-centos7.etp.kit.edu" "bms3-centos7.etp.kit.edu" "portal1-centos7.etp.kit.edu") - CURRENT_HOST=$(hostname --long) - if [[ ! " ${PORTAL_LIST[*]} " =~ " ${CURRENT_HOST} " ]]; then - echo "Current host (${CURRENT_HOST}) not in list of allowed machines:" - printf '%s\n' "${PORTAL_LIST[@]}" - return 1 - else - echo "Running on ${CURRENT_HOST}." + # Check if law already tried to set up in this shell + if ( [[ ! -z ${LAW_TRIED_TO_SET_UP} ]] && [[ ! "$@" =~ "-f" ]] ); then + echo "Kingmaker already tried to set up in this shell. This might lead to unintended behaviour." fi - #list of available analyses - ANA_LIST=("KingMaker" "GPU_example" "ML_train") - if [[ "$@" =~ "-l" ]]; then - echo "Available analyses:" - printf '%s\n' "${ANA_LIST[@]}" - return 0 - fi + export LAW_TRIED_TO_SET_UP="True" - # determine the directory of this file + # Determine the directory of this file if [ ! -z "${ZSH_VERSION}" ]; then local THIS_FILE="${(%):-%x}" else @@ -38,30 +35,53 @@ action() { local BASE_DIR="$( cd "$( dirname "${THIS_FILE}" )" && pwd )" - _addpy() { - [ ! -z "$1" ] && export PYTHONPATH="$1:${PYTHONPATH}" - } - - _addbin() { - [ ! -z "$1" ] && export PATH="$1:${PATH}" - } - + # Check if current OS is supported + source scripts/os-version.sh + local VALID_OS="False" + if [[ "$distro" == "CentOS" ]]; then + if [[ ${os_version:0:1} == "7" ]]; then + VALID_OS="True" + fi + elif [[ "$distro" == "RedHatEnterprise" || "$distro" == "Alma" || "$distro" == "Rocky" ]]; then + if [[ ${os_version:0:1} == "9" ]]; then + VALID_OS="True" + fi + elif [[ "$distro" == "Ubuntu" ]]; then + if [[ ${os_version:0:2} == "22" ]]; then + VALID_OS="True" + fi + fi + if [[ "${VALID_OS}" == "False" ]]; then + echo "Kingmaker not support on ${distro} ${os_version}" + return 1 + else + echo "Running Kingmaker on $distro Version $os_version on $(hostname) from dir ${BASE_DIR}" + fi + # Workflow to be set up ANA_NAME_GIVEN=$1 - #Determine analysis to be used. Default is first in list. + # List of available workflows + ANA_LIST=("KingMaker" "GPU_example" "ML_train") + if [[ "$@" =~ "-l" ]]; then + echo "Available workflows:" + printf '%s\n' "${ANA_LIST[@]}" + return 0 + fi + + # Determine workflow to be used. Default is first in list. if [[ -z "${ANA_NAME_GIVEN}" ]]; then - echo "No analysis chosen. Please choose from:" + echo "No workflow chosen. Please choose from:" printf '%s\n' "${ANA_LIST[@]}" return 1 else - #Check if given analysis is in list + # Check if given workflow is in list if [[ ! " ${ANA_LIST[*]} " =~ " ${ANA_NAME_GIVEN} " ]] ; then echo "Not a valid name. Allowed choices are:" printf '%s\n' "${ANA_LIST[@]}" return 1 else - echo "Using ${ANA_NAME_GIVEN} analysis." + echo "Using ${ANA_NAME_GIVEN} workflow." export ANA_NAME="${ANA_NAME_GIVEN}" fi fi @@ -78,86 +98,61 @@ action() { return 1 fi + # Ensure that submodule with KingMaker env files is present + if [ -z "$(ls -A kingmaker-images)" ]; then + git submodule update --init --recursive -- kingmaker-images + fi + # Get kingmaker-images submodule hash to find the correct image during job submission + export IMAGE_HASH=$(cd kingmaker-images/; git rev-parse --short HEAD) + # First listed is env of DEFAULT and will be used as the starting env + # Remaining envs should be sourced via provided docker images export STARTING_ENV=$(echo ${PARSED_ENVS} | head -n1 | awk '{print $1;}') - echo "The following envs will be set up: ${PARSED_ENVS}" + # echo "The following envs will be set up: ${PARSED_ENVS}" echo "${STARTING_ENV} will be sourced as the starting env." - export ENV_NAMES_LIST="" - for ENV_NAME in ${PARSED_ENVS}; do - # Check if necessary environment is present in cvmfs - # Try to install and export env via miniconda if not - # NOTE: HTCondor jobs that rely on exported miniconda envs might need additional scratch space - if [[ -d "/cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/envs/${ENV_NAME}" ]]; then - echo "${ENV_NAME} environment found in cvmfs." - CVMFS_ENV_PRESENT="True" - else - echo "${ENV_NAME} environment not found in cvmfs. Using conda." - # Install conda if necessary - if [ ! -f "miniconda/bin/activate" ]; then - # Miniconda version used for all environments - MINICONDA_VERSION="Miniconda3-py39_23.5.2-0-Linux-x86_64" - echo "conda could not be found, installing conda ..." - echo "More information can be found in" - echo "https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html" - curl -O https://repo.anaconda.com/miniconda/${MINICONDA_VERSION}.sh - bash ${MINICONDA_VERSION}.sh -b -s -p miniconda - rm -f ${MINICONDA_VERSION}.sh - fi - # source base env of conda - source miniconda/bin/activate '' - - # check if correct Conda env is running - if [ "${CONDA_DEFAULT_ENV}" != "${ENV_NAME}" ]; then - if [ -d "miniconda/envs/${ENV_NAME}" ]; then - echo "${ENV_NAME} env found using conda." - else - # Create conda env from yaml file if necessary - echo "Creating ${ENV_NAME} env from conda_environments/${ENV_NAME}_env.yml..." - if [[ ! -f "conda_environments/${ENV_NAME}_env.yml" ]]; then - echo "conda_environments/${ENV_NAME}_env.yml not found. Unable to create environment." - return 1 - fi - conda env create -f conda_environments/${ENV_NAME}_env.yml -n ${ENV_NAME} - echo "${ENV_NAME} env built using conda." - fi - fi - - # create conda tarball if env not in cvmfs and it if it doesn't already exist - if [ ! -f "tarballs/conda_envs/${ENV_NAME}.tar.gz" ]; then - # IMPORTANT: environments have to be named differently with each change - # as chaching prevents a clean overwrite of existing files - echo "Creating ${ENV_NAME}.tar.gz" - mkdir -p "tarballs/conda_envs" - conda activate ${ENV_NAME} - conda pack -n ${ENV_NAME} --output tarballs/conda_envs/${ENV_NAME}.tar.gz - if [[ "$?" -eq "1" ]]; then - echo "Conda pack failed. Does the env contain conda-pack?" - return 1 - fi - conda deactivate - fi - CVMFS_ENV_PRESENT="False" + # Check if necessary environment is present in cvmfs + # Try to install and export env via miniforge if not + # NOTE: miniforge is based on conda and uses the same syntax. Switched due to licensing concerns. + # NOTE2: HTCondor jobs that rely on exported miniforge envs might need additional scratch space + if [[ -d "/cvmfs/etp.kit.edu/LAW_envs/miniforge/envs/${STARTING_ENV}" ]]; then + echo "${STARTING_ENV} environment found in cvmfs." + echo "Activating starting-env ${STARTING_ENV} from cvmfs." + source /cvmfs/etp.kit.edu/LAW_envs/miniforge/bin/activate ${STARTING_ENV} + else + echo "${STARTING_ENV} environment not found in cvmfs. Using miniforge." + # Install miniforge if necessary + if [ ! -f "miniforge/bin/activate" ]; then + # Miniforge version used for all environments + MAMBAFORGE_VERSION="24.3.0-0" + MAMBAFORGE_INSTALLER="Mambaforge-${MAMBAFORGE_VERSION}-$(uname)-$(uname -m).sh" + echo "Miniforge could not be found, installing miniforge version ${MAMBAFORGE_INSTALLER}" + echo "More information can be found in" + echo "https://github.com/conda-forge/miniforge" + curl -L -O https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/${MAMBAFORGE_INSTALLER} + bash ${MAMBAFORGE_INSTALLER} -b -s -p miniforge + rm -f ${MAMBAFORGE_INSTALLER} fi + # Source base env of miniforge + source miniforge/bin/activate '' - # Remember status of starting-env - if [[ "${ENV_NAME}" == "${STARTING_ENV}" ]]; then - CVMFS_ENV_PRESENT_START=${CVMFS_ENV_PRESENT} + # Check if correct miniforge env is running + if [ -d "miniforge/envs/${STARTING_ENV}" ]; then + echo "${STARTING_ENV} env found using miniforge." + else + # Create miniforge env from yaml file if necessary + echo "Creating ${STARTING_ENV} env from kingmaker-images/KingMaker_envs/${STARTING_ENV}_env.yml..." + if [[ ! -f "kingmaker-images/KingMaker_envs/${STARTING_ENV}_env.yml" ]]; then + echo "kingmaker-images/KingMaker_envs/${STARTING_ENV}_env.yml not found. Unable to create environment." + return 1 + fi + conda env create -f kingmaker-images/KingMaker_envs/${STARTING_ENV}_env.yml -n ${STARTING_ENV} + echo "${STARTING_ENV} env built using miniforge." fi - # Create list of envs and their status to be later parsed by python - # Example: 'env1;True,env2;False,env3;False' - # ENV_NAMES_LIST is used by the processor/framework.py to determine whether the environments are present in cvmfs - ENV_NAMES_LIST+="${ENV_NAME},${CVMFS_ENV_PRESENT};" - done - # Actvate starting-env - if [[ "${CVMFS_ENV_PRESENT_START}" == "True" ]]; then - echo "Activating starting-env ${STARTING_ENV} from cvmfs." - source /cvmfs/etp.kit.edu/LAW_envs/conda_envs/miniconda/bin/activate ${STARTING_ENV} - else - echo "Activating starting-env ${STARTING_ENV} from conda." + echo "Activating starting-env ${STARTING_ENV} from miniforge." conda activate ${STARTING_ENV} fi - #Set up other dependencies based on analysis + # Set up other dependencies based on workflow ############################################ case ${ANA_NAME} in KingMaker) @@ -186,15 +181,21 @@ action() { export PYTHONPATH=${MODULE_PYTHONPATH}:${PYTHONPATH} fi - # Check is law was cloned, and set it up if not + # Check is law was set up, and do so if not if [ -z "$(ls -A law)" ]; then git submodule update --init --recursive -- law fi - # add voms proxy path - export X509_USER_PROXY=$(voms-proxy-info -path) - # first check if the user already has a luigid scheduler running - # start a luidigd scheduler if there is one already running + # Check for voms proxy + voms-proxy-info -exists &>/dev/null + if [[ "$?" -eq "1" ]]; then + echo "No valid voms proxy found, remote storage might be inaccessible." + echo "Please ensure that it exists and that 'X509_USER_PROXY' is properly set." + fi + + + # First check if the user already has a luigid scheduler running + # Start a luidigd scheduler if there is one already running if [ -z "$(pgrep -u ${USER} -f luigid)" ]; then echo "Starting Luigi scheduler... using a random port" while @@ -233,7 +234,7 @@ action() { _addpy "${BASE_DIR}/processor" _addpy "${BASE_DIR}/processor/tasks" - # Create law index for analysis if not previously done + # Create law index for workflow if not previously done if [[ ! -f "${LAW_HOME}/index" ]]; then law index --verbose if [[ "$?" -eq "1" ]]; then @@ -242,59 +243,26 @@ action() { fi fi - # set an alias for the sample manager - source scripts/os-version.sh - if [[ "$distro" == "CentOS" ]]; then - if [[ ${os_version:0:1} == "7" ]]; then - lcg_path="/cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-centos7-gcc11-opt/setup.sh" - else - lcg_path="Samplemanager not support on ${distro} ${os_version}" - fi - elif [[ "$distro" == "RedHatEnterprise" || "$distro" == "Alma" || "$distro" == "Rocky" ]]; then - if [[ ${os_version:0:1} == "8" ]]; then - lcg_path="Samplemanager not support on ${distro} ${os_version}" - elif [[ ${os_version:0:1} == "9" ]]; then - lcg_path="/cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-el9-gcc11-opt/setup.sh" - else - lcg_path="Samplemanager not support on ${distro} ${os_version}" - fi - elif [[ "$distro" == "Ubuntu" ]]; then - if [[ ${os_version:0:2} == "20" ]]; then - lcg_path="/cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-ubuntu2004-gcc9-opt/setup.sh" - elif [[ ${os_version:0:2} == "22" ]]; then - lcg_path="/cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh" + # Set the alias + function sample_manager () { + # Determine the directory of this file + if [ ! -z "${ZSH_VERSION}" ]; then + local THIS_FILE="${(%):-%x}" else - lcg_path="Samplemanager not support on ${distro} ${os_version}" + local THIS_FILE="${BASH_SOURCE[0]}" fi - else - lcg_path="Samplemanager not support on ${distro} ${os_version}" - fi - # now set the alias - function sample_manager () { - # determine the directory of this file - if [ ! -z "${ZSH_VERSION}" ]; then - local THIS_FILE="${(%):-%x}" - else - local THIS_FILE="${BASH_SOURCE[0]}" - fi - local BASE_DIR="$( cd "$( dirname "${THIS_FILE}" )" && pwd )" - if [[ "$lcg_path" == "Samplemanager not support on ${distro} ${os_version}" ]]; then - echo ${lcg_path} - else + local BASE_DIR="$( cd "$( dirname "${THIS_FILE}" )" && pwd )" ( - echo "Setting up LCG for Samplemanager" - source ${lcg_path} echo "Starting Samplemanager" python3 ${BASE_DIR}/sample_database/samplemanager/main.py --database-folder ${BASE_DIR}/sample_database ) - fi -} + } -function monitor_production () { - # parse all user arguments and pass them to the python script - python3 scripts/ProductionStatus.py $@ -} + function monitor_production () { + # Parse all user arguments and pass them to the python script + python3 scripts/ProductionStatus.py $@ + } export LAW_IS_SET_UP="True" } diff --git a/setup/dasclient.sh b/setup/dasclient.sh deleted file mode 100644 index 8c2c7f01..00000000 --- a/setup/dasclient.sh +++ /dev/null @@ -1,5 +0,0 @@ -source /cvmfs/grid.cern.ch/umd-c7ui-latest/etc/profile.d/setup-c7-ui-example.sh -export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch -source $VO_CMS_SW_DIR/cmsset_default.sh - -voms-proxy-info \ No newline at end of file diff --git a/setup/setup_crown_cmake.sh b/setup/setup_crown_cmake.sh deleted file mode 100644 index b02ed2fd..00000000 --- a/setup/setup_crown_cmake.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -source $ANALYSIS_PATH/CROWN/init.sh -export X509_USER_PROXY=/home/${USER}/.globus/x509up \ No newline at end of file