From 3e7417ba7e06ee6aa43f9d238daa51fcb535d7e6 Mon Sep 17 00:00:00 2001
From: Sam Stoelinga <sammiestoel@gmail.com>
Date: Sun, 1 Oct 2023 00:04:57 -0700
Subject: [PATCH] Change working directory when running notebook (#48)

* Also remove the scripts to run and convert notebooks from model-loader
and model-trainer
---
 base/scripts/run-notebook.py               | 16 ++++++++++++++--
 base/scripts/run-notebook.sh               |  9 +++------
 model-loader-huggingface/Dockerfile        |  3 ---
 model-loader-huggingface/load.sh           |  5 -----
 model-trainer-huggingface/Dockerfile       |  3 ---
 model-trainer-huggingface/scripts/train.sh |  6 ------
 6 files changed, 17 insertions(+), 25 deletions(-)
 delete mode 100755 model-loader-huggingface/load.sh
 delete mode 100755 model-trainer-huggingface/scripts/train.sh

diff --git a/base/scripts/run-notebook.py b/base/scripts/run-notebook.py
index d78c646..7606cad 100755
--- a/base/scripts/run-notebook.py
+++ b/base/scripts/run-notebook.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 from pathlib import Path
 import logging
@@ -9,6 +10,16 @@
 from papermill.utils import merge_kwargs, remove_args
 from nbconvert import HTMLExporter
 
+
+parser = argparse.ArgumentParser(
+    prog='run-notebook.py',
+    description='Run a notebook and converts it to HTML while it is running')
+
+parser.add_argument('notebook_path', help="The path to the .ipynb notebook file")
+parser.add_argument('notebook_output_path', help="The path where the .ipynb notebook file should be written")
+parser.add_argument('-c', '--cwd', help="Change the working directory")
+args = parser.parse_args()
+
 logging.basicConfig(level=logging.INFO, format="%(message)s")
 logger.setLevel(logging.INFO)
 
@@ -68,8 +79,9 @@ def execute_notebook(
 
 # Execute the notebook with papermill as usual
 papermill.execute_notebook(
-    sys.argv[1],
-    sys.argv[2],
+    args.notebook_path,
+    args.notebook_output_path,
+    cwd=args.cwd,
     log_output=True,
     engine_name="htmlengine",
 )
\ No newline at end of file
diff --git a/base/scripts/run-notebook.sh b/base/scripts/run-notebook.sh
index 593ff36..617871a 100755
--- a/base/scripts/run-notebook.sh
+++ b/base/scripts/run-notebook.sh
@@ -5,21 +5,18 @@ if [[ -z "$NOTEBOOK" ]]; then
   for nb in /content/*.ipynb; do
     [ -e "$nb" ] || continue
     echo "Executing notebook $nb"
-    run-notebook.py "$nb" "/content/artifacts/$(basename ${nb})"
-    # jupyter nbconvert --to html /content/artifacts/$(basename "$nb")
+    run-notebook.py "$nb" "/content/artifacts/$(basename ${nb})" --cwd "$(dirname ${nb})"
   done
   for nb in /content/src/*.ipynb; do
     [ -e "$nb" ] || continue
     mkdir -p /content/artifacts/src
     echo "Executing notebook $nb"
-    run-notebook.py "$nb" "/content/artifacts/src/$(basename ${nb})"
-    # jupyter nbconvert --to html /content/artifacts//src/$(basename "$nb")
+    run-notebook.py "$nb" "/content/artifacts/src/$(basename ${nb})" --cwd "$(dirname ${nb})"
   done
 else
   for nb in $NOTEBOOK; do
     [ -e "$nb" ] || continue
     echo "Executing notebook $nb"
-    run-notebook.py "$nb" "/content/artifacts/$(basename ${nb})"
-    # jupyter nbconvert --to html /content/artifacts/$(basename "$nb")
+    run-notebook.py "$nb" "/content/artifacts/$(basename ${nb})" --cwd "$(dirname ${nb})"
   done
 fi
\ No newline at end of file
diff --git a/model-loader-huggingface/Dockerfile b/model-loader-huggingface/Dockerfile
index 85555a2..ae72605 100644
--- a/model-loader-huggingface/Dockerfile
+++ b/model-loader-huggingface/Dockerfile
@@ -6,7 +6,4 @@ RUN mkdir -p /content/model
 COPY requirements.txt requirements.txt
 RUN pip3 install --no-cache-dir -r requirements.txt
 
-COPY load.sh /usr/local/bin/
 COPY src/* src/
-
-CMD load.sh
diff --git a/model-loader-huggingface/load.sh b/model-loader-huggingface/load.sh
deleted file mode 100755
index f842946..0000000
--- a/model-loader-huggingface/load.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env sh
-
-set -xe
-
-jupyter nbconvert --debug --to notebook --execute /content/src/load.ipynb --output /content/logs/load.ipynb
diff --git a/model-trainer-huggingface/Dockerfile b/model-trainer-huggingface/Dockerfile
index 13b615b..e8fe948 100644
--- a/model-trainer-huggingface/Dockerfile
+++ b/model-trainer-huggingface/Dockerfile
@@ -7,7 +7,4 @@ WORKDIR /content
 COPY requirements.txt /content/requirements.txt
 RUN pip install --no-cache-dir -r /content/requirements.txt
 
-COPY scripts/* ./scripts/
 COPY src/* ./src/
-
-CMD train.sh
diff --git a/model-trainer-huggingface/scripts/train.sh b/model-trainer-huggingface/scripts/train.sh
deleted file mode 100755
index 30186b1..0000000
--- a/model-trainer-huggingface/scripts/train.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env sh
-
-set -xe
-
-export HF_HUB_DISABLE_PROGRESS_BARS=1
-jupyter nbconvert --debug --to notebook --execute /content/src/train.ipynb --output /content/logs/train.ipynb