Merge pull request #961 from NERC-CEH/ED-86-streamlit-backend

Add streamlit backend
NERC-CEH · Apr 15, 2024 · 7dc8c10 · 7dc8c10
2 parents 8a9eda4 + 190222e
commit 7dc8c10
Show file tree

Hide file tree

Showing 13 changed files with 238 additions and 11 deletions.
diff --git a/code/development-env/config/local/image_config.json b/code/development-env/config/local/image_config.json
@@ -129,6 +129,17 @@
       "masterAddress": "spark://spark-master:7077",
       "sharedRLibs": "/data/packages/R/%p/%v"
     },
+    "streamlit": {
+      "category": "PUBLISH",
+      "userCanChooseFile": true,
+      "userCanChooseConda": true,
+      "versions": [
+        {
+          "displayName": "Dask 2023.1/Spark 3.3.0",
+          "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0"
+        }
+      ]
+    },
     "vscode": {
       "displayName": "VS Code Server",
       "category": "ANALYSIS",

diff --git a/code/workspaces/common/src/config/image_config.json b/code/workspaces/common/src/config/image_config.json
@@ -129,6 +129,17 @@
       "masterAddress": "spark://spark-master:7077",
       "sharedRLibs": "/data/packages/R/%p/%v"
     },
+    "streamlit": {
+      "category": "PUBLISH",
+      "userCanChooseFile": true,
+      "userCanChooseConda": true,
+      "versions": [
+        {
+          "displayName": "Dask 2023.1/Spark 3.3.0",
+          "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0"
+        }
+      ]
+    },
     "vscode": {
       "displayName": "VS Code Server",
       "category": "ANALYSIS",

diff --git a/code/workspaces/common/src/config/images.spec.js b/code/workspaces/common/src/config/images.spec.js
@@ -3,19 +3,19 @@ import { getImageInfoForType, imageCategory, notebookList, siteList, stackList }
 
 describe('stackList', () => {
   it('returns list of NOTEBOOKs and SITEs', () => {
-    expect(stackList().sort()).toEqual(['jupyter', 'jupyterlab', 'nbviewer', 'panel', 'rshiny', 'rstudio', 'voila', 'vscode', 'zeppelin']);
+    expect(stackList().sort()).toEqual(['jupyter', 'jupyterlab', 'nbviewer', 'panel', 'rshiny', 'rstudio', 'streamlit', 'voila', 'vscode', 'zeppelin']);
   });
 });
 
 describe('notebookList', () => {
-  it('returns list of NOTEBOOKs and SITEs', () => {
+  it('returns list of NOTEBOOKs', () => {
     expect(notebookList().sort()).toEqual(['jupyter', 'jupyterlab', 'rstudio', 'vscode', 'zeppelin']);
   });
 });
 
 describe('siteList', () => {
-  it('returns list of NOTEBOOKs and SITEs', () => {
-    expect(siteList().sort()).toEqual(['nbviewer', 'panel', 'rshiny', 'voila']);
+  it('returns list of SITEs', () => {
+    expect(siteList().sort()).toEqual(['nbviewer', 'panel', 'rshiny', 'streamlit', 'voila']);
   });
 });
 
@@ -42,12 +42,6 @@ describe('getImageInfoForType', () => {
   });
 
   it('throws an error when the specified type does not exist', () => {
-    try {
-      getImageInfoForType('does not exist');
-      // if gets to following line the not thrown error so fail test
-      expect(true).toBe(false);
-    } catch (error) {
-      expect(error.message).toEqual('Unable to find config for image of type "does not exist"');
-    }
+    expect(() => getImageInfoForType('does not exist')).toThrow('Unable to find config for image of type "does not exist"');
   });
 });
diff --git a/code/workspaces/common/src/stackTypes.js b/code/workspaces/common/src/stackTypes.js
@@ -10,6 +10,7 @@ const PROJECT = 'project';
 const RSHINY = 'rshiny';
 const RSTUDIO = 'rstudio';
 const SPARK = 'SPARK';
+const STREAMLIT = 'streamlit';
 const VOILA = 'voila';
 const VSCODE = 'vscode';
 const ZEPPELIN = 'zeppelin';
@@ -33,6 +34,7 @@ export {
   RSHINY,
   RSTUDIO,
   SPARK,
+  STREAMLIT,
   VOILA,
   VSCODE,
   ZEPPELIN,

diff --git a/code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml b/code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml
@@ -0,0 +1,61 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    name: {{ name }}
+  name: {{ name  }}
+spec:
+  selector:
+    matchLabels:
+      name: {{ name }}
+  template:
+    metadata:
+      labels:
+        name: {{ name }}
+        user-pod: {{ type }}
+    spec:
+      containers:
+      - name: {{ name }}
+        image: {{ &image }}
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8501
+          protocol: TCP
+        command: ["/bin/bash"]
+        args:
+        - -c
+        - |
+          cd /notebooks
+          [ -n "$CONDA_ENV" ] && source activate "$CONDA_ENV"
+          streamlit run $FILENAME --server.headless=true
+        env:
+        - name: CONDA_ENV
+          value: {{ &condaPath }}
+        - name: FILENAME
+          value: {{ &filename }}
+        livenessProb:
+          httpGet:
+            path: /
+            port: 8501
+          initialDelaySeconds: 5
+          periodSeconds: 10
+{{#volumeMount}}
+        volumeMounts:
+        - name: persistentfsvol
+          mountPath: /notebooks
+          subPath: {{ &sourcePath }}
+          readOnly: true
+        - name: persistentfsvol
+          mountPath: /data/conda
+          subPath: conda
+          readOnly: true
+        - name: persistentfsvol
+          mountPath: /data/.jupyter
+          subPath: .jupyter
+          readOnly: true
+      volumes:
+      - name: persistentfsvol
+        persistentVolumeClaim:
+          claimName: {{ volumeMount }}-claim
+{{/volumeMount}}
diff --git a/code/workspaces/infrastructure-api/resources/streamlit.service.template.yml b/code/workspaces/infrastructure-api/resources/streamlit.service.template.yml
@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ name }}
+spec:
+  ports:
+  - name: {{ name }}-web-ui
+    port: 80
+    targetPort: 8501
+  selector:
+    name: {{ name }}
+  type: NodePort
diff --git a/...rkspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap b/...rkspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap
@@ -733,6 +733,69 @@ spec:
 "
 `;
 
+exports[`deploymentGenerator createSiteDeployment creates expected manifest for streamlit 1`] = `
+"---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    name: streamlit-name
+  name: streamlit-name
+spec:
+  selector:
+    matchLabels:
+      name: streamlit-name
+  template:
+    metadata:
+      labels:
+        name: streamlit-name
+        user-pod: streamlit
+    spec:
+      containers:
+      - name: streamlit-name
+        image: nerc/jupyterlab:ee1e157f36f5-spark-3.3.0
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8501
+          protocol: TCP
+        command: [\\"/bin/bash\\"]
+        args:
+        - -c
+        - |
+          cd /notebooks
+          [ -n \\"$CONDA_ENV\\" ] && source activate \\"$CONDA_ENV\\"
+          streamlit run $FILENAME --server.headless=true
+        env:
+        - name: CONDA_ENV
+          value: /data/conda/my-env
+        - name: FILENAME
+          value: streamlit.py
+        livenessProb:
+          httpGet:
+            path: /
+            port: 8501
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        volumeMounts:
+        - name: persistentfsvol
+          mountPath: /notebooks
+          subPath: notebooks/my-notebook
+          readOnly: true
+        - name: persistentfsvol
+          mountPath: /data/conda
+          subPath: conda
+          readOnly: true
+        - name: persistentfsvol
+          mountPath: /data/.jupyter
+          subPath: .jupyter
+          readOnly: true
+      volumes:
+      - name: persistentfsvol
+        persistentVolumeClaim:
+          claimName: volumeMount-claim
+"
+`;
+
 exports[`deploymentGenerator createSiteService creates expected manifest 1`] = `
 "---
 apiVersion: v1
@@ -767,6 +830,23 @@ spec:
 "
 `;
 
+exports[`deploymentGenerator createSiteService creates expected manifest for streamlit 1`] = `
+"---
+apiVersion: v1
+kind: Service
+metadata:
+  name: streamlit-name
+spec:
+  ports:
+  - name: streamlit-name-web-ui
+    port: 80
+    targetPort: 8501
+  selector:
+    name: streamlit-name
+  type: NodePort
+"
+`;
+
 exports[`deploymentGenerator createVSCodeDeployment generates Jupyter Notebook manifest 1`] = `
 "---
 apiVersion: apps/v1

diff --git a/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js b/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js
@@ -240,6 +240,19 @@ describe('deploymentGenerator', () => {
 
       expect(manifest).toMatchSnapshot();
     });
+
+    it('creates expected manifest for streamlit', async () => {
+      const deploymentName = 'streamlit-name';
+      const sourcePath = 'notebooks/my-notebook';
+      const type = stackTypes.STREAMLIT;
+      const volumeMount = 'volumeMount';
+      const condaPath = '/data/conda/my-env';
+      const filename = 'streamlit.py';
+
+      const manifest = await deploymentGenerator.createSiteDeployment({ deploymentName, sourcePath, type, volumeMount, condaPath, filename });
+
+      expect(manifest).toMatchSnapshot();
+    });
   });
 
   describe('createSiteService', () => {
@@ -259,5 +272,14 @@ describe('deploymentGenerator', () => {
 
       expect(manifest).toMatchSnapshot();
     });
+
+    it('creates expected manifest for streamlit', async () => {
+      const serviceName = 'streamlit-name';
+      const type = stackTypes.STREAMLIT;
+
+      const manifest = await deploymentGenerator.createSiteService({ serviceName, type });
+
+      expect(manifest).toMatchSnapshot();
+    });
   });
 });
diff --git a/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js b/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js
@@ -15,6 +15,7 @@ const ServiceTemplates = Object.freeze({
   SPARK_DRIVER_HEADLESS_SERVICE: 'spark-driver.headless-service.template.yml',
   DATALAB_DASK_SCHEDULER_SERVICE: 'datalab-dask-scheduler.service.template.yml',
   DATALAB_SPARK_SCHEDULER_SERVICE: 'datalab-spark-scheduler.service.template.yml',
+  STREAMLIT_SERVICE: 'streamlit.service.template.yml',
 });
 
 const DeploymentTemplates = Object.freeze({
@@ -32,6 +33,7 @@ const DeploymentTemplates = Object.freeze({
   DATALAB_DASK_WORKER_DEPLOYMENT: 'datalab-dask-worker.deployment.template.yml',
   DATALAB_SPARK_SCHEDULER_DEPLOYMENT: 'datalab-spark-scheduler.deployment.template.yml',
   DATALAB_SPARK_WORKER_DEPLOYMENT: 'datalab-spark-worker.deployment.template.yml',
+  STREAMLIT_DEPLOYMENT: 'streamlit.deployment.template.yml',
 });
 
 const IngressTemplates = Object.freeze({

diff --git a/code/workspaces/infrastructure-api/src/stacks/Stacks.js b/code/workspaces/infrastructure-api/src/stacks/Stacks.js
@@ -53,6 +53,11 @@ const STACKS = Object.freeze({
     create: siteStack.createSiteStack,
     delete: siteStack.deleteSiteStack,
   },
+  STREAMLIT: {
+    type: stackTypes.STREAMLIT,
+    create: siteStack.createSiteStack,
+    delete: siteStack.deleteSiteStack,
+  },
 });
 
 const getStack = type => find(STACKS, ['type', type]);

diff --git a/code/workspaces/web-app/public/image_config.json b/code/workspaces/web-app/public/image_config.json
@@ -129,6 +129,17 @@
       "masterAddress": "spark://spark-master:7077",
       "sharedRLibs": "/data/packages/R/%p/%v"
     },
+    "streamlit": {
+      "category": "PUBLISH",
+      "userCanChooseFile": true,
+      "userCanChooseConda": true,
+      "versions": [
+        {
+          "displayName": "Dask 2023.1/Spark 3.3.0",
+          "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0"
+        }
+      ]
+    },
     "vscode": {
       "displayName": "VS Code Server",
       "category": "ANALYSIS",

diff --git a/...rkspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap b/...rkspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap
@@ -38,6 +38,10 @@ Array [
     "text": undefined,
     "value": "spark",
   },
+  Object {
+    "text": undefined,
+    "value": "streamlit",
+  },
   Object {
     "text": "VS Code Server",
     "value": "vscode",

diff --git a/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap b/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap
@@ -108,6 +108,17 @@ Object {
       },
     ],
   },
+  "streamlit": Object {
+    "category": "PUBLISH",
+    "userCanChooseConda": true,
+    "userCanChooseFile": true,
+    "versions": Array [
+      Object {
+        "displayName": "Dask 2023.1/Spark 3.3.0",
+        "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0",
+      },
+    ],
+  },
   "voila": Object {
     "category": "PUBLISH",
     "displayName": "Voila",