From 5a6ad67ce72b9639ff67b406134265b85ea210d0 Mon Sep 17 00:00:00 2001 From: Josh Holland Date: Mon, 8 Apr 2024 13:59:42 +0100 Subject: [PATCH 1/2] Improve images.spec.js test in common Use the Jest `toThrow` built-in matcher, and fix the descriptions of some other test cases. --- code/workspaces/common/src/config/images.spec.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/code/workspaces/common/src/config/images.spec.js b/code/workspaces/common/src/config/images.spec.js index f7cbefe1b..71a628766 100644 --- a/code/workspaces/common/src/config/images.spec.js +++ b/code/workspaces/common/src/config/images.spec.js @@ -8,13 +8,13 @@ describe('stackList', () => { }); describe('notebookList', () => { - it('returns list of NOTEBOOKs and SITEs', () => { + it('returns list of NOTEBOOKs', () => { expect(notebookList().sort()).toEqual(['jupyter', 'jupyterlab', 'rstudio', 'vscode', 'zeppelin']); }); }); describe('siteList', () => { - it('returns list of NOTEBOOKs and SITEs', () => { + it('returns list of SITEs', () => { expect(siteList().sort()).toEqual(['nbviewer', 'panel', 'rshiny', 'voila']); }); }); @@ -42,12 +42,6 @@ describe('getImageInfoForType', () => { }); it('throws an error when the specified type does not exist', () => { - try { - getImageInfoForType('does not exist'); - // if gets to following line the not thrown error so fail test - expect(true).toBe(false); - } catch (error) { - expect(error.message).toEqual('Unable to find config for image of type "does not exist"'); - } + expect(() => getImageInfoForType('does not exist')).toThrow('Unable to find config for image of type "does not exist"'); }); }); From 190222ed718c0133bb78ee9a6f84ad5860611528 Mon Sep 17 00:00:00 2001 From: Josh Holland Date: Mon, 8 Apr 2024 13:59:42 +0100 Subject: [PATCH 2/2] Add backend support for Streamlit sites --- .../config/local/image_config.json | 11 +++ .../common/src/config/image_config.json | 11 +++ .../common/src/config/images.spec.js | 4 +- code/workspaces/common/src/stackTypes.js | 2 + .../streamlit.deployment.template.yml | 61 ++++++++++++++ .../resources/streamlit.service.template.yml | 13 +++ .../deploymentGenerator.spec.js.snap | 80 +++++++++++++++++++ .../kubernetes/deploymentGenerator.spec.js | 22 +++++ .../src/kubernetes/manifestGenerator.js | 2 + .../infrastructure-api/src/stacks/Stacks.js | 5 ++ .../web-app/public/image_config.json | 11 +++ .../typeAndVersionFormUtils.spec.js.snap | 4 + .../config/__snapshots__/images.spec.js.snap | 11 +++ 13 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml create mode 100644 code/workspaces/infrastructure-api/resources/streamlit.service.template.yml diff --git a/code/development-env/config/local/image_config.json b/code/development-env/config/local/image_config.json index 1e9d98525..d42459d8c 100644 --- a/code/development-env/config/local/image_config.json +++ b/code/development-env/config/local/image_config.json @@ -129,6 +129,17 @@ "masterAddress": "spark://spark-master:7077", "sharedRLibs": "/data/packages/R/%p/%v" }, + "streamlit": { + "category": "PUBLISH", + "userCanChooseFile": true, + "userCanChooseConda": true, + "versions": [ + { + "displayName": "Dask 2023.1/Spark 3.3.0", + "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0" + } + ] + }, "vscode": { "displayName": "VS Code Server", "category": "ANALYSIS", diff --git a/code/workspaces/common/src/config/image_config.json b/code/workspaces/common/src/config/image_config.json index 8f9cae03a..b0a65ee13 100644 --- a/code/workspaces/common/src/config/image_config.json +++ b/code/workspaces/common/src/config/image_config.json @@ -129,6 +129,17 @@ "masterAddress": "spark://spark-master:7077", "sharedRLibs": "/data/packages/R/%p/%v" }, + "streamlit": { + "category": "PUBLISH", + "userCanChooseFile": true, + "userCanChooseConda": true, + "versions": [ + { + "displayName": "Dask 2023.1/Spark 3.3.0", + "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0" + } + ] + }, "vscode": { "displayName": "VS Code Server", "category": "ANALYSIS", diff --git a/code/workspaces/common/src/config/images.spec.js b/code/workspaces/common/src/config/images.spec.js index 71a628766..763af6642 100644 --- a/code/workspaces/common/src/config/images.spec.js +++ b/code/workspaces/common/src/config/images.spec.js @@ -3,7 +3,7 @@ import { getImageInfoForType, imageCategory, notebookList, siteList, stackList } describe('stackList', () => { it('returns list of NOTEBOOKs and SITEs', () => { - expect(stackList().sort()).toEqual(['jupyter', 'jupyterlab', 'nbviewer', 'panel', 'rshiny', 'rstudio', 'voila', 'vscode', 'zeppelin']); + expect(stackList().sort()).toEqual(['jupyter', 'jupyterlab', 'nbviewer', 'panel', 'rshiny', 'rstudio', 'streamlit', 'voila', 'vscode', 'zeppelin']); }); }); @@ -15,7 +15,7 @@ describe('notebookList', () => { describe('siteList', () => { it('returns list of SITEs', () => { - expect(siteList().sort()).toEqual(['nbviewer', 'panel', 'rshiny', 'voila']); + expect(siteList().sort()).toEqual(['nbviewer', 'panel', 'rshiny', 'streamlit', 'voila']); }); }); diff --git a/code/workspaces/common/src/stackTypes.js b/code/workspaces/common/src/stackTypes.js index b1e75b9b0..fbb728b5e 100644 --- a/code/workspaces/common/src/stackTypes.js +++ b/code/workspaces/common/src/stackTypes.js @@ -10,6 +10,7 @@ const PROJECT = 'project'; const RSHINY = 'rshiny'; const RSTUDIO = 'rstudio'; const SPARK = 'SPARK'; +const STREAMLIT = 'streamlit'; const VOILA = 'voila'; const VSCODE = 'vscode'; const ZEPPELIN = 'zeppelin'; @@ -33,6 +34,7 @@ export { RSHINY, RSTUDIO, SPARK, + STREAMLIT, VOILA, VSCODE, ZEPPELIN, diff --git a/code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml b/code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml new file mode 100644 index 000000000..583a6a8a9 --- /dev/null +++ b/code/workspaces/infrastructure-api/resources/streamlit.deployment.template.yml @@ -0,0 +1,61 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + name: {{ name }} + name: {{ name }} +spec: + selector: + matchLabels: + name: {{ name }} + template: + metadata: + labels: + name: {{ name }} + user-pod: {{ type }} + spec: + containers: + - name: {{ name }} + image: {{ &image }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8501 + protocol: TCP + command: ["/bin/bash"] + args: + - -c + - | + cd /notebooks + [ -n "$CONDA_ENV" ] && source activate "$CONDA_ENV" + streamlit run $FILENAME --server.headless=true + env: + - name: CONDA_ENV + value: {{ &condaPath }} + - name: FILENAME + value: {{ &filename }} + livenessProb: + httpGet: + path: / + port: 8501 + initialDelaySeconds: 5 + periodSeconds: 10 +{{#volumeMount}} + volumeMounts: + - name: persistentfsvol + mountPath: /notebooks + subPath: {{ &sourcePath }} + readOnly: true + - name: persistentfsvol + mountPath: /data/conda + subPath: conda + readOnly: true + - name: persistentfsvol + mountPath: /data/.jupyter + subPath: .jupyter + readOnly: true + volumes: + - name: persistentfsvol + persistentVolumeClaim: + claimName: {{ volumeMount }}-claim +{{/volumeMount}} diff --git a/code/workspaces/infrastructure-api/resources/streamlit.service.template.yml b/code/workspaces/infrastructure-api/resources/streamlit.service.template.yml new file mode 100644 index 000000000..6a3e0ab33 --- /dev/null +++ b/code/workspaces/infrastructure-api/resources/streamlit.service.template.yml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ name }} +spec: + ports: + - name: {{ name }}-web-ui + port: 80 + targetPort: 8501 + selector: + name: {{ name }} + type: NodePort diff --git a/code/workspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap b/code/workspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap index 1876537a3..4b20d7246 100644 --- a/code/workspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap +++ b/code/workspaces/infrastructure-api/src/kubernetes/__snapshots__/deploymentGenerator.spec.js.snap @@ -733,6 +733,69 @@ spec: " `; +exports[`deploymentGenerator createSiteDeployment creates expected manifest for streamlit 1`] = ` +"--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + name: streamlit-name + name: streamlit-name +spec: + selector: + matchLabels: + name: streamlit-name + template: + metadata: + labels: + name: streamlit-name + user-pod: streamlit + spec: + containers: + - name: streamlit-name + image: nerc/jupyterlab:ee1e157f36f5-spark-3.3.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8501 + protocol: TCP + command: [\\"/bin/bash\\"] + args: + - -c + - | + cd /notebooks + [ -n \\"$CONDA_ENV\\" ] && source activate \\"$CONDA_ENV\\" + streamlit run $FILENAME --server.headless=true + env: + - name: CONDA_ENV + value: /data/conda/my-env + - name: FILENAME + value: streamlit.py + livenessProb: + httpGet: + path: / + port: 8501 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: persistentfsvol + mountPath: /notebooks + subPath: notebooks/my-notebook + readOnly: true + - name: persistentfsvol + mountPath: /data/conda + subPath: conda + readOnly: true + - name: persistentfsvol + mountPath: /data/.jupyter + subPath: .jupyter + readOnly: true + volumes: + - name: persistentfsvol + persistentVolumeClaim: + claimName: volumeMount-claim +" +`; + exports[`deploymentGenerator createSiteService creates expected manifest 1`] = ` "--- apiVersion: v1 @@ -767,6 +830,23 @@ spec: " `; +exports[`deploymentGenerator createSiteService creates expected manifest for streamlit 1`] = ` +"--- +apiVersion: v1 +kind: Service +metadata: + name: streamlit-name +spec: + ports: + - name: streamlit-name-web-ui + port: 80 + targetPort: 8501 + selector: + name: streamlit-name + type: NodePort +" +`; + exports[`deploymentGenerator createVSCodeDeployment generates Jupyter Notebook manifest 1`] = ` "--- apiVersion: apps/v1 diff --git a/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js b/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js index 890fa69bb..fef99b7ad 100644 --- a/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js +++ b/code/workspaces/infrastructure-api/src/kubernetes/deploymentGenerator.spec.js @@ -240,6 +240,19 @@ describe('deploymentGenerator', () => { expect(manifest).toMatchSnapshot(); }); + + it('creates expected manifest for streamlit', async () => { + const deploymentName = 'streamlit-name'; + const sourcePath = 'notebooks/my-notebook'; + const type = stackTypes.STREAMLIT; + const volumeMount = 'volumeMount'; + const condaPath = '/data/conda/my-env'; + const filename = 'streamlit.py'; + + const manifest = await deploymentGenerator.createSiteDeployment({ deploymentName, sourcePath, type, volumeMount, condaPath, filename }); + + expect(manifest).toMatchSnapshot(); + }); }); describe('createSiteService', () => { @@ -259,5 +272,14 @@ describe('deploymentGenerator', () => { expect(manifest).toMatchSnapshot(); }); + + it('creates expected manifest for streamlit', async () => { + const serviceName = 'streamlit-name'; + const type = stackTypes.STREAMLIT; + + const manifest = await deploymentGenerator.createSiteService({ serviceName, type }); + + expect(manifest).toMatchSnapshot(); + }); }); }); diff --git a/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js b/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js index 1bcf5ed82..43d6743b9 100644 --- a/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js +++ b/code/workspaces/infrastructure-api/src/kubernetes/manifestGenerator.js @@ -15,6 +15,7 @@ const ServiceTemplates = Object.freeze({ SPARK_DRIVER_HEADLESS_SERVICE: 'spark-driver.headless-service.template.yml', DATALAB_DASK_SCHEDULER_SERVICE: 'datalab-dask-scheduler.service.template.yml', DATALAB_SPARK_SCHEDULER_SERVICE: 'datalab-spark-scheduler.service.template.yml', + STREAMLIT_SERVICE: 'streamlit.service.template.yml', }); const DeploymentTemplates = Object.freeze({ @@ -32,6 +33,7 @@ const DeploymentTemplates = Object.freeze({ DATALAB_DASK_WORKER_DEPLOYMENT: 'datalab-dask-worker.deployment.template.yml', DATALAB_SPARK_SCHEDULER_DEPLOYMENT: 'datalab-spark-scheduler.deployment.template.yml', DATALAB_SPARK_WORKER_DEPLOYMENT: 'datalab-spark-worker.deployment.template.yml', + STREAMLIT_DEPLOYMENT: 'streamlit.deployment.template.yml', }); const IngressTemplates = Object.freeze({ diff --git a/code/workspaces/infrastructure-api/src/stacks/Stacks.js b/code/workspaces/infrastructure-api/src/stacks/Stacks.js index f0f3c6db8..1b47910fb 100644 --- a/code/workspaces/infrastructure-api/src/stacks/Stacks.js +++ b/code/workspaces/infrastructure-api/src/stacks/Stacks.js @@ -53,6 +53,11 @@ const STACKS = Object.freeze({ create: siteStack.createSiteStack, delete: siteStack.deleteSiteStack, }, + STREAMLIT: { + type: stackTypes.STREAMLIT, + create: siteStack.createSiteStack, + delete: siteStack.deleteSiteStack, + }, }); const getStack = type => find(STACKS, ['type', type]); diff --git a/code/workspaces/web-app/public/image_config.json b/code/workspaces/web-app/public/image_config.json index 1e9d98525..d42459d8c 100644 --- a/code/workspaces/web-app/public/image_config.json +++ b/code/workspaces/web-app/public/image_config.json @@ -129,6 +129,17 @@ "masterAddress": "spark://spark-master:7077", "sharedRLibs": "/data/packages/R/%p/%v" }, + "streamlit": { + "category": "PUBLISH", + "userCanChooseFile": true, + "userCanChooseConda": true, + "versions": [ + { + "displayName": "Dask 2023.1/Spark 3.3.0", + "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0" + } + ] + }, "vscode": { "displayName": "VS Code Server", "category": "ANALYSIS", diff --git a/code/workspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap b/code/workspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap index 5afe841fd..a26ef3118 100644 --- a/code/workspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap +++ b/code/workspaces/web-app/src/components/stacks/__snapshots__/typeAndVersionFormUtils.spec.js.snap @@ -38,6 +38,10 @@ Array [ "text": undefined, "value": "spark", }, + Object { + "text": undefined, + "value": "streamlit", + }, Object { "text": "VS Code Server", "value": "vscode", diff --git a/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap b/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap index 5bf473c43..1829d19fd 100644 --- a/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap +++ b/code/workspaces/web-app/src/config/__snapshots__/images.spec.js.snap @@ -108,6 +108,17 @@ Object { }, ], }, + "streamlit": Object { + "category": "PUBLISH", + "userCanChooseConda": true, + "userCanChooseFile": true, + "versions": Array [ + Object { + "displayName": "Dask 2023.1/Spark 3.3.0", + "image": "nerc/jupyterlab:ee1e157f36f5-spark-3.3.0", + }, + ], + }, "voila": Object { "category": "PUBLISH", "displayName": "Voila",