First class files

.github/workflows/test.yaml

@@ -66,6 +66,7 @@ jobs:
       - name: "Run unit tests"
         id: runtests
         run: |
+          export SM_ENVIRONMENT=local
           coverage run -m unittest discover -p 'test*.py' -s '.'
           rc=$?
           coverage xml

.gitignore

@@ -60,3 +60,9 @@ web/src/__generated__
 
 # pulumi config files
 Pulumi*.yaml
+
+# pnpm package manager
+pnpm-lock.yaml
+
+# env
+.env

api/graphql/schema.py

@@ -235,27 +235,29 @@ class GraphQLAnalysis:
     id: int
     type: str
     status: strawberry.enum(AnalysisStatus)
-    output: str | None
     timestamp_completed: datetime.datetime | None = None
     active: bool
     meta: strawberry.scalars.JSON
-
+    output: strawberry.scalars.JSON
+    outputs: strawberry.scalars.JSON
     @staticmethod
     def from_internal(internal: AnalysisInternal) -> 'GraphQLAnalysis':
         return GraphQLAnalysis(
             id=internal.id,
             type=internal.type,
             status=internal.status,
-            output=internal.output,
             timestamp_completed=internal.timestamp_completed,
             active=internal.active,
             meta=internal.meta,
+            output=internal.output,
+            outputs=internal.outputs,
         )
 
     @strawberry.field
     async def sequencing_groups(
         self, info: Info, root: 'GraphQLAnalysis'
     ) -> list['GraphQLSequencingGroup']:
+
         loader = info.context[LoaderKeys.SEQUENCING_GROUPS_FOR_ANALYSIS]
         sgs = await loader.load(root.id)
         return [GraphQLSequencingGroup.from_internal(sg) for sg in sgs]

api/routes/analysis.py

@@ -1,7 +1,7 @@
 import csv
 import io
 from datetime import date
-from typing import Any
+from typing import Any, Optional, Union
 
 from fastapi import APIRouter
 from fastapi.params import Body, Query
@@ -45,7 +45,7 @@ class AnalysisModel(BaseModel):
     type: str
     status: AnalysisStatus
     meta: dict[str, Any] | None = None
-    output: str | None = None
+    outputs: Optional[Union[str, dict]] = None
     active: bool = True
     # please don't use this, unless you're the analysis-runner,
     # the usage is tracked ... (Ծ_Ծ)
@@ -56,7 +56,7 @@ class AnalysisUpdateModel(BaseModel):
     """Update analysis model"""
 
     status: AnalysisStatus
-    output: str | None = None
+    outputs: str | None = None
     meta: dict[str, Any] | None = None
     active: bool | None = None
 
@@ -73,7 +73,7 @@ class AnalysisQueryModel(BaseModel):
     type: str | None = None
     status: AnalysisStatus | None = None
     meta: dict[str, Any] | None = None
-    output: str | None = None
+    outputs: str | None = None
     active: bool | None = None
 
     def to_filter(self, project_id_map: dict[str, int]) -> AnalysisFilter:
@@ -130,7 +130,7 @@ async def update_analysis(
     """Update status of analysis"""
     atable = AnalysisLayer(connection)
     await atable.update_analysis(
-        analysis_id, status=analysis.status, output=analysis.output, meta=analysis.meta
+        analysis_id, status=analysis.status, outputs=analysis.outputs, meta=analysis.meta
     )
     return True
 

db/project.xml

@@ -1107,4 +1107,90 @@
 		<sql>ALTER TABLE sequencing_group_assay CHANGE author author VARCHAR(255) NULL;</sql>
 		<sql>ALTER TABLE sequencing_group_external_id CHANGE author author VARCHAR(255) NULL;</sql>
 	</changeSet>
+	<changeSet id="2024-01-23_output_file" author="yash.pankhania">
+		<sql>SET @@system_versioning_alter_history = 1;</sql>
+		<createTable tableName="output_file">
+			<column name="id" type="INT" autoIncrement="true">
+				<constraints primaryKey="true" nullable="false" />
+			</column>
+			<column name="path" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+					unique="true"
+				/>
+			</column>
+			<column name="basename" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="dirname" type="VARCHAR(100)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="nameroot" type="VARCHAR(255)">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="nameext" type="VARCHAR(25)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="file_checksum" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="size" type="BIGINT">
+				<constraints
+					nullable="false"
+				/>
+			</column>
+			<column name="meta" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+			<column name="valid" type="BOOLEAN" />
+			<column name="parent_id" type="INT">
+				<constraints
+					nullable="true"
+					foreignKeyName="FK_SECONDARY_FILE_PARENT_ID"
+					references="output_file(id)"
+				/>
+			</column>
+		</createTable>
+		<createTable tableName="analysis_outputs">
+			<column name="analysis_id" type="INT">
+				<constraints
+				nullable="false"
+				foreignKeyName="FK_ANALYSIS_OUTPUTS_ANALYSIS_ID"
+				references="analysis(id)"
+				/>
+			</column>
+			<column name="file_id" type="INT">
+				<constraints
+				nullable="true"
+				foreignKeyName="FK_ANALYSIS_OUTPUTS_FILE_ID"
+				references="output_file(id)"
+				/>
+			</column>
+			<column name="output" type="VARCHAR(255)">
+				<constraints
+				nullable="true"
+				/>
+			</column>
+			<column name="json_structure" type="VARCHAR(255)">
+				<constraints
+					nullable="true"
+				/>
+			</column>
+		</createTable>
+		<sql>ALTER TABLE `output_file` ADD SYSTEM VERSIONING;</sql>
+		<sql>ALTER TABLE `analysis_outputs` ADD SYSTEM VERSIONING;</sql>
+		<sql>ALTER TABLE `analysis_outputs` ADD CONSTRAINT `chk_file_id_output` CHECK ((file_id IS NOT NULL AND output IS NULL) OR (file_id IS NULL AND output IS NOT NULL));</sql>
+	</changeSet>
 </databaseChangeLog>

db/python/connect.py

@@ -26,6 +26,7 @@
     'sequencing_group',
     'assay',
     'sequencing_group_assay',
+    'analysis_outputs',
     'analysis_sequencing_group',
     'analysis_sample',
     'assay_external_id',