Merge pull request #928 from populationgenomics/dev

Release - ETL acessors, pip audit, Sonarqube & Metamist comments

.github/workflows/lint.yaml

@@ -34,6 +34,18 @@ jobs:
           # openapi-generator
           wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/5.3.0/openapi-generator-cli-5.3.0.jar -O openapi-generator-cli.jar
 
+      - name: 'install frontend deps'
+        working-directory: ./web
+        run: npm ci
+
+      - name: 'check frontend formatting'
+        working-directory: ./web
+        run: npm run format:check
+
+      - name: 'check frontend linting'
+        working-directory: ./web
+        run: npm run lint
+
       - name: 'build image'
         run: |
           docker build \

.github/workflows/security_checks.yaml

@@ -0,0 +1,16 @@
+name: Security Checks
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  pip-audit:
+    runs-on: ubuntu-latest
+    name: Pip Audit
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pypa/gh-action-pip-audit@v1.1.0
+        with:
+          inputs: requirements.txt requirements-dev.txt
+          summary: true

.github/workflows/test.yaml

@@ -1,8 +1,9 @@
-name: Unittests
+name: Test
 on: push
 
 jobs:
-  run-unittests:
+  unittests:
+    name: Run unit tests
     # Run on merge to main, where the commit name starts with "Bump version:" (for bump2version)
     #    if: "startsWith(github.event.head_commit.message, 'Bump version:')"
     runs-on: ubuntu-latest
@@ -63,7 +64,7 @@ jobs:
       - name: 'Run unit tests'
         id: runtests
         run: |
-          coverage run -m pytest --doctest-modules --doctest-continue-on-failure test/
+          coverage run -m pytest --doctest-modules --doctest-continue-on-failure test/ --junitxml=test-execution.xml
           rc=$?
           coverage xml
 
@@ -75,6 +76,18 @@ jobs:
           files: ./coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
 
+      - name: 'Save coverage report as an Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: ./coverage.xml
+
+      - name: 'Save execution report as an Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: execution-report
+          path: ./test-execution.xml
+
       - name: 'build web front-end'
         run: |
           set -eo pipefail
@@ -101,3 +114,34 @@ jobs:
         with:
           script: |
             core.setFailed('Web failed to build with rc = ${{ steps.runtests.outputs.web_rc }}')
+
+  sonarqube:
+    name: SonarQube scan
+    runs-on: ubuntu-latest
+    needs: unittests
+    environment: production
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+
+      # Download the coverage report artifact
+      - name: 'Download coverage and execution report'
+        uses: actions/download-artifact@v4
+        with:
+          pattern: '*-report'
+
+      # Perform the SonarQube scan
+      - uses: sonarsource/sonarqube-scan-action@master
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+
+      # Optional: Fail the job if Quality Gate is red
+      # If you wish to fail your job when the Quality Gate is red, uncomment the
+      # following lines. This would typically be used to fail a deployment.
+      # - uses: sonarsource/sonarqube-quality-gate-action@master
+      #   timeout-minutes: 5
+      #   env:
+      #     SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -37,6 +37,7 @@ build/
 test/*.sh
 .coverage
 coverage.xml
+test-execution.xml
 temp.json
 data-dump.sql
 local.py
@@ -64,3 +65,6 @@ Pulumi*.yaml
 
 # performance profile files
 profiles
+
+# sonarqube
+.scannerwork/

.pre-commit-config.yaml

@@ -72,34 +72,3 @@ repos:
         additional_dependencies:
           - strawberry-graphql[fastapi]==0.206.0
           - types-PyMySQL==1.1.0.1
-
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.0.0-alpha.4"
-    hooks:
-      - id: prettier
-        # I'm not exactly sure why it changes behaviour, but
-        # calling `cd web`, then calling `ls src/**/*.tsx`
-        # returns different results to `cd web && ls src/**/*.tsx`
-        # so just include both patterns here
-        entry: bash -c 'cd web && prettier --write --ignore-unknown --check src/*.{ts,tsx,css} src/**/*.{ts,tsx,css}'
-
-  - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: "v8.33.0"
-    hooks:
-      - id: eslint
-        entry: bash -c 'cd web && eslint'
-        files: \.[jt]sx?$
-        types: [file]
-        additional_dependencies:
-          - eslint@^7.32.0
-          - eslint-config-airbnb@^19.0.4
-          - eslint-config-airbnb-base@^15.0.0
-          - eslint-config-airbnb-typescript@^17.0.0
-          - eslint-config-prettier@^8.6.0
-          - eslint-plugin-import@^2.26.0
-          - eslint-plugin-jsx-a11y@^6.6.1
-          - eslint-plugin-prettier@^4.2.1
-          - eslint-plugin-react@^7.31.11
-          - eslint-plugin-react-hooks@^4.6.0
-          - "@typescript-eslint/eslint-plugin@^5.48.0"
-          - "@typescript-eslint/parser@^5.48.0"

api/graphql/loaders.py

@@ -22,6 +22,7 @@
     SampleLayer,
     SequencingGroupLayer,
 )
+from db.python.layers.comment import CommentLayer
 from db.python.tables.analysis import AnalysisFilter
 from db.python.tables.assay import AssayFilter
 from db.python.tables.family import FamilyFilter
@@ -40,6 +41,7 @@
     SequencingGroupInternal,
 )
 from models.models.audit_log import AuditLogInternal
+from models.models.comment import CommentEntityType, DiscussionInternal
 from models.models.family import PedRowInternal
 
 
@@ -56,6 +58,7 @@ class LoaderKeys(enum.Enum):
 
     ANALYSES_FOR_SEQUENCING_GROUPS = 'analyses_for_sequencing_groups'
 
+    ASSAYS_FOR_IDS = 'assays_for_ids'
     ASSAYS_FOR_SAMPLES = 'sequences_for_samples'
     ASSAYS_FOR_SEQUENCING_GROUPS = 'assays_for_sequencing_groups'
 
@@ -80,6 +83,13 @@ class LoaderKeys(enum.Enum):
     SEQUENCING_GROUPS_FOR_PROJECTS = 'sequencing_groups_for_projects'
     SEQUENCING_GROUPS_FOR_ANALYSIS = 'sequencing_groups_for_analysis'
 
+    COMMENTS_FOR_SAMPLE_IDS = 'comments_for_sample_ids'
+    COMMENTS_FOR_PARTICIPANT_IDS = 'comments_for_participant_ids'
+    COMMENTS_FOR_ASSAY_IDS = 'comments_for_assay_ids'
+    COMMENTS_FOR_PROJECT_IDS = 'comments_for_project_ids'
+    COMMENTS_FOR_SEQUENCING_GROUP_IDS = 'comments_for_sequencing_group_ids'
+    COMMENTS_FOR_FAMILY_IDS = 'comments_for_family_ids'
+
 
 loaders: dict[LoaderKeys, Any] = {}
 
@@ -184,6 +194,20 @@ async def load_audit_logs_by_analysis_ids(
     return [logs.get(a) or [] for a in analysis_ids]
 
 
+@connected_data_loader(LoaderKeys.ASSAYS_FOR_IDS)
+async def load_assays_for_ids(
+    assay_ids: list[int], connection: Connection
+) -> list[AssayInternal]:
+    """
+    DataLoader: get_samples_for_ids
+    """
+    assaylayer = AssayLayer(connection)
+    assays = await assaylayer.query(AssayFilter(id=GenericFilter(in_=assay_ids)))
+    # in case it's not ordered
+    assays_map = {a.id: a for a in assays}
+    return [assays_map.get(a) for a in assay_ids]
+
+
 @connected_data_loader_with_params(LoaderKeys.ASSAYS_FOR_SAMPLES, default_factory=list)
 async def load_assays_by_samples(
     connection: Connection, ids, filter: AssayFilter
@@ -501,6 +525,90 @@ async def load_family_participants_for_participants(
     return [fp_map.get(pid, []) for pid in participant_ids]
 
 
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_SAMPLE_IDS)
+async def load_comments_for_sample_ids(
+    sample_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_sample_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.sample, entity_ids=sample_ids
+    )
+    return comments
+
+
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_PARTICIPANT_IDS)
+async def load_comments_for_participant_ids(
+    participant_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_participant_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.participant, entity_ids=participant_ids
+    )
+    return comments
+
+
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_FAMILY_IDS)
+async def load_comments_for_family_ids(
+    family_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_family_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.family, entity_ids=family_ids
+    )
+    return comments
+
+
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_ASSAY_IDS)
+async def load_comments_for_assay_ids(
+    assay_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_assay_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.assay, entity_ids=assay_ids
+    )
+    return comments
+
+
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_PROJECT_IDS)
+async def load_comments_for_project_ids(
+    project_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_project_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.project, entity_ids=project_ids
+    )
+    return comments
+
+
+@connected_data_loader(LoaderKeys.COMMENTS_FOR_SEQUENCING_GROUP_IDS)
+async def load_comments_for_sequencing_group_ids(
+    sequencing_group_ids: list[int], connection: Connection
+) -> list[DiscussionInternal | None]:
+    """
+    DataLoader: load_comments_for_sequencing_group_ids
+    """
+    clayer = CommentLayer(connection)
+    comments = await clayer.get_discussion_for_entity_ids(
+        entity=CommentEntityType.sequencing_group, entity_ids=sequencing_group_ids
+    )
+    return comments
+
+
 class GraphQLContext(TypedDict):
     """Basic dict type for GraphQL context to be passed to resolvers"""
 

api/graphql/mutations/__init__.py

@@ -0,0 +1,56 @@
+import strawberry
+
+from api.graphql.mutations.assay import AssayMutations
+from api.graphql.mutations.comment import CommentMutations
+from api.graphql.mutations.family import FamilyMutations
+from api.graphql.mutations.participant import ParticipantMutations
+from api.graphql.mutations.project import ProjectMutations
+from api.graphql.mutations.sample import SampleMutations
+from api.graphql.mutations.sequencing_group import SequencingGroupMutations
+
+
+@strawberry.type
+class Mutation:
+    """Mutation class"""
+
+    # Comments
+    @strawberry.field
+    def comment(self) -> CommentMutations:
+        """Comment mutations"""
+        return CommentMutations()
+
+    # Samples
+    @strawberry.field
+    def sample(self) -> SampleMutations:
+        """Sample mutations"""
+        return SampleMutations()
+
+    # Assays
+    @strawberry.field
+    def assay(self) -> AssayMutations:
+        """Assay mutations"""
+        return AssayMutations()
+
+    # Participant
+    @strawberry.field
+    def participant(self) -> ParticipantMutations:
+        """Participant mutations"""
+        return ParticipantMutations()
+
+    # Family
+    @strawberry.field
+    def family(self) -> FamilyMutations:
+        """Family mutations"""
+        return FamilyMutations()
+
+    # Project
+    @strawberry.field
+    def project(self) -> ProjectMutations:
+        """Project mutations"""
+        return ProjectMutations()
+
+    # SequencingGroup
+    @strawberry.field
+    def sequencing_group(self) -> SequencingGroupMutations:
+        """Sequencing Group mutations"""
+        return SequencingGroupMutations()