broadinstitute · rileyhgrant · Aug 29, 2024 · Aug 5, 2024 · Aug 27, 2024 · Aug 28, 2024
diff --git a/.github/workflows/data-pipeline-ci.yml b/.github/workflows/data-pipeline-ci.yml
@@ -34,6 +34,6 @@ jobs:
       - name: Check formatting
         run: black --check data-pipeline/src/data_pipeline
       - name: Run Ruff
-        run: ruff data-pipeline/src/data_pipeline
+        run: ruff check data-pipeline/src/data_pipeline
       - name: Run Pyright
         run: pyright --project data-pipeline
diff --git a/browser/help/helpPageTableOfContents.ts b/browser/help/helpPageTableOfContents.ts
@@ -19,6 +19,7 @@ const helpPageTableOfContents: { topics: string[]; faq: FaqTopic[] } = {
     'variant-cooccurrence',
     'hgdp-1kg-annotations',
     'v4-hts',
+    'v4-browser-hts',
     'exome-capture-tech',
     'combined-freq-stats',
     'allele-count-zero',

diff --git a/browser/help/topics/v4-browser-hts.md b/browser/help/topics/v4-browser-hts.md
diff --git a/browser/help/topics/v4-hts.md b/browser/help/topics/v4-hts.md
diff --git a/browser/src/DownloadsPage/GnomadV2Downloads.tsx b/browser/src/DownloadsPage/GnomadV2Downloads.tsx
@@ -15,6 +15,8 @@ import {
   StyledParagraph,
 } from './downloadsPageStyles'
 
+import Link from '../Link'
+
 const exomeChromosomeVcfs = [
   { chrom: '1', size: '5.77 GiB', md5: '9817acdf1d9600efb3355e4cb4b7ee1f' },
   { chrom: '2', size: '4.2 GiB', md5: 'c2822d7a914d06ede28d1078970336ac' },
@@ -284,6 +286,30 @@ const GnomadV2Downloads = () => {
         </ColumnsWrapper>
       </DownloadsSection>
 
+      <DownloadsSection>
+        <SectionTitle id="v2-browser-tables">Browser Tables</SectionTitle>
+
+        <p>
+          For more information about these files, see our{' '}
+          <Link to="https://gnomad.broadinstitute.org/new/2024-08-release-gnomad-browser-tables">
+            changelog entry
+          </Link>{' '}
+          on the browser tables, and the <Link to="/help/v4-browser-hts">help text</Link>.
+        </p>
+
+        <h3>Browser gene models</h3>
+        <FileList>
+          {/* @ts-expect-error TS(2745) FIXME: This JSX tag's 'children' prop expects type 'never... Remove this comment to see the full error message */}
+          <ListItem>
+            <GetUrlButtons
+              label="Browser GRCh37 gene models Hail Table"
+              path="/resources/grch37/genes/gnomad.genes.GRCh37.GENCODEv19.ht"
+              includeAzure={false}
+            />
+          </ListItem>
+        </FileList>
+      </DownloadsSection>
+
       <DownloadsSection>
         <SectionTitle id="v2-coverage">Coverage</SectionTitle>
         <FileList>

diff --git a/browser/src/DownloadsPage/GnomadV4Downloads.tsx b/browser/src/DownloadsPage/GnomadV4Downloads.tsx
@@ -265,6 +265,42 @@ const GnomadV4Downloads = () => {
         </ColumnsWrapper>
       </DownloadsSection>
 
+      <DownloadsSection>
+        <SectionTitle id="v4-browser-tables">Browser Tables</SectionTitle>
+
+        <p>
+          For more information about these files, see our{' '}
+          <Link to="https://gnomad.broadinstitute.org/new/2024-08-release-gnomad-browser-tables">
+            changelog entry
+          </Link>{' '}
+          on the browser tables, and the <Link to="/help/v4-browser-hts">help text</Link>.
+        </p>
+
+        <h3>Browser variants</h3>
+        <FileList>
+          {/* @ts-expect-error TS(2745) FIXME: This JSX tag's 'children' prop expects type 'never... Remove this comment to see the full error message */}
+          <ListItem>
+            <GetUrlButtons
+              label="Browser variants Hail Table"
+              path="/release/4.1/ht/browser/gnomad.browser.v4.1.sites.ht"
+              includeAzure={false}
+            />
+          </ListItem>
+        </FileList>
+
+        <h3>Browser gene models</h3>
+        <FileList>
+          {/* @ts-expect-error TS(2745) FIXME: This JSX tag's 'children' prop expects type 'never... Remove this comment to see the full error message */}
+          <ListItem>
+            <GetUrlButtons
+              label="Browser GRCh38 gene models Hail Table"
+              path="/resources/grch38/genes/gnomad.genes.GRCh38.GENCODEv39.ht"
+              includeAzure={false}
+            />
+          </ListItem>
+        </FileList>
+      </DownloadsSection>
+
       <DownloadsSection>
         <SectionTitle id="v4-coverage">Coverage</SectionTitle>
         <FileList>

diff --git a/browser/src/DownloadsPage/__snapshots__/DownloadsPage.spec.tsx.snap b/browser/src/DownloadsPage/__snapshots__/DownloadsPage.spec.tsx.snap
@@ -7310,6 +7310,122 @@ exports[`Downloads Page has no unexpected changes 1`] = `
         </div>
       </div>
     </section>
+    <section
+      className="c15"
+    >
+      <span
+        className="c6"
+      >
+        <h2
+          className="c16"
+        >
+          <a
+            aria-hidden="true"
+            className="c8 c9"
+            href="#v4-browser-tables"
+            id="v4-browser-tables"
+          >
+            <img
+              alt=""
+              aria-hidden="true"
+              height={12}
+              src="test-file-stub"
+              width={12}
+            />
+          </a>
+          Browser Tables
+        </h2>
+      </span>
+      <p>
+        For more information about these files, see our
+
+        <a
+          className="-Link c13"
+          href="https://gnomad.broadinstitute.org/new/2024-08-release-gnomad-browser-tables"
+          onClick={[Function]}
+        >
+          changelog entry
+        </a>
+
+        on the browser tables, and the 
+        <a
+          className="-Link c13"
+          href="/help/v4-browser-hts"
+          onClick={[Function]}
+        >
+          help text
+        </a>
+        .
+      </p>
+      <h3>
+        Browser variants
+      </h3>
+      <ul
+        className="c19"
+      >
+        <li
+          className="c20"
+        >
+          <span>
+            Browser variants Hail Table
+          </span>
+          <br />
+          Show URL for
+
+          <button
+            aria-label="Show Google URL for Browser variants Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Google
+          </button>
+           / 
+          <button
+            aria-label="Show Amazon URL for Browser variants Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Amazon
+          </button>
+        </li>
+      </ul>
+      <h3>
+        Browser gene models
+      </h3>
+      <ul
+        className="c19"
+      >
+        <li
+          className="c20"
+        >
+          <span>
+            Browser GRCh38 gene models Hail Table
+          </span>
+          <br />
+          Show URL for
+
+          <button
+            aria-label="Show Google URL for Browser GRCh38 gene models Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Google
+          </button>
+           / 
+          <button
+            aria-label="Show Amazon URL for Browser GRCh38 gene models Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Amazon
+          </button>
+        </li>
+      </ul>
+    </section>
     <section
       className="c15"
     >
@@ -24426,6 +24542,88 @@ exports[`Downloads Page has no unexpected changes 1`] = `
         </div>
       </div>
     </section>
+    <section
+      className="c15"
+    >
+      <span
+        className="c6"
+      >
+        <h2
+          className="c16"
+        >
+          <a
+            aria-hidden="true"
+            className="c8 c9"
+            href="#v2-browser-tables"
+            id="v2-browser-tables"
+          >
+            <img
+              alt=""
+              aria-hidden="true"
+              height={12}
+              src="test-file-stub"
+              width={12}
+            />
+          </a>
+          Browser Tables
+        </h2>
+      </span>
+      <p>
+        For more information about these files, see our
+
+        <a
+          className="-Link c13"
+          href="https://gnomad.broadinstitute.org/new/2024-08-release-gnomad-browser-tables"
+          onClick={[Function]}
+        >
+          changelog entry
+        </a>
+
+        on the browser tables, and the 
+        <a
+          className="-Link c13"
+          href="/help/v4-browser-hts"
+          onClick={[Function]}
+        >
+          help text
+        </a>
+        .
+      </p>
+      <h3>
+        Browser gene models
+      </h3>
+      <ul
+        className="c19"
+      >
+        <li
+          className="c20"
+        >
+          <span>
+            Browser GRCh37 gene models Hail Table
+          </span>
+          <br />
+          Show URL for
+
+          <button
+            aria-label="Show Google URL for Browser GRCh37 gene models Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Google
+          </button>
+           / 
+          <button
+            aria-label="Show Amazon URL for Browser GRCh37 gene models Hail Table"
+            className="c21"
+            onClick={[Function]}
+            type="button"
+          >
+            Amazon
+          </button>
+        </li>
+      </ul>
+    </section>
     <section
       className="c15"
     >

diff --git a/browser/src/help/__snapshots__/HelpPage.spec.tsx.snap b/browser/src/help/__snapshots__/HelpPage.spec.tsx.snap
@@ -760,6 +760,15 @@ exports[`Help Page has no unexpected changes 1`] = `
           v4-hts
         </a>
       </li>
+      <li>
+        <a
+          className="-Link c3"
+          href="/help/v4-browser-hts"
+          onClick={[Function]}
+        >
+          v4-browser-hts
+        </a>
+      </li>
       <li>
         <a
           className="-Link c3"
@@ -1067,6 +1076,17 @@ exports[`Help Page has no unexpected changes 1`] = `
               v4-hts
             </a>
           </li>
+          <li
+            className="c10"
+          >
+            <a
+              className="-Link c3"
+              href="/help/v4-browser-hts"
+              onClick={[Function]}
+            >
+              v4-browser-hts
+            </a>
+          </li>
           <li
             className="c10"
           >

diff --git a/data-pipeline/src/data_pipeline/data_types/coverage.py b/data-pipeline/src/data_pipeline/data_types/coverage.py
@@ -13,7 +13,7 @@ def prepare_coverage(coverage_path: str, filter_intervals: Optional[List[str]] =
 
     # Specific to v4, the data is contained in a coverage_stats field and we take the first element
     if "coverage_stats" in coverage_fields:
-        if type(coverage.coverage_stats) == hl.expr.expressions.typed_expressions.ArrayStructExpression:
+        if type(coverage.coverage_stats) is hl.expr.expressions.typed_expressions.ArrayStructExpression:
             coverage = coverage.annotate(**coverage.coverage_stats[0])
 
     # Median field name is different in v3.0.1 vs v2

diff --git a/data-pipeline/src/data_pipeline/data_types/gene.py b/data-pipeline/src/data_pipeline/data_types/gene.py
@@ -226,6 +226,16 @@ def import_hgnc(path):
     return ds
 
 
+def prepare_gene_table_for_release(genes_path, keep_mane_version_global_annotation):
+    ds = hl.read_table(genes_path)
+    if keep_mane_version_global_annotation:
+        globals_dict = ds.index_globals()
+        ds = ds.select_globals(mane_select_version=globals_dict["annotations"]["mane_select_transcript"]["version"])
+    else:
+        ds = ds.select_globals()
+    return ds
+
+
 def prepare_genes(gencode_path, hgnc_path, reference_genome):
     genes = import_gencode(gencode_path, reference_genome)
 

diff --git a/...ta_pipeline/data_types/variant/transcript_consequence/annotate_transcript_consequences.py b/...ta_pipeline/data_types/variant/transcript_consequence/annotate_transcript_consequences.py
@@ -85,7 +85,7 @@ def annotate_transcript_consequences(variants_path, transcripts_path, mane_trans
 
     if mane_transcripts_path:
         mane_transcripts = hl.read_table(mane_transcripts_path)
-        mane_transcripts_version = hl.eval(mane_transcripts.globals.version)
+        mane_select_transcripts_version = hl.eval(mane_transcripts.globals.version)
 
         mane_transcripts = hl.dict([(row.gene_id, row.drop("gene_id")) for row in mane_transcripts.collect()])
 
@@ -138,7 +138,7 @@ def annotate_transcript_consequences(variants_path, transcripts_path, mane_trans
         )
 
         ds = ds.annotate(transcript_consequences=transcript_consequences).drop("vep")
-        ds = ds.annotate_globals(mane_transcripts_version=mane_transcripts_version)
+        ds = ds.annotate_globals(mane_select_version=mane_select_transcripts_version)
 
     else:
         transcript_consequences = hl.sorted(

diff --git a/data-pipeline/src/data_pipeline/datasets/gnomad_v4/gnomad_v4_constraint.py b/data-pipeline/src/data_pipeline/datasets/gnomad_v4/gnomad_v4_constraint.py
@@ -43,3 +43,9 @@ def prepare_gnomad_v4_constraint(path):
     ds = ds.key_by("transcript_id")
 
     return ds
+
+
+def remove_gnomad_v4_constraint(genes_path):
+    ds = hl.read_table(genes_path)
+    ds = ds.drop("gnomad_constraint")
+    return ds