0M_data_extraction_mouse.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "religious-seafood",
   "metadata": {},
   "source": [
    "# Data extraction\n",
    "\n",
    "In this notebook we are going to extract the raw or processed data from all datasets, to later analyze it in different notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "783a7209-0199-4858-802d-7340e696003c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "micro-currency",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import mygene\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import scanpy as sc\n",
    "import scanpy.external as sce\n",
    "from tqdm import tqdm\n",
    "import triku as tk\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcf67bbb-74b8-45cb-ab50-c7cd84e371c8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from cellassign import assign_cats\n",
    "from fb_functions import metadata_assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d89ffd1-4827-447b-8825-12f53628cd78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Palettes for UMAP gene expression\n",
    "\n",
    "magma = [plt.get_cmap('magma')(i) for i in np.linspace(0,1, 80)]\n",
    "magma[0] = (0.88, 0.88, 0.88, 1)\n",
    "magma = mpl.colors.LinearSegmentedColormap.from_list(\"\", magma[:65])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "latin-spoke",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_dir = os.getcwd() + '/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "outdoor-jewelry",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mouse_gencode_dir = \"/media/seth/SETH_DATA/SETH_Alex/Programs/mouse_GRCm38_gencode.v31\"\n",
    "program_dir = \"/media/seth/SETH_DATA/SETH_Alex/Programs/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c0cc635-5279-4091-9a20-989cadc5fae3",
   "metadata": {},
   "source": [
    "## Abbasi et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e16c68f-429f-48b7-9f15-bc721da97752",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "abbasi_2020_dir = data_dir + '/abassi_2020'\n",
    "os.makedirs(abbasi_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4c0113b-ca48-444c-a5f1-27458be8924c",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{abbasi_2020_dir}/GSM2910020\", exist_ok=True)\n",
    "!cd {abbasi_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM2910nnn/GSM2910020/suppl/GSM2910020_sample_4_matrix.mtx.gz -O GSM2910020/matrix.mtx.gz\n",
    "!cd {abbasi_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM2910nnn/GSM2910020/suppl/GSM2910020_sample_4_barcodes.tsv.gz -O GSM2910020/barcodes.tsv.gz\n",
    "!cd {abbasi_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM2910nnn/GSM2910020/suppl/GSM2910020_sample_4_genes.tsv.gz -O GSM2910020/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ad4aa91-3af7-4606-99ef-8e35ba20d444",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_abassi_2020 = sc.read_mtx(f\"{abbasi_2020_dir}/GSM2910020/matrix.mtx.gz\").transpose()\n",
    "adata_abassi_2020.var_names = pd.read_csv(f\"{abbasi_2020_dir}/GSM2910020/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_abassi_2020.obs_names = pd.read_csv(f\"{abbasi_2020_dir}/GSM2910020/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_abassi_2020.var_names_make_unique()\n",
    "metadata_assignment(adata_abassi_2020, 'Abbasi', 2020, 'Ctrl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a83bd284-8730-4ba4-a20b-85a38eb42d0d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_abassi_2020.write_h5ad(f\"{abbasi_2020_dir}/abassi_2020_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11e0c4dc-8500-40f1-83e8-d090555f2a0d",
   "metadata": {},
   "source": [
    "## Buechler et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c6c7cb2-89cb-4cea-b928-5b8b78b1375b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "buechler_2021_dir = data_dir + '/buechler_2021'\n",
    "os.makedirs(buechler_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08dc558c-02c2-4b35-8f95-a599b5cfbc51",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {buechler_2021_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-10315/LIB5436740_SAM24390211_S4_L001_R2_001.fastq.gz\n",
    "!cd {buechler_2021_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-10315/LIB5436740_SAM24390211_S4_L001_R1_001.fastq.gz\n",
    "    \n",
    "!cd {buechler_2021_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-10315/LIB5436740_SAM24390211_S4_L002_R1_001.fastq.gz\n",
    "!cd {buechler_2021_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-10315/LIB5436740_SAM24390211_S4_L002_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97fe2552-eb25-4e34-a961-6597576c0041",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['buechler_2021'], 'technology': ['10xv3'], 'targetnumcells': [5000]})\n",
    "df.to_csv(buechler_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f7a0c4d-0842-479b-9837-340468b8da4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {buechler_2021_dir} && loompy fromfq buechler_2021.loom buechler_2021 {mouse_gencode_dir} metadata.tab \\\n",
    "LIB5436740_SAM24390211_S4_L001_R1_001.fastq.gz LIB5436740_SAM24390211_S4_L001_R2_001.fastq.gz \\\n",
    "LIB5436740_SAM24390211_S4_L002_R1_001.fastq.gz LIB5436740_SAM24390211_S4_L002_R2_001.fastq.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e3258e5-8955-4179-a06d-c0ab23e0bc7d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "buechler_2021 = sc.read(f\"{buechler_2021_dir}/buechler_2021.loom\")\n",
    "metadata_assignment(buechler_2021, 'Buechler', 2021, '0')\n",
    "buechler_2021.write_h5ad(f\"{buechler_2021_dir}/buechler_2021_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71e678ca-3586-46b5-bc70-79cab6fe4f90",
   "metadata": {
    "toc-hr-collapsed": true
   },
   "source": [
    "## Boothby et al. 2021 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4343068b-ea21-4fae-a357-5aae77e32366",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_dir = data_dir + '/boothby_2021'\n",
    "os.makedirs(boothby_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "866fe786-3de8-429b-a8a3-1297d40b5dd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549901/suppl/GSM5549901%5FMs%5FPBS%5Ffiltered%5Fgene%5Fbc%5Fmatrices%5Fh5%2Eh5\n",
    "!cd {boothby_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5549nnn/GSM5549902/suppl/GSM5549902%5FMs%5FDT%5Ffiltered%5Fgene%5Fbc%5Fmatrices%5Fh5%2Eh5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54ba2001-680a-4010-adc7-e13934938391",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_PBS = sc.read_10x_h5(boothby_2021_dir + '/GSM5549901_Ms_PBS_filtered_gene_bc_matrices_h5.h5')\n",
    "boothby_2021_PBS.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_PBS, 'Boothby', 2021, 'PBS')\n",
    "\n",
    "boothby_2021_DT = sc.read_10x_h5(boothby_2021_dir + '/GSM5549902_Ms_DT_filtered_gene_bc_matrices_h5.h5')\n",
    "boothby_2021_DT.var_names_make_unique()\n",
    "metadata_assignment(boothby_2021_DT, 'Boothby', 2021, 'DT')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb91150e-b6cb-43a1-bcd1-7280605fa796",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "boothby_2021_PBS.write_h5ad(boothby_2021_dir + '/boothby_2021_mouse_ctrl_mouse.h5')\n",
    "\n",
    "boothby_2021_DT.write_h5ad(boothby_2021_dir + '/boothby_2021_mouse_DT.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5af74c1-90b4-4150-a005-c450cbf3833d",
   "metadata": {},
   "source": [
    "## Efremova, Mirjana Panglao DB) 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cd5c0bb-bc6d-4085-88db-b4cf26fb14bc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "efremova_2018_dir = data_dir + '/efremova_2018'\n",
    "os.makedirs(efremova_2018_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc8baa75-f4dc-4421-bb55-b465ecc83132",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {efremova_2018_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-7417/3421STDY7639028_S1_L001_R1_001.fastq.gz\n",
    "!cd {efremova_2018_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-7417/3421STDY7639028_S1_L001_R2_001.fastq.gz\n",
    "    \n",
    "!cd {efremova_2018_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-7417/3421STDY7639029_S1_L001_R1_001.fastq.gz\n",
    "!cd {efremova_2018_dir} && wget ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/experiment/MTAB/E-MTAB-7417/3421STDY7639029_S1_L001_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8369b8d1-3648-44b9-9113-693425a6d042",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': ['efremova_2018_S1', 'efremova_2018_S2'], 'technology': ['10xv2']*2, 'targetnumcells': [5000]*2})\n",
    "df.to_csv(efremova_2018_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3aef556a-2dcf-48af-b1d7-ef5cd5484680",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {efremova_2018_dir} && loompy fromfq efremova_2018_S1.loom efremova_2018_S1 {mouse_gencode_dir} metadata.tab \\\n",
    "3421STDY7639028_S1_L001_R1_001.fastq.gz 3421STDY7639028_S1_L001_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "944ce1de-978b-4848-89d3-ae06bc3ad0f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {efremova_2018_dir} && loompy fromfq efremova_2018_S2.loom efremova_2018_S2 {mouse_gencode_dir} metadata.tab \\\n",
    "3421STDY7639029_S1_L001_R1_001.fastq.gz 3421STDY7639029_S1_L001_R2_001.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8090c864-b38c-4905-81bf-8ec3b50139ad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_efremova_2018_S1 = sc.read(f\"{efremova_2018_dir}/efremova_2018_S1.loom\")\n",
    "adata_efremova_2018_S1.var_names_make_unique()\n",
    "metadata_assignment(adata_efremova_2018_S1, 'Efremova', 2018, '0')\n",
    "\n",
    "adata_efremova_2018_S2 = sc.read(f\"{efremova_2018_dir}/efremova_2018_S2.loom\")\n",
    "adata_efremova_2018_S2.var_names_make_unique()\n",
    "metadata_assignment(adata_efremova_2018_S1, 'Efremova', 2018, '1')\n",
    "\n",
    "adata_efremova_2018 = sc.AnnData.concatenate(adata_efremova_2018_S1, adata_efremova_2018_S2, batch_key='Internal sample identifier')\n",
    "adata_efremova_2018.write_h5ad(f\"{efremova_2018_dir}/efremova_2018_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cba7f7c5-5492-4725-b827-90916b87838a",
   "metadata": {},
   "source": [
    "## Haensel et al. 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7564cb8-3e0b-4a07-b18e-941daf927ad5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "haensel_2021_dir = data_dir + '/haensel_2021'\n",
    "os.makedirs(haensel_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71b31a79-9e00-446d-a078-0e224cdac485",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{haensel_dir}/GSM4230076\", exist_ok=True)\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230076/suppl/GSM4230076_Un-Wounded_1_scRNA-Seq.mtx.gz -O GSM4230076/matrix.mtx.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230076/suppl/GSM4230076_barcodes_Un-Wounded_1_scRNA-Seq.tsv.gz  -O GSM4230076/barcodes.tsv.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230076/suppl/GSM4230076_genes_Un-Wounded_1_scRNA-Seq.tsv.gz -O GSM4230076/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8e124e8-6688-4de2-888e-20527d66205b",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{haensel_2021_dir}/GSM4230077\", exist_ok=True)\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230077/suppl/GSM4230077_Un-Wounded_2_scRNA-Seq.mtx.gz -O GSM4230077/matrix.mtx.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230077/suppl/GSM4230077_barcodes_Un-Wounded_2_scRNA-Seq.tsv.gz -O GSM4230077/barcodes.tsv.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230077/suppl/GSM4230077_genes_Un-Wounded_2_scRNA-Seq.tsv.gz -O GSM4230077/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fe8b0f1-9cd9-45c2-bbe9-e240d4801c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{haensel_2021_dir}/GSM4230078\", exist_ok=True)\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230078/suppl/GSM4230078_Wounded_1_scRNA-Seq.mtx.gz -O GSM4230078/matrix.mtx.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230078/suppl/GSM4230078_barcodes_Wounded_1_scRNA-Seq.tsv.gz -O GSM4230078/barcodes.tsv.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230078/suppl/GSM4230078_genes_Wounded_1_scRNA-Seq.tsv.gz -O GSM4230078/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced8e10f-d25e-4505-b47c-4fb99d85f771",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{haensel_2021_dir}/GSM4230079\", exist_ok=True)\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230079/suppl/GSM4230079_Wounded_2_scRNA-Seq.mtx.gz -O GSM4230079/matrix.mtx.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230079/suppl/GSM4230079_barcodes_Wounded_2_scRNA-Seq.tsv.gz -O GSM4230079/barcodes.tsv.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230079/suppl/GSM4230079_genes_Wounded_2_scRNA-Seq.tsv.gz -O GSM4230079/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17047d0b-4583-425d-9699-50d0daf2aa6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{haensel_2021_dir}/GSM4230080\", exist_ok=True)\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230080/suppl/GSM4230080_Wounded_3_scRNA-Seq.mtx.gz -O GSM4230080/matrix.mtx.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230080/suppl/GSM4230080_barcodes_Wounded_3_scRNA-Seq.tsv.gz -O GSM4230080/barcodes.tsv.gz\n",
    "!cd {haensel_2021_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4230nnn/GSM4230080/suppl/GSM4230080_genes_Wounded_3_scRNA-Seq.tsv.gz -O GSM4230080/features.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0bad92f-1ffd-4715-a72f-230449cb5466",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM4230076 = sc.read_mtx(f\"{haensel_2021_dir}/GSM4230076/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4230076.var_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230076/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4230076.obs_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230076/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4230076.var_names_make_unique()\n",
    "metadata_assignment(adata_GSM4230076, 'Haensel', 2021, 'Un1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89fe0f7d-e109-48b4-ba36-05ab5e1c0222",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM4230077 = sc.read_mtx(f\"{haensel_2021_dir}/GSM4230077/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4230077.var_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230077/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4230077.obs_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230077/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4230077.var_names_make_unique()\n",
    "metadata_assignment(adata_GSM4230077, 'Haensel', 2021, 'Un2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a88bd44-b0d4-414c-96d1-1435b9c54604",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_haensel = sc.AnnData.concatenate(adata_GSM4230076, adata_GSM4230077, batch_key='Internal sample identifier', batch_categories=['Un1', 'Un2'])\n",
    "adata_haensel.write_h5ad(f\"{haensel_2021_dir}/haensel_2021_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97933c5c-1cc9-4651-953f-5014aa6a7ff2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM4230078 = sc.read_mtx(f\"{haensel_2021_dir}/GSM4230078/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4230078.var_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230078/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4230078.obs_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230078/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4230078.var_names_make_unique()\n",
    "metadata_assignment(adata_GSM4230078, 'Haensel', 2021, 'Wo1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78c507e9-d212-42c8-bf97-9c93e4df5123",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM4230079 = sc.read_mtx(f\"{haensel_2021_dir}/GSM4230079/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4230079.var_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230079/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4230079.obs_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230079/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4230079.var_names_make_unique()\n",
    "metadata_assignment(adata_GSM4230079, 'Haensel', 2021, 'Wo2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e449c5d-4e2b-4992-ba69-e156b12f84cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM4230080 = sc.read_mtx(f\"{haensel_2021_dir}/GSM4230080/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4230080.var_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230080/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4230080.obs_names = pd.read_csv(f\"{haensel_2021_dir}/GSM4230080/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4230080.var_names_make_unique()\n",
    "metadata_assignment(adata_GSM4230080, 'Haensel', 2021, 'Wo3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5372a930-26b9-4bf1-aac7-a098c4467137",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_haensel_wounded = sc.AnnData.concatenate(adata_GSM4230078, adata_GSM4230079, adata_GSM4230080, batch_key='Internal sample identifier', batch_categories=['Wo1', 'Wo2', 'Wo3'])\n",
    "adata_haensel_wounded.write_h5ad(f\"{haensel_2021_dir}/haensel_2021_wounding_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "884e802c-8bda-4f1f-8c10-81a9da534339",
   "metadata": {},
   "source": [
    "## Joost et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff2f00a1-67a3-4064-a1b9-66f6804c068a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "joost_2020_dir = data_dir + '/joost_2020'\n",
    "os.makedirs(joost_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a4fee03-5577-4ca1-b504-6f0249af49af",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fbarcodes%5F5w%2Etsv%2Egz\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fgenes%5F5w%2Etsv%2Egz\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fmatrix%5F5w%2Emtx%2Egz\n",
    "\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fbarcodes%5F9w%2Etsv%2Egz\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fgenes%5F9w%2Etsv%2Egz\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fmatrix%5F9w%2Emtx%2Egz\n",
    "    \n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/series/GSE129nnn/GSE129218/suppl/GSE129218%5Fassigned%5Fbarcodes%2Etxt%2Egz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "faee4571-bba8-4689-aedf-6eec8246df8f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186888/suppl/GSM4186888%5F10X%5F19%5F067%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186889/suppl/GSM4186889%5F10X%5F19%5F069%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186890/suppl/GSM4186890%5F10X%5F19%5F071%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186891/suppl/GSM4186891%5F10X%5F19%5F068%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186892/suppl/GSM4186892%5F10X%5F19%5F070%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5\n",
    "! cd {joost_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4186nnn/GSM4186893/suppl/GSM4186893%5F10X%5F19%5F072%5Ffiltered%5Ffeature%5Fbc%5Fmatrix%2Eh5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47cdf81b-8060-4456-884c-e9b219521219",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(f\"{joost_2020_dir}/GSE129218_assigned_barcodes.txt.gz\", sep='\\t', header=None).set_index([0])\n",
    "dict_map = {'5wk1': '5w - replicate 1', '5wk2': '5w - replicate 2', '5wk3': '5w - replicate 3', '9wk1': '9w - replicate 1', '9wk2': '9w - replicate 2', '9wk3': '9w - replicate 3'} \n",
    "df[1] = df[1].map(dict_map)\n",
    "df = df[~df.index.duplicated(keep='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04a30887-0ccd-4b25-ace8-d9146ae33f7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_5w = sc.read_mtx(f\"{joost_2020_dir}/GSE129218_matrix_5w.mtx.gz\").transpose()\n",
    "adata_joost_2020_5w.var_names = pd.read_csv(f\"{joost_2020_dir}/GSE129218_genes_5w.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_joost_2020_5w.obs_names = pd.read_csv(f\"{joost_2020_dir}/GSE129218_barcodes_5w.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "adata_joost_2020_5w.obs[''] = pd.read_csv(f\"{joost_2020_dir}/GSE129218_barcodes_5w.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "common_idx = np.intersect1d(adata_joost_2020_5w.obs_names, df.index)\n",
    "\n",
    "adata_joost_2020_5w.var_names_make_unique()\n",
    "adata_joost_2020_5w = adata_joost_2020_5w[common_idx]\n",
    "adata_joost_2020_5w.obs['Internal sample identifier'] = df.loc[common_idx][1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d88461b-e94c-47a6-a8f2-ef8651f43b3f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_5wr1 = adata_joost_2020_5w[adata_joost_2020_5w.obs['Internal sample identifier'] == '5w - replicate 1']\n",
    "metadata_assignment(adata_joost_2020_5wr1, 'Joost', 2020, '5w - replicate 1')\n",
    "adata_joost_2020_5wr2 = adata_joost_2020_5w[adata_joost_2020_5w.obs['Internal sample identifier'] == '5w - replicate 2']\n",
    "metadata_assignment(adata_joost_2020_5wr2, 'Joost', 2020, '5w - replicate 2')\n",
    "adata_joost_2020_5wr3 = adata_joost_2020_5w[adata_joost_2020_5w.obs['Internal sample identifier'] == '5w - replicate 3']\n",
    "metadata_assignment(adata_joost_2020_5wr3, 'Joost', 2020, '5w - replicate 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "963fb826-4001-4c3b-9bee-96794a96b5ac",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_9w = sc.read_mtx(f\"{joost_2020_dir}/GSE129218_matrix_9w.mtx.gz\").transpose()\n",
    "adata_joost_2020_9w.var_names = pd.read_csv(f\"{joost_2020_dir}/GSE129218_genes_9w.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_joost_2020_9w.obs_names = pd.read_csv(f\"{joost_2020_dir}/GSE129218_barcodes_9w.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "adata_joost_2020_9w.obs[''] = pd.read_csv(f\"{joost_2020_dir}/GSE129218_barcodes_9w.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "common_idx = np.intersect1d(adata_joost_2020_9w.obs_names, df.index)\n",
    "\n",
    "adata_joost_2020_9w.var_names_make_unique()\n",
    "adata_joost_2020_9w = adata_joost_2020_9w[common_idx]\n",
    "adata_joost_2020_9w.obs['Internal sample identifier'] = df.loc[common_idx][1].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0124428e-6b8a-4bf1-b6f9-b5f579c09aab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_9wr1 = adata_joost_2020_9w[adata_joost_2020_9w.obs['Internal sample identifier'] == '9w - replicate 1']\n",
    "metadata_assignment(adata_joost_2020_9wr1, 'Joost', 2020, '9w - replicate 1')\n",
    "adata_joost_2020_9wr2 = adata_joost_2020_9w[adata_joost_2020_9w.obs['Internal sample identifier'] == '9w - replicate 2']\n",
    "metadata_assignment(adata_joost_2020_9wr2, 'Joost', 2020, '9w - replicate 2')\n",
    "adata_joost_2020_9wr3 = adata_joost_2020_9w[adata_joost_2020_9w.obs['Internal sample identifier'] == '9w - replicate 3']\n",
    "metadata_assignment(adata_joost_2020_9wr3, 'Joost', 2020, '9w - replicate 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d79592e0-c951-4b14-af8f-db545ded2a2c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_v5wr1 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186888_10X_19_067_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v5wr1.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v5wr1, 'Joost', 2020, 'validation - 5w - replicate 1')\n",
    "\n",
    "adata_joost_2020_v5wr2 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186889_10X_19_069_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v5wr2.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v5wr2, 'Joost', 2020, 'validation - 5w - replicate 2')\n",
    "\n",
    "adata_joost_2020_v5wr3 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186890_10X_19_071_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v5wr3.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v5wr3, 'Joost', 2020, 'validation - 5w - replicate 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f010ad5-5193-4f33-990f-f6d295833a30",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_v9wr1 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186891_10X_19_068_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v9wr1.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v9wr1, 'Joost', 2020, 'validation - 9w - replicate 1')\n",
    "\n",
    "adata_joost_2020_v9wr2 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186892_10X_19_070_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v9wr2.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v9wr2, 'Joost', 2020, 'validation - 9w - replicate 2')\n",
    "\n",
    "adata_joost_2020_v9wr3 = sc.read_10x_h5(f\"{joost_2020_dir}/GSM4186893_10X_19_072_filtered_feature_bc_matrix.h5\")\n",
    "adata_joost_2020_v9wr3.var_names_make_unique()\n",
    "metadata_assignment(adata_joost_2020_v9wr3, 'Joost', 2020, 'validation - 9w - replicate 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a8805c7-6d92-48eb-adeb-b62753636921",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_joost_2020_ctrl_mouse = sc.AnnData.concatenate(adata_joost_2020_5wr1, adata_joost_2020_5wr2, adata_joost_2020_5wr3, adata_joost_2020_v5wr1, adata_joost_2020_v5wr2, adata_joost_2020_v5wr3,\n",
    "                                               adata_joost_2020_9wr1, adata_joost_2020_9wr2, adata_joost_2020_9wr3, adata_joost_2020_v9wr1, adata_joost_2020_v9wr2, adata_joost_2020_v9wr3,)\n",
    "del adata_joost_2020_ctrl_mouse.obs['']  # gives error\n",
    "\n",
    "adata_joost_2020_ctrl_mouse.write_h5ad(f\"{joost_2020_dir}/adata_joost_2020_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea0c2ab9-c543-40b1-b81c-67c0897b5f86",
   "metadata": {},
   "source": [
    "## Leyva-Castillo et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c731664e-3918-4307-99e7-c51ee8b8a3a8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "leyva_castillo_2022_dir = data_dir + '/leyca_castillo_2022'\n",
    "os.makedirs(leyva_castillo_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "828eebd8-f8a5-4fd5-a93c-d31df2cfa6e6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831748\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831748/suppl/GSM5831748%5FEC%2DSAL%2D1%5Fbarcodes%2Etsv%2Egz -O GSM5831748/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831748/suppl/GSM5831748%5FEC%2DSAL%2D1%5Ffeatures%2Etsv%2Egz -O GSM5831748/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831748/suppl/GSM5831748%5FEC%2DSAL%2D1%5Fmatrix%2Emtx%2Egz -O GSM5831748/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7116b85b-b1ea-4ac5-8141-41ee862a5522",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM5831748_SAL1 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831748/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831748_SAL1.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831748/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831748_SAL1.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831748/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831748_SAL1.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d05aa949-4792-4c47-aca5-e2827de4f713",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831749\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831749/suppl/GSM5831749%5FEC%2DSAL%2D2%5Fbarcodes%2Etsv%2Egz -O GSM5831749/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831749/suppl/GSM5831749%5FEC%2DSAL%2D2%5Ffeatures%2Etsv%2Egz -O GSM5831749/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831749/suppl/GSM5831749%5FEC%2DSAL%2D2%5Fmatrix%2Emtx%2Egz -O GSM5831749/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a46acc17-0b33-405c-95dc-8619133f67cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM5831749_SAL2 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831749/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831749_SAL2.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831749/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831749_SAL2.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831749/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831749_SAL2.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12664eed-4301-426f-a728-2cda2d78d48d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831750\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831750/suppl/GSM5831750%5FEC%2DSAL%2D3%5Fbarcodes%2Etsv%2Egz -O GSM5831750/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831750/suppl/GSM5831750%5FEC%2DSAL%2D3%5Ffeatures%2Etsv%2Egz -O GSM5831750/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831750/suppl/GSM5831750%5FEC%2DSAL%2D3%5Fmatrix%2Emtx%2Egz -O GSM5831750/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "696e21a7-17f7-49c6-bc27-0b8467e0c618",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM5831750_SAL3 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831750/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831750_SAL3.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831750/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831750_SAL3.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831750/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831750_SAL3.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de4bf2a-92ca-4697-a01f-37288f67752a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831751\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831751/suppl/GSM5831751%5FEC%2DOVA%2D1%5Fbarcodes%2Etsv%2Egz -O GSM5831751/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831751/suppl/GSM5831751%5FEC%2DOVA%2D1%5Ffeatures%2Etsv%2Egz -O GSM5831751/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831751/suppl/GSM5831751%5FEC%2DOVA%2D1%5Fmatrix%2Emtx%2Egz -O GSM5831751/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ab79bed-87d6-4d08-88eb-5554842a12ee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM5831751_OVA1 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831751/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831751_OVA1.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831751/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831751_OVA1.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831751/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831751_OVA1.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67063c1f-ad7a-4f55-9054-aab31cb81c76",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831752\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831752/suppl/GSM5831752%5FEC%2DOVA%2D2%5Fbarcodes%2Etsv%2Egz -O GSM5831752/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831752/suppl/GSM5831752%5FEC%2DOVA%2D2%5Ffeatures%2Etsv%2Egz -O GSM5831752/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831752/suppl/GSM5831752%5FEC%2DOVA%2D2%5Fmatrix%2Emtx%2Egz -O GSM5831752/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c604989-1731-4ec2-9b03-19bab8f8b69c",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM5831752_OVA2 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831752/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831752_OVA2.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831752/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831752_OVA2.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831752/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831752_OVA2.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b07cb0c1-6cfd-41cb-9c7d-ea1b743f9142",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{leyva_castillo_2022_dir}/GSM5831753\", exist_ok=True)\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831753/suppl/GSM5831753%5FEC%2DOVA%2D3%5Fbarcodes%2Etsv%2Egz -O GSM5831753/barcodes.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831753/suppl/GSM5831753%5FEC%2DOVA%2D3%5Ffeatures%2Etsv%2Egz -O GSM5831753/features.tsv.gz\n",
    "!cd {leyva_castillo_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5831nnn/GSM5831753/suppl/GSM5831753%5FEC%2DOVA%2D3%5Fmatrix%2Emtx%2Egz -O GSM5831753/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c8e175c-5862-428a-aeb9-a311a6333f55",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_GSM5831753_OVA3 = sc.read_mtx(f\"{leyva_castillo_2022_dir}/GSM5831753/matrix.mtx.gz\").transpose()\n",
    "adata_GSM5831753_OVA3.var_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831753/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM5831753_OVA3.obs_names = pd.read_csv(f\"{leyva_castillo_2022_dir}/GSM5831753/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM5831753_OVA3.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3739f72-7ae2-4b9e-8f84-be7d534d53fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "metadata_assignment(adata_GSM5831748_SAL1, 'Leyva-Castillo', 2022, 'EC-SAL-1')\n",
    "metadata_assignment(adata_GSM5831749_SAL2, 'Leyva-Castillo', 2022, 'EC-SAL-2')\n",
    "metadata_assignment(adata_GSM5831750_SAL3, 'Leyva-Castillo', 2022, 'EC-SAL-3')\n",
    "\n",
    "metadata_assignment(adata_GSM5831751_OVA1, 'Leyva-Castillo', 2022, 'EC-OVA-1')\n",
    "metadata_assignment(adata_GSM5831752_OVA2, 'Leyva-Castillo', 2022, 'EC-OVA-2')\n",
    "metadata_assignment(adata_GSM5831753_OVA3, 'Leyva-Castillo', 2022, 'EC-OVA-3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "738b2d51-d626-4954-acf5-50a8606c1d1a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "leyva_castillo_2022_SAL = sc.AnnData.concatenate(adata_GSM5831748_SAL1, adata_GSM5831749_SAL2, adata_GSM5831750_SAL3, \n",
    "                                         batch_key='Internal sample identifier', batch_categories=['EC-SAL-1', 'EC-SAL-2', 'EC-SAL-3'])\n",
    "\n",
    "leyva_castillo_2022_OVA = sc.AnnData.concatenate(adata_GSM5831751_OVA1, adata_GSM5831752_OVA2, adata_GSM5831753_OVA3, \n",
    "                                         batch_key='Internal sample identifier', batch_categories=['EC-OVA-1', 'EC-OVA-2', 'EC-OVA-3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "820565fc-cf72-4d57-b072-7e6b93570e88",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "leyva_castillo_2022_SAL.write_h5ad(f\"{leyva_castillo_2022_dir}/leyva_castillo_2022_SAL.h5\")\n",
    "leyva_castillo_2022_OVA.write_h5ad(f\"{leyva_castillo_2022_dir}/leyva_castillo_2022_OVA.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bc80738-896d-4006-8abd-558af586402b",
   "metadata": {},
   "source": [
    "## Lin et al. 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ca4fdf-2a30-4648-bd1b-e228b74c7be9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "lin_2022_dir = data_dir + '/lin_2022'\n",
    "os.makedirs(lin_2022_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11e62228-cb12-4ff2-b918-4f2e32450047",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{lin_2022_dir}/GSM5266942\", exist_ok=True)\n",
    "!cd {lin_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5266nnn/GSM5266942/suppl/GSM5266942%5FC5%5Fmatrix%2Etsv%2Egz\n",
    "!cd {lin_2022_dir} && gzip -d GSM5266942_C5_matrix.tsv.gz "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a64bdb0-d9d5-48eb-85b8-ff96a40986f8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_lin_2022_ctrl = sc.read(f\"{lin_2022_dir}/GSM5266942_C5_matrix.tsv\").transpose()\n",
    "metadata_assignment(adata_lin_2022_ctrl, 'Lin', 2022, 'Control')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e789ce-cddc-40d5-8b09-0cd24311950b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{lin_2022_dir}/GSM5266943\", exist_ok=True)\n",
    "!cd {lin_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5266nnn/GSM5266943/suppl/GSM5266943%5FUV%5Fmatrix%2Etsv%2Egz\n",
    "!cd {lin_2022_dir} && gzip -d GSM5266943_UV_matrix.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f44a7c4-3024-4f97-a38a-ba61d3392c35",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_lin_2022_UV = sc.read(f\"{lin_2022_dir}/GSM5266943_UV_matrix.tsv\").transpose()\n",
    "metadata_assignment(adata_lin_2022_UV, 'Lin', 2022, 'UV')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9699798-7b3d-4e3f-8eec-1fc5ca2dabb6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "os.makedirs(f\"{lin_2022_dir}/GSM5266944\", exist_ok=True)\n",
    "!cd {lin_2022_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM5266nnn/GSM5266944/suppl/GSM5266944%5FVD%5Fmatrix%2Etsv%2Egz\n",
    "!cd {lin_2022_dir} && gzip -d GSM5266944_VD_matrix.tsv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9995fed-3e69-4adf-b87a-aad83e21b807",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_lin_2022_VD = sc.read(f\"{lin_2022_dir}/GSM5266944_VD_matrix.tsv\").transpose()\n",
    "metadata_assignment(adata_lin_2022_VD, 'Lin', 2022, 'VD')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8bc6dc6-eb2b-45b2-b8cb-f1f49fa6b748",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_lin_2022_ctrl.write_h5ad(f\"{lin_2022_dir}/lin_2022_ctrl.h5\")\n",
    "adata_lin_2022_UV.write_h5ad(f\"{lin_2022_dir}/lin_2022_UV.h5\")\n",
    "adata_lin_2022_VD.write_h5ad(f\"{lin_2022_dir}/lin_2022_VD.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6500b3e6-bac1-484b-b96e-83f1afb089bf",
   "metadata": {},
   "source": [
    "## Ma et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb2ed840-1fea-4e6e-9277-4f23750462d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "ma_dir = data_dir + '/ma_2020'\n",
    "os.makedirs(ma_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9b9e5cb-8770-4b11-93d6-19bb62fb2827",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331840\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331840/suppl/GSM4331840%5FSkin%2DM%2DY%5Fbarcodes%2Etsv%2Egz -O GSM4331840/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331840/suppl/GSM4331840%5FSkin%2DM%2DY%5Fgenes%2Etsv%2Egz -O GSM4331840/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331840/suppl/GSM4331840%5FSkin%2DM%2DY%5Fmatrix%2Emtx%2Egz -O GSM4331840/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b23c4027-7689-4716-ab1a-715c933637f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331840_M_Y = sc.read_mtx(f\"{ma_dir}/GSM4331840/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331840_M_Y.var_names = pd.read_csv(f\"{ma_dir}/GSM4331840/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331840_M_Y.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331840/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331840_M_Y.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1617b3c-ad08-4746-97ee-5132d7981720",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331841\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331841/suppl/GSM4331841%5FSkin%2DM%2DO%5Fbarcodes%2Etsv%2Egz -O GSM4331841/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331841/suppl/GSM4331841%5FSkin%2DM%2DO%5Fgenes%2Etsv%2Egz -O GSM4331841/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331841/suppl/GSM4331841%5FSkin%2DM%2DO%5Fmatrix%2Emtx%2Egz -O GSM4331841/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a67fb92b-46af-4b7a-b442-dca9e9a5fa87",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331841_M_O = sc.read_mtx(f\"{ma_dir}/GSM4331841/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331841_M_O.var_names = pd.read_csv(f\"{ma_dir}/GSM4331841/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331841_M_O.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331841/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331841_M_O.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a657ccf2-2f7c-4b67-89ba-9ad37180a127",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331842\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331842/suppl/GSM4331842%5FSkin%2DM%2DCR%5Fbarcodes%2Etsv%2Egz -O GSM4331842/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331842/suppl/GSM4331842%5FSkin%2DM%2DCR%5Fgenes%2Etsv%2Egz -O GSM4331842/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331842/suppl/GSM4331842%5FSkin%2DM%2DCR%5Fmatrix%2Emtx%2Egz -O GSM4331842/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60bf1cd8-385e-44a2-8091-a40a2a50ee77",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331842_M_CR = sc.read_mtx(f\"{ma_dir}/GSM4331842/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331842_M_CR.var_names = pd.read_csv(f\"{ma_dir}/GSM4331842/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331842_M_CR.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331842/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331842_M_CR.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8e0b224-8802-4220-9444-247f28f9bae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331843\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331843/suppl/GSM4331843%5FSkin%2DF%2DY%5Fbarcodes%2Etsv%2Egz -O GSM4331843/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331843/suppl/GSM4331843%5FSkin%2DF%2DY%5Fgenes%2Etsv%2Egz -O GSM4331843/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331843/suppl/GSM4331843%5FSkin%2DF%2DY%5Fmatrix%2Emtx%2Egz -O GSM4331843/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3de3990e-5083-40de-bf7f-6b763ffa73a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331843_F_Y = sc.read_mtx(f\"{ma_dir}/GSM4331843/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331843_F_Y.var_names = pd.read_csv(f\"{ma_dir}/GSM4331843/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331843_F_Y.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331843/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331843_F_Y.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6f8c774-b3a3-4b24-bfee-e9cbe92aea52",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331844\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331844/suppl/GSM4331844%5FSkin%2DF%2DO%5Fbarcodes%2Etsv%2Egz -O GSM4331844/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331844/suppl/GSM4331844%5FSkin%2DF%2DO%5Fgenes%2Etsv%2Egz -O GSM4331844/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331844/suppl/GSM4331844%5FSkin%2DF%2DO%5Fmatrix%2Emtx%2Egz -O GSM4331844/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0384318f-cbec-4392-88bf-4c776fc516b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331844_F_O = sc.read_mtx(f\"{ma_dir}/GSM4331844/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331844_F_O.var_names = pd.read_csv(f\"{ma_dir}/GSM4331844/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331844_F_O.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331844/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331844_F_O.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "021e99b9-63e9-49b4-8234-90d0989c3055",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f\"{ma_dir}/GSM4331845\", exist_ok=True)\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331845/suppl/GSM4331845%5FSkin%2DF%2DCR%5Fbarcodes%2Etsv%2Egz -O GSM4331845/barcodes.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331845/suppl/GSM4331845%5FSkin%2DF%2DCR%5Fgenes%2Etsv%2Egz -O GSM4331845/features.tsv.gz\n",
    "!cd {ma_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4331nnn/GSM4331845/suppl/GSM4331845%5FSkin%2DF%2DCR%5Fmatrix%2Emtx%2Egz -O GSM4331845/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af4018a8-1b99-43a3-9ecf-edea309f64b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_GSM4331845_F_CR = sc.read_mtx(f\"{ma_dir}/GSM4331845/matrix.mtx.gz\").transpose()\n",
    "adata_GSM4331845_F_CR.var_names = pd.read_csv(f\"{ma_dir}/GSM4331845/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "adata_GSM4331845_F_CR.obs_names = pd.read_csv(f\"{ma_dir}/GSM4331845/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "adata_GSM4331845_F_CR.var_names_make_unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "826bdb5e-de89-4f6b-9914-7531f71e1550",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_ma_Y = sc.AnnData.concatenate(adata_GSM4331840_M_Y, adata_GSM4331843_F_Y)\n",
    "adata_ma_O = sc.AnnData.concatenate(adata_GSM4331841_M_O, adata_GSM4331844_F_O)\n",
    "adata_ma_CR = sc.AnnData.concatenate(adata_GSM4331842_M_CR, adata_GSM4331845_F_CR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95fa7d4b-6568-425a-913d-a445c71e7d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_ma_Y.write_h5ad(f\"{ma_dir}/adata_ma_Y.h5\")\n",
    "adata_ma_O.write_h5ad(f\"{ma_dir}/adata_ma_O.h5\")\n",
    "adata_ma_CR.write_h5ad(f\"{ma_dir}/adata_ma_CR.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9c6e3ae-ed2a-4f50-a15a-9e12e06c7bcc",
   "metadata": {},
   "source": [
    "## Phan et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "467d87c2-e9ca-4ae7-acf4-7b40f2dc1b08",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "phan_2020_dir = data_dir + '/phan_2020'\n",
    "os.makedirs(phan_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52f2b18d-c5fd-4c85-b09d-73ff86a7a973",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {phan_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4647nnn/GSM4647788/suppl/GSM4647788_P21_1.loom.gz\n",
    "!cd {phan_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4647nnn/GSM4647789/suppl/GSM4647789_P21_2.loom.gz\n",
    "!cd {phan_2020_dir} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/GSM4647nnn/GSM4647790/suppl/GSM4647790_P21_3.loom.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d90b95b-654b-4815-928f-fe806c3805bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd {phan_2020_dir} && gunzip *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1a6e148-c384-4fc7-9bc1-fc076b743fee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_phan_2020_1 = sc.read(f\"{phan_2020_dir}/GSM4647788_P21_1.loom\")\n",
    "adata_phan_2020_1.var_names_make_unique()\n",
    "metadata_assignment(adata_phan_2020_1, 'Phan', 2020, 'P21_Un_1')\n",
    "\n",
    "adata_phan_2020_2 = sc.read(f\"{phan_2020_dir}/GSM4647789_P21_2.loom\")\n",
    "adata_phan_2020_2.var_names_make_unique()\n",
    "metadata_assignment(adata_phan_2020_2, 'Phan', 2020, 'P21_Un_2')\n",
    "\n",
    "\n",
    "adata_phan_2020_3 = sc.read(f\"{phan_2020_dir}/GSM4647790_P21_3.loom\")\n",
    "adata_phan_2020_3.var_names_make_unique()\n",
    "metadata_assignment(adata_phan_2020_2, 'Phan', 2020, 'P21_Un_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9fd8eb8-c21a-434e-a9a4-c39c7660519c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_phan_2020 = sc.AnnData.concatenate(adata_phan_2020_1, adata_phan_2020_2, adata_phan_2020_3, \n",
    "                                         batch_key='Internal sample identifier', batch_categories=['P21_Un_1', 'P21_Un_2', 'P21_Un_3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecd27f5d-c2db-4248-8815-ee814193006d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_phan_2020.write_h5ad(f\"{phan_2020_dir}/phan_2020_ctrl_mouse_21d.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc8a64e2-0023-4a5c-aa03-263830ee0a99",
   "metadata": {},
   "source": [
    "## Salzer 2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5068d53f-b517-4c5a-9088-61e4c9ab1aaf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "salzer_2018_dir = data_dir + '/salzer_2018'\n",
    "os.makedirs(salzer_2018_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d97154d6-4dad-4c9a-bb4a-6d083a6e44ff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_salzer = sc.read(salzer_2018_dir + '/Salzer.h5ad')  # This h5ad files was provided externally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73a666bd-7290-418f-a49c-6f2b38d7dce6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_samples = ['P2143_N704', 'P2143_N708', 'P1963_N701', 'P1963_N705', 'P1962_N701', 'P1962_N705', 'P2134_N704', 'P2134_N708', \n",
    "                'P2004_N701', 'P2004_N705', 'P2139_N704', 'P2139_N708', 'P1964_N701', 'P1964_N705', 'P2135_N704', 'P2135_N708', \n",
    "                'P1961_N701', 'P1961_N705', 'P2005_N701', 'P2005_N705', 'P1960_N701', 'P1960_N705', ]\n",
    "\n",
    "list_batches = ['Y1_R2', 'O1_R2', 'Y2_R1', 'O2_R1', 'Y3_R1', 'O3_R1', 'Y4_R2', 'O4_R2', 'Y5_R1', 'O5_R1', 'Y6_R2', 'O6_R2', 'Y7_R1', 'O7_R1', 'Y8_R2', 'O8_R2', 'Y9_R1', \n",
    "                'O9_R1', 'Y10_R1', 'O10_R1', 'Y11_R1', 'O11_R1', ]\n",
    "\n",
    "list_adatas = []\n",
    "\n",
    "for sample, batch in zip(list_samples, list_batches):\n",
    "    adata_salzer_batch = adata_salzer[np.array([True if sample in i else False for i in adata_salzer.obs_names]), :]\n",
    "    adata_salzer_batch = metadata_assignment(adata_salzer_batch, 'Salzer', 2018, batch, do_return=True)\n",
    "    list_adatas.append(adata_salzer_batch)\n",
    "\n",
    "adata_salzer = sc.AnnData.concatenate(*list_adatas, batch_key='Internal sample identifier', batch_categories=list_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfd9c3f1-7e9e-4861-816a-07076929d466",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_salzer.write_h5ad(salzer_2018_dir + '/salzer_2018_young_old_mouse.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6d63a9f-ab06-45fb-8a9e-7a9fddd4a5ce",
   "metadata": {},
   "source": [
    "## Shin et al. 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4de13d46-9c9b-4741-837d-d5bc57176ed7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shin_2020 = data_dir + '/shin_2020'\n",
    "os.makedirs(shin_2020, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcfc2455-530e-4718-9353-a75ed25e9a97",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for geo, name, sample in zip(['GSM' + str(i) for i in [3177991, 3177992, 4155928, 4155929]],['1', '2', '3%5FWisoo%5FYoung%5FRev%5FCount', '4%5FWisoo_Aged%5FRev%5FCount'], ['2mo1', '18mo', '2mo2', '12mo']):\n",
    "    os.makedirs(f\"{shin_2020}/{geo}\", exist_ok=True)\n",
    "    !cd {shin_2020} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{geo[:7]}nnn/{geo}/suppl/{geo}%5Fsample%5F{name}%5Fbarcodes%2Etsv%2Egz -O {geo}/barcodes.tsv.gz\n",
    "    if name in ['1', '2']:\n",
    "        !cd {shin_2020} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{geo[:7]}nnn/{geo}/suppl/{geo}%5Fsample%5F{name}%5Fgenes%2Etsv%2Egz -O {geo}/features.tsv.gz\n",
    "    else:\n",
    "        !cd {shin_2020} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{geo[:7]}nnn/{geo}/suppl/{geo}%5Fsample%5F{name}%5Ffeatures%2Etsv%2Egz -O {geo}/features.tsv.gz\n",
    "    !cd {shin_2020} && wget https://ftp.ncbi.nlm.nih.gov/geo/samples/{geo[:7]}nnn/{geo}/suppl/{geo}%5Fsample%5F{name}%5Fmatrix%2Emtx%2Egz -O {geo}/matrix.mtx.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2606a156-8381-46f2-b130-ffd9d14308a8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "list_adatas = []\n",
    "for geo, sample in zip(['GSM' + str(i) for i in [3177991, 3177992, 4155928, 4155929]], ['2mo1', '18mo', '2mo2', '12mo']):\n",
    "    adata = sc.read_mtx(f\"{shin_2020}/{geo}/matrix.mtx.gz\").transpose()\n",
    "    adata.var_names = pd.read_csv(f\"{shin_2020}/{geo}/features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "    adata.obs_names = pd.read_csv(f\"{shin_2020}/{geo}/barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "    adata.var_names_make_unique()\n",
    "    metadata_assignment(adata, 'Shin', 2020, sample)\n",
    "    list_adatas.append(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65c42995-1ba1-4a8b-affd-2dfba876689b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_shin_2020 = sc.AnnData.concatenate(*list_adatas, batch_key='Internal sample identifier', \n",
    "                                        batch_categories=['2mo1', '18mo', '2mo2', '12mo'])\n",
    "adata_shin_2020.write_h5ad(f\"{shin_2020}/shin_2020_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06671f20-63cf-430a-a09f-1169e705b229",
   "metadata": {},
   "source": [
    "## Shook 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2123f5a-6dac-410d-95e7-9b3e5d47fb26",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shook_2020_dir = data_dir + '/shook_2020'\n",
    "os.makedirs(shook_2020_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fce0098-28a8-4b8d-a234-6d7aa82cba61",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_names = {\n",
    "              'SRR10480641': 'Non_Wounded_S1', \n",
    "              'SRR10480643': 'Non_Wounded_S2',\n",
    "              'SRR10480644': 'Non_Wounded_S3', \n",
    "              'SRR10480645': 'Non_Wounded_S4', \n",
    "              'SRR10480646': 'Non_Wounded_S5', \n",
    "              'SRR10480636': 'Wounded_S1', \n",
    "              'SRR10480637': 'Wounded_S2', \n",
    "              'SRR10480638': 'Wounded_S3', \n",
    "              'SRR10480639': 'Wounded_S4',\n",
    "              'SRR10480640': 'Wounded_S5',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2973aa36-3bfc-413f-ad36-1af0497e67ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'name': list(dict_names.values()), 'technology': ['10xv3']*len(dict_names), 'targetnumcells': [5000]*len(dict_names)})\n",
    "df.to_csv(shook_2020_dir + '/metadata.tab', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e0620d0-a004-4382-9c2b-9371cf67d208",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR, name in dict_names.items():\n",
    "    !cd {shook_2020_dir} && parallel-fastq-dump -s {SRR} --gzip --split-files -t 8 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7de1fc8-6f80-467e-94e6-9b37b2f09d54",
   "metadata": {},
   "outputs": [],
   "source": [
    "for SRR, name in dict_names.items():\n",
    "    !cd {shook_2020_dir} && loompy fromfq {name}.loom {name} {mouse_gencode_dir} metadata.tab {shook_dir}/{SRR}_1.fastq.gz {shook_dir}/{SRR}_2.fastq.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8bdd499-9453-49d8-8d5e-2166788f2b88",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_Non_Wounded_S1 = sc.read(f\"{shook_2020_dir}/Non_Wounded_S1.loom\")\n",
    "adata_Non_Wounded_S1.var_names_make_unique()\n",
    "metadata_assignment(adata_Non_Wounded_S1, 'Shook', 2020, 'Ctrl_S1')\n",
    "\n",
    "adata_Non_Wounded_S2 = sc.read(f\"{shook_2020_dir}/Non_Wounded_S3.loom\")\n",
    "adata_Non_Wounded_S2.var_names_make_unique()\n",
    "metadata_assignment(adata_Non_Wounded_S2, 'Shook', 2020, 'Ctrl_S2')\n",
    "\n",
    "adata_Non_Wounded_S3 = sc.read(f\"{shook_2020_dir}/Non_Wounded_S3.loom\")\n",
    "adata_Non_Wounded_S3.var_names_make_unique()\n",
    "metadata_assignment(adata_Non_Wounded_S3, 'Shook', 2020, 'Ctrl_S3')\n",
    "\n",
    "adata_Non_Wounded_S4 = sc.read(f\"{shook_2020_dir}/Non_Wounded_S4.loom\")\n",
    "adata_Non_Wounded_S4.var_names_make_unique()\n",
    "metadata_assignment(adata_Non_Wounded_S4, 'Shook', 2020, 'Ctrl_S4')\n",
    "\n",
    "adata_Non_Wounded_S5 = sc.read(f\"{shook_2020_dir}/Non_Wounded_S5.loom\")\n",
    "adata_Non_Wounded_S5.var_names_make_unique()\n",
    "metadata_assignment(adata_Non_Wounded_S5, 'Shook', 2020, 'Ctrl_S5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab9abeb3-ec49-43b2-a688-328c6c2b3f7f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "adata_shook_NW = sc.AnnData.concatenate(adata_Non_Wounded_S1, adata_Non_Wounded_S2, adata_Non_Wounded_S3, \n",
    "                                       adata_Non_Wounded_S4, adata_Non_Wounded_S5, batch_key='Internal sample identifier', \n",
    "                                        batch_categories=['Ctrl_S1', 'Ctrl_S2', 'Ctrl_S3', 'Ctrl_S4', 'Ctrl_S5'])\n",
    "adata_shook_NW.write_h5ad(f\"{shook_2020_dir}/shook_2020_ctrl_mouse.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62fae7d7-28f3-45c8-a98c-161b5a15f7c4",
   "metadata": {},
   "source": [
    "## Vorstandlechner 2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76ce0124-2cdc-4e99-a739-6644343bed8c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vorstandlechner_2021_dir = data_dir + '/Vorstandlechner_2021'\n",
    "os.makedirs(vorstandlechner_2021_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a927cf94-f63a-4575-9f53-d4da0bcf268c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data was provided by personal request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52b2413b-e084-4442-9df3-9bd3d1710e64",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vorstandlechner_2021_healthy_1 = sc.read_mtx(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh1_matrix.mtx.gz\").transpose()\n",
    "vorstandlechner_2021_healthy_1.var_names = pd.read_csv(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh1_features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "vorstandlechner_2021_healthy_1.obs_names = pd.read_csv(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh1_barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "vorstandlechner_2021_healthy_1.var_names_make_unique()\n",
    "metadata_assignment(vorstandlechner_2021_healthy_1, 'Vorstandlechner', 2021, 'mouse_healthy_1')\n",
    "\n",
    "\n",
    "vorstandlechner_2021_healthy_2 = sc.read_mtx(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh2_matrix.mtx.gz\").transpose()\n",
    "vorstandlechner_2021_healthy_2.var_names = pd.read_csv(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh2_features.tsv.gz\", sep='\\t', header=None)[1].values\n",
    "vorstandlechner_2021_healthy_2.obs_names = pd.read_csv(f\"{vorstandlechner_2021_dir}/mouse_healthy/nh2_barcodes.tsv.gz\", sep='\\t', header=None)[0].values\n",
    "\n",
    "vorstandlechner_2021_healthy_2.var_names_make_unique()\n",
    "metadata_assignment(vorstandlechner_2021_healthy_2, 'Vorstandlechner', 2021, 'mouse_healthy_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a907e5e2-792b-4188-9d80-08c962bb67c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vorstandlechner_2021_ctrl_mouse = sc.AnnData.concatenate(vorstandlechner_2021_healthy_1, vorstandlechner_2021_healthy_2, batch_key='Internal sample identifier', \n",
    "                                        batch_categories=['mouse_healthy_1', 'mouse_healthy_2'])\n",
    "vorstandlechner_2021_ctrl_mouse.write_h5ad(f\"{vorstandlechner_2021_dir}/vorstandlechner_2021_ctrl_mouse.h5\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:alex-base] *",
   "language": "python",
   "name": "conda-env-alex-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}