notebooks/generate_dataset_variants.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "#\n",
    "# The copyright of this file belongs to Feedzai. The file cannot be\n",
    "# reproduced in whole or in part, stored in a retrieval system,\n",
    "# transmitted in any form, or by any means electronic, mechanical,\n",
    "# photocopying, or otherwise, without the prior permission of the owner.\n",
    "#\n",
    "# (c) 2022 Feedzai, Strictly Confidential"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np   # Seed generation\n",
    "import pandas as pd  # Matrix operations"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b512d99",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Reading the 2.5M sample:\n",
    "large_sample_path = \"<2.5M_sample_path>\"\n",
    "large_sample_df = pd.read_parquet(large_sample_path)\n",
    "\n",
    "# Reading the original (with same preprocessed features) dataset:\n",
    "original_sample_path = \"<original_sample_path>\"\n",
    "original_sample_df = pd.read_parquet(original_sample_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "724feace",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Obtain month frequency and fraud prevalence per month (on original data).\n",
    "month_frequency =  original_sample_df[\"month\"].value_counts(normalize=True).to_dict()\n",
    "month_fraud_prev = original_sample_df.groupby(\"month\")[\"fraud_bool\"].mean().to_dict()\n",
    "# We cast to dict in order to facilitate the next operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51be6b2c",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Calculating the expected number of positive and negative instances,\n",
    "# per month, given the observed month frequency and prevalence.\n",
    "\n",
    "sample_size = 1e6\n",
    "\n",
    "expected_positives = {}\n",
    "expected_negatives = {}\n",
    "\n",
    "for month in month_fraud_prev.keys():\n",
    "    expected_positives[month] = round(sample_size * month_frequency[month] * month_fraud_prev[month], 0)\n",
    "    expected_negatives[month] = round(sample_size * month_frequency[month] * (1-month_fraud_prev[month]), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e62035e5",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Sampling the \"Base\" dataset: Same month frequency and fraud rate per month.\n",
    "base_dfs = []\n",
    "\n",
    "SEED = 42\n",
    "\n",
    "num_months = len(large_sample_df[\"month\"].unique())\n",
    "seed_possible_values = list(range(1_000_000))\n",
    "seed_list = np.random.choice(seed_possible_values, size=num_months, replace=False)\n",
    "\n",
    "for month, seed in zip(large_sample_df[\"month\"].unique(), seed_list):\n",
    "    positive_pool = large_sample_df[(large_sample_df[\"month\"]==month) & (large_sample_df[\"fraud_bool\"]==1)]\n",
    "    negative_pool = large_sample_df[(large_sample_df[\"month\"]==month) & (large_sample_df[\"fraud_bool\"]==0)]\n",
    "    \n",
    "    positive_sample = positive_pool.sample(expected_positives[month], random_state=seed)\n",
    "    negative_sample = negative_pool.sample(expected_negatives[month], random_state=seed+SEED)\n",
    "    \n",
    "    final_dfs.extend([positive_sample, negative_sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e980b2b1",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Concatenate the filtered samples to obtain the final dataset.\n",
    "base_df = pd.concat(base_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b23c18ab",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Now generating the biased samples.\n",
    "# We will start by defining the protected groups.\n",
    "large_sample_df[\"group\"] = (large_sample_df[\"customer_age\"] > 50).map({True:\"Minority\", False: \"Majority\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ad941c9",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Helper method to define the joint probability of each combination of\n",
    "# group and label.\n",
    "\n",
    "def calculate_probabilities(\n",
    "    original_prevalence: float,\n",
    "    prev_ratio: float,\n",
    "    maj_pct: float,\n",
    "):\n",
    "    # Probability notation (p_maj = P(A=maj))\n",
    "    p_maj = maj_pct\n",
    "    p_min = 1 - p_maj\n",
    "\n",
    "    # Calculate prevalence for each class\n",
    "    prev_min = original_prevalence / (prev_ratio * p_maj + (1 - p_maj))\n",
    "    prev_maj = prev_ratio * prev_min\n",
    "\n",
    "    # Calculate joint and conditional probabilities of majority group\n",
    "    p_maj_and_pos = prev_maj * p_maj\n",
    "    p_maj_giv_pos: float = p_maj_and_pos / original_prevalence\n",
    "    p_maj_and_neg = p_maj - p_maj_and_pos\n",
    "    p_maj_giv_neg: float = p_maj_and_neg / (1 - original_prevalence)\n",
    "\n",
    "    # Calculate joint and conditional probabilities of minority group\n",
    "    p_min_and_pos = prev_min * p_min\n",
    "    p_min_giv_pos: float = p_min_and_pos / original_prevalence\n",
    "    p_min_and_neg = p_min - p_min_and_pos\n",
    "    p_min_giv_neg: float = p_min_and_neg / (1 - original_prevalence)\n",
    "\n",
    "    return p_min_and_pos, p_maj_and_pos, p_min_and_neg, p_maj_and_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd064658",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Helper method to obtain a dataframe from given group, month and label.\n",
    "def get_filtered_df(large_sample_df, group, month, label):\n",
    "    return large_sample_df[\n",
    "        (large_sample_df[\"month\"]==month) & \n",
    "        (big_sample_df[\"group\"]==group) & \n",
    "        (big_sample_df[\"fraud_bool\"]==label)]\n",
    "\n",
    "\n",
    "# Method to generate a biased sample controling group size or prevalence (fraud rate)\n",
    "def group_prevalence_disparity(large_sample_df, original_sample_df, majority_size, fraud_rate_disparity):\n",
    "    seed_list = np.random.choice(seed_possible_values, size=num_months, replace=False)\n",
    "\n",
    "    bias_dfs = []\n",
    "    \n",
    "    # Allow for different majority sizes/fraud rates depending on the month of data.\n",
    "    # This replicates a value if only one is passed.\n",
    "    if isinstance(majority_size, float):\n",
    "        majority_size=[majority_size]*original_sample_df[\"month\"].unique().shape[0] \n",
    "    if isinstance(fraud_rate_disparity, (int, float)):\n",
    "        fraud_rate_disparity=[fraud_rate_disparity]*original_sample_df[\"month\"].unique().shape[0]\n",
    "\n",
    "    for month, seed, maj_size, fr_disp in zip(large_sample_df[\"month\"].unique(), seed_list, majority_size, fraud_rate_disparity):\n",
    "        month_prevalence = original_sample_df[original_sample_df[\"month\"]==month][\"fraud_bool\"].mean()\n",
    "        (\n",
    "            p_min_and_pos, \n",
    "            p_maj_and_pos, \n",
    "            p_min_and_neg, \n",
    "            p_maj_and_neg,\n",
    "        ) = calculate_probabilities(month_prevalence, 1/fr_disp, maj_size)\n",
    "\n",
    "        month_size = original_sample_df[\"month\"].value_counts(normalize=True)[month]*sample_size\n",
    "\n",
    "        # Calculate the needed amount of each combination of group/label to satisfy the disparities in month.\n",
    "        n_minority_positive = round(month_size*p_min_and_pos, 0)\n",
    "        n_minority_negative = round(month_size*p_min_and_neg, 0)\n",
    "        n_majority_positive = round(month_size*p_maj_and_pos, 0)\n",
    "        n_majority_negative = round(month_size*p_maj_and_neg, 0)\n",
    "        \n",
    "        # Sample the large sample with expected values.\n",
    "        bias_dfs.extend(\n",
    "        [\n",
    "            get_filtered_df(large_sample_df, \"Minority\", month, 1).sample(int(minority_positive), random_state=seed),\n",
    "            get_filtered_df(large_sample_df, \"Minority\", month, 0).sample(int(minority_negative), random_state=seed+SEED),\n",
    "            get_filtered_df(large_sample_df, \"Majority\", month, 1).sample(int(majority_positive), random_state=seed+2*SEED),\n",
    "            get_filtered_df(large_sample_df, \"Majority\", month, 0).sample(int(majority_negative), random_state=seed+3*SEED), \n",
    "        ]\n",
    "        )\n",
    "\n",
    "    return pd.concat(final_bias_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce5f6ad4",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Params for the generated sample\n",
    "majority_size = 0.9      # Relative size of the majority group\n",
    "fraud_rate_disparity = 1 # fraud prevalence in minority / fraud prevalence in majority\n",
    "\n",
    "# For Type I we want to test group size disparity. \n",
    "# Majority will have 90% of instances, Minority 10% of instances."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07204aad",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "typeI_df = group_prevalence_disparity(large_sample_df, original_sample_df, majority_size, fraud_rate_disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "925c459d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Params for the generated sample\n",
    "majority_size = 0.5      # Relative size of the majority group\n",
    "fraud_rate_disparity = 5 # fraud prevalence in minority / fraud prevalence in majority\n",
    "\n",
    "# For Type II we want to test prevalence disparity. \n",
    "# Minority will have 5 times more fraud, when compared to Majority."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a41a8d8",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "typeII_df = group_prevalence_disparity(large_sample_df, original_sample_df, majority_size, fraud_rate_disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2c8ae90",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Helper class that wraps the logic of the multivariate normal distribution mean calculation.\n",
    "from mvn import TypeIIIBiasSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bec43b5a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Params for the generated sample\n",
    "majority_size = 0.5      # Relative size of the majority group\n",
    "fraud_rate_disparity = 1 # fraud prevalence in minority / fraud prevalence in majority\n",
    "\n",
    "typeIII_df = group_prevalence_disparity(large_sample_df, original_sample_df, majority_size, fraud_rate_disparity)\n",
    "# For the TypeIII, we want only to change the separability of the groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "100b4c2e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# To do so, we use the wrapper for the MVN distributions.\n",
    "\n",
    "bias_sampler = TypeIIIBiasSampler(\"fraud_bool\", \"group\", 0.9, 0.05, protected_attribute_values=[\"Majority\", \"Minority\"])\n",
    "bias_sampler(typeIII_df)  # This operation is inplace and injects the bias to new columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c651a4",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# For TypeIV, we control the prevalence to be higher in train months:\n",
    "majority_size = 0.9\n",
    "fraud_rate_disparity = [5, 5, 5, 5, 5, 5, 1, 1]  # First 6 months for train, last 2 months for test.\n",
    "\n",
    "typeIV_df = group_prevalence_disparity(large_sample_df, original_sample_df, majority_size, fraud_rate_disparity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "865c0435",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Helper method to obtain last dataset, with differences in separability by month.\n",
    "def separability_disparities(large_sample_df, recalls, seed=42):\n",
    "    final_bias_dfs = []\n",
    "    df = group_prevalence_disparity(large_sample_df, 0.5, 1)\n",
    "    for month, recall in zip(sorted(df[\"month\"].unique()), recalls):\n",
    "        # For each month we will create a different bias sampler, with the defined separability.\n",
    "        bias_sampler = TypeIIIBiasSampler(\n",
    "            \"fraud_bool\",\n",
    "            \"group\",\n",
    "            recall,\n",
    "            0.05,\n",
    "            protected_attribute_values=[\"Majority\", \"Minority\"],\n",
    "            seed=month \n",
    "        )\n",
    "        final_bias_dfs.append(bias_sampler(df[df[\"month\"] ==month]))\n",
    "    return pd.concat(final_bias_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01cd2530",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# The separabilities are defined to be high in train data and negligible in test data.\n",
    "separability = [0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.05, 0.05]\n",
    "\n",
    "typeV_df = separability_disparities(large_sample_df, separability)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}