7a) Basic Statistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7a) Basic Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind_from_stats\n",
    "#use pandas to load csv file\n",
    "import pandas as pandas\n",
    "#use numpy to calculate stuff\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data\n",
    "redwinedata = pandas.read_csv('data/winequality-red.csv', sep =';')\n",
    "whitewinedata = pandas.read_csv('data/winequality-white.csv', sep =';')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Redwine Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "      <td>1599.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.319637</td>\n",
       "      <td>0.527821</td>\n",
       "      <td>0.270976</td>\n",
       "      <td>2.538806</td>\n",
       "      <td>0.087467</td>\n",
       "      <td>15.874922</td>\n",
       "      <td>46.467792</td>\n",
       "      <td>0.996747</td>\n",
       "      <td>3.311113</td>\n",
       "      <td>0.658149</td>\n",
       "      <td>10.422983</td>\n",
       "      <td>5.636023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.741096</td>\n",
       "      <td>0.179060</td>\n",
       "      <td>0.194801</td>\n",
       "      <td>1.409928</td>\n",
       "      <td>0.047065</td>\n",
       "      <td>10.460157</td>\n",
       "      <td>32.895324</td>\n",
       "      <td>0.001887</td>\n",
       "      <td>0.154386</td>\n",
       "      <td>0.169507</td>\n",
       "      <td>1.065668</td>\n",
       "      <td>0.807569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>4.600000</td>\n",
       "      <td>0.120000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.012000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.990070</td>\n",
       "      <td>2.740000</td>\n",
       "      <td>0.330000</td>\n",
       "      <td>8.400000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.100000</td>\n",
       "      <td>0.390000</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>1.900000</td>\n",
       "      <td>0.070000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>0.995600</td>\n",
       "      <td>3.210000</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>9.500000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>7.900000</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.260000</td>\n",
       "      <td>2.200000</td>\n",
       "      <td>0.079000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>0.996750</td>\n",
       "      <td>3.310000</td>\n",
       "      <td>0.620000</td>\n",
       "      <td>10.200000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.200000</td>\n",
       "      <td>0.640000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>2.600000</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.997835</td>\n",
       "      <td>3.400000</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>11.100000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>15.900000</td>\n",
       "      <td>1.580000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>15.500000</td>\n",
       "      <td>0.611000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>1.003690</td>\n",
       "      <td>4.010000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>14.900000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       fixed acidity  volatile acidity  citric acid  residual sugar  \\\n",
       "count    1599.000000       1599.000000  1599.000000     1599.000000   \n",
       "mean        8.319637          0.527821     0.270976        2.538806   \n",
       "std         1.741096          0.179060     0.194801        1.409928   \n",
       "min         4.600000          0.120000     0.000000        0.900000   \n",
       "25%         7.100000          0.390000     0.090000        1.900000   \n",
       "50%         7.900000          0.520000     0.260000        2.200000   \n",
       "75%         9.200000          0.640000     0.420000        2.600000   \n",
       "max        15.900000          1.580000     1.000000       15.500000   \n",
       "\n",
       "         chlorides  free sulfur dioxide  total sulfur dioxide      density  \\\n",
       "count  1599.000000          1599.000000           1599.000000  1599.000000   \n",
       "mean      0.087467            15.874922             46.467792     0.996747   \n",
       "std       0.047065            10.460157             32.895324     0.001887   \n",
       "min       0.012000             1.000000              6.000000     0.990070   \n",
       "25%       0.070000             7.000000             22.000000     0.995600   \n",
       "50%       0.079000            14.000000             38.000000     0.996750   \n",
       "75%       0.090000            21.000000             62.000000     0.997835   \n",
       "max       0.611000            72.000000            289.000000     1.003690   \n",
       "\n",
       "                pH    sulphates      alcohol      quality  \n",
       "count  1599.000000  1599.000000  1599.000000  1599.000000  \n",
       "mean      3.311113     0.658149    10.422983     5.636023  \n",
       "std       0.154386     0.169507     1.065668     0.807569  \n",
       "min       2.740000     0.330000     8.400000     3.000000  \n",
       "25%       3.210000     0.550000     9.500000     5.000000  \n",
       "50%       3.310000     0.620000    10.200000     6.000000  \n",
       "75%       3.400000     0.730000    11.100000     6.000000  \n",
       "max       4.010000     2.000000    14.900000     8.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "redwinedata.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Whitewine Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "      <td>4898.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.854788</td>\n",
       "      <td>0.278241</td>\n",
       "      <td>0.334192</td>\n",
       "      <td>6.391415</td>\n",
       "      <td>0.045772</td>\n",
       "      <td>35.308085</td>\n",
       "      <td>138.360657</td>\n",
       "      <td>0.994027</td>\n",
       "      <td>3.188267</td>\n",
       "      <td>0.489847</td>\n",
       "      <td>10.514267</td>\n",
       "      <td>5.877909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.843868</td>\n",
       "      <td>0.100795</td>\n",
       "      <td>0.121020</td>\n",
       "      <td>5.072058</td>\n",
       "      <td>0.021848</td>\n",
       "      <td>17.007137</td>\n",
       "      <td>42.498065</td>\n",
       "      <td>0.002991</td>\n",
       "      <td>0.151001</td>\n",
       "      <td>0.114126</td>\n",
       "      <td>1.230621</td>\n",
       "      <td>0.885639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.800000</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.009000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.987110</td>\n",
       "      <td>2.720000</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6.300000</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.270000</td>\n",
       "      <td>1.700000</td>\n",
       "      <td>0.036000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>0.991723</td>\n",
       "      <td>3.090000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>9.500000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>6.800000</td>\n",
       "      <td>0.260000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>5.200000</td>\n",
       "      <td>0.043000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>0.993740</td>\n",
       "      <td>3.180000</td>\n",
       "      <td>0.470000</td>\n",
       "      <td>10.400000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.300000</td>\n",
       "      <td>0.320000</td>\n",
       "      <td>0.390000</td>\n",
       "      <td>9.900000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>167.000000</td>\n",
       "      <td>0.996100</td>\n",
       "      <td>3.280000</td>\n",
       "      <td>0.550000</td>\n",
       "      <td>11.400000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>14.200000</td>\n",
       "      <td>1.100000</td>\n",
       "      <td>1.660000</td>\n",
       "      <td>65.800000</td>\n",
       "      <td>0.346000</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>440.000000</td>\n",
       "      <td>1.038980</td>\n",
       "      <td>3.820000</td>\n",
       "      <td>1.080000</td>\n",
       "      <td>14.200000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       fixed acidity  volatile acidity  citric acid  residual sugar  \\\n",
       "count    4898.000000       4898.000000  4898.000000     4898.000000   \n",
       "mean        6.854788          0.278241     0.334192        6.391415   \n",
       "std         0.843868          0.100795     0.121020        5.072058   \n",
       "min         3.800000          0.080000     0.000000        0.600000   \n",
       "25%         6.300000          0.210000     0.270000        1.700000   \n",
       "50%         6.800000          0.260000     0.320000        5.200000   \n",
       "75%         7.300000          0.320000     0.390000        9.900000   \n",
       "max        14.200000          1.100000     1.660000       65.800000   \n",
       "\n",
       "         chlorides  free sulfur dioxide  total sulfur dioxide      density  \\\n",
       "count  4898.000000          4898.000000           4898.000000  4898.000000   \n",
       "mean      0.045772            35.308085            138.360657     0.994027   \n",
       "std       0.021848            17.007137             42.498065     0.002991   \n",
       "min       0.009000             2.000000              9.000000     0.987110   \n",
       "25%       0.036000            23.000000            108.000000     0.991723   \n",
       "50%       0.043000            34.000000            134.000000     0.993740   \n",
       "75%       0.050000            46.000000            167.000000     0.996100   \n",
       "max       0.346000           289.000000            440.000000     1.038980   \n",
       "\n",
       "                pH    sulphates      alcohol      quality  \n",
       "count  4898.000000  4898.000000  4898.000000  4898.000000  \n",
       "mean      3.188267     0.489847    10.514267     5.877909  \n",
       "std       0.151001     0.114126     1.230621     0.885639  \n",
       "min       2.720000     0.220000     8.000000     3.000000  \n",
       "25%       3.090000     0.410000     9.500000     5.000000  \n",
       "50%       3.180000     0.470000    10.400000     6.000000  \n",
       "75%       3.280000     0.550000    11.400000     6.000000  \n",
       "max       3.820000     1.080000    14.200000     9.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whitewinedata.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the data above there are several features which distinguish redwine from whitewine e.g. residual sugar seems to be significantly different (white wine is indeed sweeter) as well as volatile acidity and sulfur dioxide values. \n",
    "I assume that a clustering algorithm is able to pick up those differences, maybe dropping some features would be of help."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "redwinestats = redwinedata.describe()\n",
    "whitewinestats = whitewinedata.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### T-Tests\n",
    "Because it will also be useful for 7.c) we calculate some T-Test. \n",
    "The results show that all of the differences in the features are significant.\n",
    "In order of descending absolute t-values, the significance is:\n",
    "1. total sulfur dioxide \n",
    "2. volatile acidity\n",
    "3. chlorides\n",
    "4. sulvates\n",
    "5. fixed acidity\n",
    "6. free sulfur dioxide\n",
    "7. density\n",
    "8. residual sugar\n",
    "9. pH\n",
    "10. citric acid\n",
    "11. quality\n",
    "12. alcohol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fixed acidity: \n",
      "Ttest_indResult(statistic=44.90556823034994, pvalue=0.0)\n",
      "\n",
      "volatile acidity: \n",
      "Ttest_indResult(statistic=69.49328807560799, pvalue=0.0)\n",
      "\n",
      "citric acid: \n",
      "Ttest_indResult(statistic=-15.374954114169297, pvalue=2.003735701834766e-52)\n",
      "\n",
      "residual sugar: \n",
      "Ttest_indResult(statistic=-29.996104565841236, pvalue=2.9593541527042885e-185)\n",
      "\n",
      "chlorides: \n",
      "Ttest_indResult(statistic=48.12306603007984, pvalue=0.0)\n",
      "\n",
      "free sulfur dioxide: \n",
      "Ttest_indResult(statistic=-43.10610584206572, pvalue=0.0)\n",
      "\n",
      "total sulfur dioxide: \n",
      "Ttest_indResult(statistic=-79.07462032921993, pvalue=0.0)\n",
      "\n",
      "density: \n",
      "Ttest_indResult(statistic=34.20022289522647, pvalue=7.31960462516311e-236)\n",
      "\n",
      "pH: \n",
      "Ttest_indResult(statistic=28.09002558145368, pvalue=5.923307967110629e-164)\n",
      "\n",
      "sulphates: \n",
      "Ttest_indResult(statistic=44.96340217575478, pvalue=0.0)\n",
      "\n",
      "alcohol: \n",
      "Ttest_indResult(statistic=-2.658512898748449, pvalue=0.007867873993300209)\n",
      "\n",
      "quality: \n",
      "Ttest_indResult(statistic=-9.685649554187581, pvalue=4.888069044207019e-22)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for featureNr in range(0, redwinestats.shape[1]):\n",
    "    print(list(redwinestats)[featureNr] + \": \")\n",
    "    print(\n",
    "    ttest_ind_from_stats(\n",
    "        mean1=redwinestats.iloc[1,featureNr], std1=redwinestats.iloc[2,featureNr], nobs1=redwinestats.iloc[0,featureNr], \n",
    "        mean2=whitewinestats.iloc[1,featureNr], std2=whitewinestats.iloc[2,featureNr], nobs2=whitewinestats.iloc[0,featureNr]\n",
    "    )\n",
    "    )\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}