20190403코드 및 설명.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import sklearn.metrics as met\n",
    "import lightgbm as lg\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('0306_dataset.pickle', 'rb') as handle:\n",
    "    raw_data,prepare_training = pickle.load(handle)\n",
    "    \n",
    "with open('0326_predData.pickle', 'rb') as handle:\n",
    "    stock_score,stock_pred,max_precision = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### raw_data : 주식데이터 + 파생변수 \\n\n",
    "##### prepare_training : 이전에 했던 코릴레이션으로 파생변수 선택리스트 \\n\n",
    "##### stock_score : 주식별 Y에 대한 score 정리 (사실상 안씀) \\n\n",
    "##### stock_pred : 주식별 Y별 pred \\n\n",
    "##### max_precision : 주식별 최고성적의 Y에 대한 precision과 profit \\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  profit을 구하는 알고리즘에 대한 설명 \\n\n",
    "##### 먼저 모델링을 통해 prediction을 하고 train data에 대한 score를 구한다. (lgbm, xgb를 둘다 사용해봐서 코드상엔 lgbm = xgb.model로 되어있음)\\n\n",
    "##### 그다음 test시작일 (2017년 6월 30일)을 기준으로 최근 90일 동안의 precision이 조건에 맞지않으면 -1로 바꿔준다. \\n\n",
    "##### Y_{}_{}에서 첫번째는 가장 precision이 높았던 number이고 두번째는 voting에 사용하기 때문에 전부 사용한다. \\n\n",
    "##### 따라서 아래의 코드의 경우 1가지 날에 대한 pred을 voting할 때 30개 중에서 -1로 분류된것을 제외하고 0,1값만을 사용하여 voting한다 \\n\n",
    "##### 이렇게 최종적으로 pred이 산출된 것을 사용해서 profit을 측정한다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 수익률 구하는 함수\n",
    "\n",
    "def transaction(stock, pred, term1):\n",
    "    tran_table = pd.DataFrame(columns = ['profit'])\n",
    "    now_index = 0\n",
    "    tran_table.loc['2017-06-30'] = 0\n",
    "    while now_index <= len(pred):\n",
    "        if pred.iloc[now_index] > 0:\n",
    "            tran_table.loc[pred.index[now_index+term1]] = raw_data[str(stock)]['Y1_{:02}'.format(term1)].iloc[now_index]\n",
    "            now_index = now_index + term1\n",
    "        else:\n",
    "            now_index = now_index + 1\n",
    "\n",
    "        if now_index + term1 >= len(pred):\n",
    "            break\n",
    "\n",
    "    tran_table = tran_table+1\n",
    "    tran_table = tran_table.cumprod()\n",
    "    \n",
    "    return tran_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 .모델링 코드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stock_pred_pram = {}\n",
    "\n",
    "main_epoc = pd.DataFrame()\n",
    "for stock in raw_data.keys():\n",
    "    i = int(max_precision['number'].loc[stock])\n",
    "    main_epoc = pd.DataFrame()\n",
    "    for j in np.arange(1,90,3):\n",
    "        column_list = []\n",
    "        for col in prepare_training[stock]['Y2_{:02}_{:02}'.format(i,j)].keys():\n",
    "            column_list.append(col)\n",
    "        lgbm = xgb.XGBRegressor(random_state=0,n_jobs=-1)\n",
    "        X_train = raw_data[stock][column_list][:'2017-06-30']\n",
    "        y_train = raw_data[stock]['Y2_{:02}_{:02}'.format(i,j)][:'2017-06-30']\n",
    "        X_test = raw_data[stock][column_list]['2017-06-30':'2018-06-30']\n",
    "        y_test = raw_data[stock]['Y1_{:02}'.format(i)]['2017-06-30':'2018-06-30']\n",
    "        lgbm.fit(X_train, y_train)\n",
    "        over_term_pred = pd.concat([X_train['2017-01-01':],X_test['2017-06-30':'2018-06-30']],axis=0)\n",
    "        a = lgbm.predict(over_term_pred)\n",
    "        a =  np.where(a > 0,1,0)\n",
    "        epoc_series = pd.DataFrame(a, columns = ['{}_pred'.format(j)], index = over_term_pred.index)\n",
    "        main_epoc = pd.concat([main_epoc,epoc_series],axis=1)\n",
    "    stock_pred_pram[stock] = main_epoc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 최근 precision을 보고 pred을 믿을지 말지 판단 일정 수준 이하면 해당 pred을 voting에 사용하지 않는다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for stock in raw_data.keys():\n",
    "    i = int(max_precision['number'][stock])\n",
    "    stock_pred_copy = stock_pred_pram[stock].copy()\n",
    "    for j in np.arange(1,90,3):\n",
    "\n",
    "        raw_Y = raw_data[stock]['Y1_{:02}'.format(int(max_precision['number'][stock]))]\n",
    "        raw_Y = raw_Y[stock_pred_pram[stock].index]\n",
    "        real_Y = pd.DataFrame(np.where(raw_Y > 0,1,0), index = raw_Y.index, columns = [raw_Y.name])\n",
    "\n",
    "        column_list = []\n",
    "        for col in prepare_training[stock]['Y2_{:02}_{:02}'.format(i,j)].keys():\n",
    "            column_list.append(col)\n",
    "\n",
    "        xgboost_model = xgb.XGBRegressor(random_state=0,n_jobs=-1)\n",
    "        X_train = raw_data[stock][column_list][:'2017-06-30']\n",
    "        y_train = raw_data[stock]['Y2_{:02}_{:02}'.format(i,j)][:'2017-06-30']\n",
    "        xgboost_model.fit(X_train, y_train)\n",
    "        train_pred = xgboost_model.predict(raw_data[stock][column_list][:'2017-06-30'])\n",
    "        main_score = met.precision_score(np.where(train_pred > 0,1,0), np.where(raw_data[stock]['Y1_{:02}'.format(i)][:'2017-06-30'] > 0,1,0))\n",
    "\n",
    "        k = 0\n",
    "        while 90 + k <= len(raw_Y):\n",
    "            if met.precision_score(stock_pred_copy['{}_pred'.format(j)].iloc[k:90+k],real_Y.iloc[k:90+k]) < 0.5:\n",
    "                stock_pred_pram[stock]['{}_pred'.format(j)].iloc[89+k] = -1\n",
    "                \n",
    "            elif met.precision_score(stock_pred_copy['{}_pred'.format(j)].iloc[k:90+k],real_Y.iloc[k:90+k]) < main_score*0.9:\n",
    "                stock_pred_pram[stock]['{}_pred'.format(j)].iloc[89+k] = -1\n",
    "            k = k + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 위에서 최종적으로 pred을 0,1,-1로 분류하여 -1인 것을 제외하고 voting을 통해 최종 pred을 결정한다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "votng_df = pd.DataFrame(index = stock_pred_pram[stock]['2017-06-30':].iloc[1:].index,columns = list(raw_data.keys()))\n",
    "\n",
    "for stock in raw_data.keys():\n",
    "    datas = stock_pred_pram[stock]['2017-06-30':].iloc[1:]\n",
    "    for ind in datas.index:\n",
    "        a = datas.loc[ind].values\n",
    "        if len(a[a >=0]) == 0:\n",
    "            votng_df[stock].loc[ind] = 0\n",
    "        \n",
    "        elif a[a >= 0].sum()/len(a[a >= 0]) >= 0.5:\n",
    "            votng_df[stock].loc[ind] = 1\n",
    "            \n",
    "        else:\n",
    "            votng_df[stock].loc[ind] = 0\n",
    "            \n",
    "profit_dict = {}\n",
    "\n",
    "for stock in raw_data.keys():\n",
    "    pred = votng_df[stock]\n",
    "    i = int(max_precision['number'][stock])\n",
    "    profit_dict[stock] = transaction(stock, pred, i)\n",
    "\n",
    "    indexing = []\n",
    "    for i in range(len(profit_dict[stock])):\n",
    "        indexing.append(str(profit_dict[stock].index[i])[:10])\n",
    "    profit_dict[stock].index = indexing\n",
    "    \n",
    "profit_df = pd.DataFrame(columns = ['profit'])\n",
    "\n",
    "for stock in raw_data.keys():\n",
    "    profit_df.loc[stock] = float(profit_dict[stock].iloc[len(profit_dict[stock])-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}