20181105 작업준비.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 0\n",
    "b = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AK홀딩스\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-17273ee6e750>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[0mmodel_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    131\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0my\u001b[0m \u001b[1;32min\u001b[0m \u001b[0my_li\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 132\u001b[1;33m         \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodelfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0malg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_li\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    133\u001b[0m         \u001b[0mmodel_set\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    134\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-4-17273ee6e750>\u001b[0m in \u001b[0;36mmodelfit\u001b[1;34m(alg, train, test, X, y)\u001b[0m\n\u001b[0;32m     74\u001b[0m                     \u001b[0mnum_boost_round\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0malg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'n_estimators'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m                     \u001b[0mnfold\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mearly_stopping_rounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m                     show_stdv=False)\n\u001b[0m\u001b[0;32m     77\u001b[0m     \u001b[0malg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcvresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[0malg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meval_metric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'auc'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\moon\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mcv\u001b[1;34m(params, dtrain, num_boost_round, nfold, stratified, folds, metrics, obj, feval, maximize, early_stopping_rounds, fpreproc, as_pandas, verbose_eval, show_stdv, seed, callbacks, shuffle)\u001b[0m\n\u001b[0;32m    404\u001b[0m                            evaluation_result_list=None))\n\u001b[0;32m    405\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mfold\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcvfolds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 406\u001b[1;33m             \u001b[0mfold\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    407\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maggcv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcvfolds\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\moon\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, iteration, fobj)\u001b[0m\n\u001b[0;32m    216\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    217\u001b[0m         \u001b[1;34m\"\"\"\"Update the boosters for one iteration\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 218\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    220\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\moon\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m    892\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    893\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m--> 894\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m    895\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    896\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pickle\n",
    "with open('raw_data_20181008.pickle', 'rb') as handle:\n",
    "    raw_data = pickle.load(handle)\n",
    "ks11 = pd.read_csv('ks11.csv')\n",
    "ks11.index = ks11.Date\n",
    "del ks11['Date']\n",
    "\n",
    "copy_keys = list(raw_data.keys())\n",
    "for stock in copy_keys:\n",
    "    if len(raw_data[stock].loc[:'2013-01-01']) < 1:\n",
    "        del raw_data[stock]\n",
    "        \n",
    "ks11 = ks11.drop(ks11.index[np.where(ks11.Volume==0)[0]])\n",
    "        \n",
    "for key, df in raw_data.items():\n",
    "    raw_data[key] = df.drop(df.index[np.where(df.Volume==0)[0]], axis=0)\n",
    "    \n",
    "# data = ks11.copy()\n",
    "# data = raw_data['삼성전자'].copy()\n",
    "# data = raw_data['NAVER'].copy()\n",
    "# data = raw_data['LG화학'].copy()\n",
    "data = raw_data['미래에셋대우'].copy()\n",
    "\n",
    "data[\"pct_change\"] = data.Close.pct_change()\n",
    "for i in range(3, 31, 2):\n",
    "    data[\"close_ma_{}\".format(i)] = data.Close.rolling(window = i).mean()\n",
    "for i in [7, 21, 30, 60, 90, 91, 100, 120]:\n",
    "    data[\"his_vol_{}\".format(i)] = data[\"pct_change\"].rolling(window = i).std()*(252**0.5)\n",
    "for i in [1, 3, 5, 7, 10, 12, 14]:\n",
    "    data['returnsCC{}'.format(i) ]= np.log(data.Open / data.Close.shift(i))\n",
    "    data['returnsOO{}'.format(i) ] = np.log(data.Open / data.Open.shift(i))\n",
    "    data['returnsCC{}'.format(i) ]= np.log(data.Close / data.Close.shift(i))\n",
    "    data['returnsOC{}'.format(i) ] = np.log(data.Close / data.Open.shift(i))\n",
    "for i in [1, 3, 5, 7, 10, 15, 20, 25, 30, 35, 40, 45, 50,\n",
    "          55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120]:\n",
    "    for shift in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:\n",
    "        data['Y1_{}_s{}'.format(i, shift)] = (data.Close - data.Close.shift(shift).rolling(i).mean())\n",
    "        data['Y3_{}_s{}'.format(i, shift)] = (data.Close.rolling(i).mean() - data.Close.shift(shift).rolling(i).mean())\n",
    "        data['sh{}_Y1_{}'.format(shift, i)] = np.where((data['Y1_{}_s{}'.format(i, shift)]).shift(-1*shift)>=0, 1, 0)\n",
    "        data['sh{}_Y3_{}'.format(shift, i)] = np.where((data['Y3_{}_s{}'.format(i, shift)]).shift(-1*shift)>=0, 1, 0)\n",
    "        \n",
    "data_ = data.loc['2018-07-01':'2018-10-01']\n",
    "data = data.loc['2013-07-01':'2018-07-01']\n",
    "\n",
    "X_li = [i for i in data.columns if (i.find('sh') == -1) if (i.find('Volume') == -1)]\n",
    "\n",
    "X_li2 = [i for i in data.columns \n",
    "         if (i.find('sh') == -1) \n",
    "         if (i.find('Volume') == -1)\n",
    "         if (i.find('Y') == -1)]\n",
    "\n",
    "%run moon_code\n",
    "%run xgb_modelfit\n",
    "\n",
    "import copy\n",
    "\n",
    "train, test = train_test_split(data, 0.8, dtrain=True)\n",
    "\n",
    "y_li = [i for i in data.columns if i.find('sh') != -1]\n",
    "\n",
    "def modelfit(alg, train, test, X, y):\n",
    "    alg = copy.deepcopy(alg)\n",
    "    xgb_params = alg.get_xgb_params()\n",
    "    xgtrain = xgb.DMatrix(train[X].values, label=train[y].values)\n",
    "    cvresult=xgb.cv(xgb_params, xgtrain,\n",
    "                    num_boost_round=alg.get_params()['n_estimators'],\n",
    "                    nfold=5, early_stopping_rounds=50,\n",
    "                    show_stdv=False)\n",
    "    alg.set_params(n_estimators=cvresult.shape[0])\n",
    "    alg.fit(train[X], train[y], eval_metric='auc')\n",
    "    dtrain_predictions = alg.predict(train[X])\n",
    "    dtrain_predprob = alg.predict_proba(train[X])[:,1]\n",
    "    dtest_predictions = alg.predict(test[X])\n",
    "    dtest_predprob = alg.predict_proba(test[X])[:,1]\n",
    "#     print(\"Train Accuracy : {:.4f}\".format(metrics.accuracy_score(\n",
    "#                             train[y].values, dtrain_predictions)))\n",
    "#     print(\"Train AUC Score : {:.4f}\".format(metrics.roc_auc_score(\n",
    "#                             train[y].values, dtrain_predprob)))\n",
    "#     print(\"Test  Accuracy : {:.4f}\".format(metrics.accuracy_score(\n",
    "#                             test[y].values, dtest_predictions)))\n",
    "#     print(\"Test  AUC Score : {:.4f}\".format(metrics.roc_auc_score(\n",
    "#                             test[y].values, dtest_predprob)))\n",
    "    return alg\n",
    "\n",
    "for idx, stock in enumerate(raw_data.keys()):\n",
    "        \n",
    "    if idx < a : \n",
    "        continue\n",
    "    if idx > b :\n",
    "        break\n",
    "        \n",
    "    print(stock)\n",
    "    data = raw_data[stock].copy()\n",
    "\n",
    "    data[\"pct_change\"] = data.Close.pct_change()\n",
    "    for i in range(3, 31, 2):\n",
    "        data[\"close_ma_{}\".format(i)] = data.Close.rolling(window = i).mean()\n",
    "    for i in [7, 21, 30, 60, 90, 91, 100, 120]:\n",
    "        data[\"his_vol_{}\".format(i)] = data[\"pct_change\"].rolling(window = i).std()*(252**0.5)\n",
    "    for i in [1, 3, 5, 7, 10, 12, 14]:\n",
    "        data['returnsCC{}'.format(i) ]= np.log(data.Open / data.Close.shift(i))\n",
    "        data['returnsOO{}'.format(i) ] = np.log(data.Open / data.Open.shift(i))\n",
    "        data['returnsCC{}'.format(i) ]= np.log(data.Close / data.Close.shift(i))\n",
    "        data['returnsOC{}'.format(i) ] = np.log(data.Close / data.Open.shift(i))\n",
    "    for i in [1, 3, 5, 7, 10, 15, 20, 25, 30, 35, 40, 45, 50,\n",
    "              55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 105, 110, 115, 120]:\n",
    "        for shift in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:\n",
    "            data['Y1_{}_s{}'.format(i, shift)] = (data.Close - data.Close.shift(shift).rolling(i).mean())\n",
    "            data['Y3_{}_s{}'.format(i, shift)] = (data.Close.rolling(i).mean() - data.Close.shift(shift).rolling(i).mean())\n",
    "            data['sh{}_Y1_{}'.format(shift, i)] = np.where((data['Y1_{}_s{}'.format(i, shift)]).shift(-1*shift)>=0, 1, 0)\n",
    "            data['sh{}_Y3_{}'.format(shift, i)] = np.where((data['Y3_{}_s{}'.format(i, shift)]).shift(-1*shift)>=0, 1, 0)\n",
    "\n",
    "    data_ = data.loc['2018-07-01':'2018-10-01']\n",
    "    data = data.loc['2013-07-01':'2018-07-01']\n",
    "\n",
    "    X_li = [i for i in data.columns if (i.find('sh') == -1) if (i.find('Volume') == -1)]\n",
    "\n",
    "    train, test = train_test_split(data, 0.8, dtrain=True)\n",
    "\n",
    "    y_li = [i for i in data.columns if i.find('sh') != -1]\n",
    "\n",
    "    model_set = {}\n",
    "    for y in y_li:\n",
    "        model = modelfit(alg, train, test, X_li, y)\n",
    "        model_set[y] = model\n",
    "        \n",
    "    try:    \n",
    "        y1 = pd.DataFrame()\n",
    "        y3 = pd.DataFrame()\n",
    "        for idx, y in enumerate(y_li):\n",
    "            acc = metrics.accuracy_score(test[y], model_set[y].predict(test[X_li]))\n",
    "            auc = metrics.roc_auc_score(test[y], model_set[y].predict(test[X_li]))\n",
    "            res = pd.DataFrame([acc,auc], index=['accuracy', 'AUC'], columns=[y])\n",
    "            if idx%2==0:\n",
    "                y1 = pd.concat((y1, res), axis=1)\n",
    "            else:\n",
    "                y3 = pd.concat((y3, res), axis=1)\n",
    "\n",
    "        Y1 = np.array([])\n",
    "        for i in range(int(270/10)):\n",
    "            i *= 10\n",
    "            if i == 0:\n",
    "                Y1 = y1.values[:,i:i+10]\n",
    "            else:\n",
    "                Y1 = np.vstack((Y1, y1.values[:,i:i+10]))\n",
    "    except ValueError:\n",
    "        continue\n",
    "\n",
    "    Y3 = np.array([])\n",
    "    for i in range(int(270/10)):\n",
    "        i *= 10\n",
    "        if i == 0:\n",
    "            Y3 = y3.values[:,i:i+10]\n",
    "        else:\n",
    "            Y3 = np.vstack((Y3, y3.values[:,i:i+10]))\n",
    "\n",
    "    Y = np.hstack((Y3, Y1))\n",
    "\n",
    "    y1.to_excel('./20181105_result/y1_{}.xlsx'.format(stock))\n",
    "    y3.to_excel('./20181105_result/y3_{}.xlsx'.format(stock))\n",
    "    pd.DataFrame(Y).to_excel('./20181105_result/Y_{}.xlsx'.format(stock))\n",
    "\n",
    "    with open('./20181105_result/model_set_{}_20181031.pickle'.format(stock), 'wb') as handle:\n",
    "        pickle.dump(model_set, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}