LayerNormalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "from numpy.random import seed\n",
    "seed(1)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras.layers import Input, Dense, Conv2D, MaxPool2D, Flatten, Multiply, Permute, Embedding, Bidirectional, Merge\n",
    "from keras.preprocessing import sequence\n",
    "from keras.datasets import imdb\n",
    "\n",
    "from LayerNormalizationRNN import LSTM, GRU\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "25000 train sequences\n",
      "25000 test sequences\n",
      "Pad sequences (samples x time)\n",
      "x_train shape: (25000, 80)\n",
      "x_test shape: (25000, 80)\n"
     ]
    }
   ],
   "source": [
    "max_features = 20000\n",
    "maxlen = 80  # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 128\n",
    "INDEX_FROM = 3\n",
    "\n",
    "print('Loading data...')\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features, index_from = INDEX_FROM)\n",
    "print(len(x_train), 'train sequences')\n",
    "print(len(x_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "K.get_session().close()\n",
    "cfg = K.tf.ConfigProto()\n",
    "cfg.gpu_options.allow_growth = True\n",
    "K.set_session(K.tf.Session(config=cfg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = Input(shape=(maxlen,))\n",
    "x = Embedding(max_features, 128)(inputs)\n",
    "\n",
    "x = GRU(64, layer_to_normalize=(\"input_gate\", \"input_recurrent\", \"recurrent_gate\", \"recurrent_recurrent\"), normalize_seperately=True)(x)\n",
    "\n",
    "predictions = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "\n",
    "model = Model(inputs=inputs, outputs=predictions)\n",
    "model.compile(optimizer='rmsprop',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/5\n",
      "25000/25000 [==============================] - 134s 5ms/step - loss: 0.6461 - acc: 0.6650 - val_loss: 0.5243 - val_acc: 0.8154\n",
      "Epoch 2/5\n",
      "25000/25000 [==============================] - 131s 5ms/step - loss: 0.3807 - acc: 0.8562 - val_loss: 0.4887 - val_acc: 0.7623\n",
      "Epoch 3/5\n",
      "25000/25000 [==============================] - 131s 5ms/step - loss: 0.2424 - acc: 0.9076 - val_loss: 0.4184 - val_acc: 0.8208\n",
      "Epoch 4/5\n",
      "25000/25000 [==============================] - 131s 5ms/step - loss: 0.1793 - acc: 0.9338 - val_loss: 0.4125 - val_acc: 0.8272\n",
      "Epoch 5/5\n",
      "25000/25000 [==============================] - 131s 5ms/step - loss: 0.1338 - acc: 0.9510 - val_loss: 0.4893 - val_acc: 0.8136\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_train, y_train, batch_size=batch_size, epochs=5, validation_data=(x_test, y_test))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}