cifar100-10_816_class_incremental.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import argparse\n",
    "import copy\n",
    "# import data_loader\n",
    "import pdb\n",
    "import torch.nn.functional as F\n",
    "import re, random, collections\n",
    "import pickle\n",
    "\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class args():\n",
    "\n",
    "    data_path = \"./Datasets/CIFAR100/\"\n",
    "    num_class = 100\n",
    "    class_per_task = 10\n",
    "    num_task = 10\n",
    "    dataset = \"cifar100\"\n",
    "    train_batch = 128\n",
    "    test_batch = 500\n",
    "    workers = 16\n",
    "    \n",
    "    random_classes = False\n",
    "    validation = 0\n",
    "    overflow=False\n",
    "    \n",
    "    n_classes=10\n",
    "    batch_size=128\n",
    "    lr=0.01\n",
    "    resume=False\n",
    "    total_epoch=250\n",
    "    model_path='ckpt816/cifar100-10_816'\n",
    "    \n",
    "args=args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    \n",
    "# seed=torch.randint(1,10000,[1])   \n",
    "set_seed(3473)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "import torch.backends.cudnn as cudnn\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import argparse\n",
    "# from utils import progress_bar\n",
    "from torch.optim.lr_scheduler import MultiStepLR\n",
    "torch.set_printoptions(precision=5,sci_mode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvAdapt(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, p):\n",
    "        super(ConvAdapt, self).__init__()\n",
    "        # Groupwise Convolution\n",
    "        self.gwc = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, groups=int(p/gp), bias=True)\n",
    "        # Pointwise Convolution\n",
    "        self.pwc = nn.Conv2d(in_channels, out_channels, kernel_size=1, groups=int(p/pt),bias=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gwc(x) + self.pwc(x)\n",
    "\n",
    "'''ResNet in PyTorch.\n",
    "\n",
    "For Pre-activation ResNet, see 'preact_resnet.py'.\n",
    "\n",
    "Reference:\n",
    "[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun\n",
    "    Deep Residual Learning for Image Recognition. arXiv:1512.03385\n",
    "'''\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class BasicBlock(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_planes, planes, stride=1):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.lhc1=ConvAdapt(planes,planes,planes)\n",
    "        self.bn1 = nn.BatchNorm2d(planes)\n",
    "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.lhc2=ConvAdapt(planes,planes,planes)\n",
    "        self.bn2 = nn.BatchNorm2d(planes)\n",
    "\n",
    "        self.shortcut = nn.Sequential()\n",
    "        if stride != 1 or in_planes != self.expansion*planes:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),\n",
    "                ConvAdapt(self.expansion*planes,self.expansion*planes,int(self.expansion*planes)),\n",
    "                nn.BatchNorm2d(self.expansion*planes)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.lhc1(self.conv1(x))))\n",
    "        out = self.bn2(self.lhc2(self.conv2(out)))\n",
    "        out += self.shortcut(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, num_blocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_planes = 64\n",
    "\n",
    "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.hc1=ConvAdapt(64,64,64)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n",
    "        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n",
    "        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n",
    "        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n",
    "        self.linear = nn.Linear(512*block.expansion, num_classes)\n",
    "\n",
    "    def _make_layer(self, block, planes, num_blocks, stride):\n",
    "        strides = [stride] + [1]*(num_blocks-1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(block(self.in_planes, planes, stride))\n",
    "            self.in_planes = planes * block.expansion\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out=self.conv1(x)\n",
    "        out = F.relu(self.bn1(out))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.layer4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        feat = out.view(out.size(0), -1)\n",
    "        out = self.linear(feat)\n",
    "        return out,feat\n",
    "\n",
    "\n",
    "def Net():\n",
    "    return ResNet(BasicBlock, [2,2,2,2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(task,acc,model):\n",
    "    print('Saving..')\n",
    "    statem = {\n",
    "        'net': model.state_dict(),\n",
    "        'acc': acc,\n",
    "    }\n",
    "    fname=args.model_path\n",
    "    if not os.path.isdir(fname):\n",
    "        os.makedirs(fname)\n",
    "    torch.save(statem, fname+'/ckpt_task'+str(task)+'.pth')\n",
    "\n",
    "        \n",
    "def load_model(task,model):\n",
    "    fname=args.model_path\n",
    "    # Load checkpoint.\n",
    "    print('==> Resuming from checkpoint..')\n",
    "    print(fname+'/ckpt_task'+str(task)+'.pth')\n",
    "    checkpoint = torch.load(fname+'/ckpt_task'+str(task)+'.pth')\n",
    "    model.load_state_dict(checkpoint['net'])\n",
    "    best_acc = checkpoint['acc']\n",
    "    return best_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_loader,epoch,task,model,task_model):\n",
    "    print('\\nEpoch: %d' % epoch)\n",
    "    model.train()\n",
    "    global best_acc\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "\n",
    "    Sloss=0\n",
    "    for batch_idx, (inputs, targets) in enumerate(train_loader):\n",
    "        inputs, targets = inputs.cuda(), targets.cuda()\n",
    "        targets=targets-task*args.class_per_task # assume sequential split for random  split mapping should me changed\n",
    "        optimizer.zero_grad()\n",
    "        outputs,feat_current = model(inputs)\n",
    " \n",
    "        loss = criterion(outputs, targets)\n",
    "    \n",
    "        ################################################################################\n",
    "        ############# this is slow version here we dont save \\mu and \\sigma of KLD######\n",
    "        # saving mu and sigma and maximize the distance from the current batch significantly \n",
    "        # speedup the model####\n",
    "        if task>=1:\n",
    "            for m in range(task):\n",
    "                with torch.no_grad():\n",
    "                    task_model[m].eval()\n",
    "                    out,feat_prev=task_model[m](inputs)\n",
    "                mean_loss=torch.max(1-torch.dist(feat_current,feat_prev,2)/args.train_batch,\n",
    "                                    torch.zeros(1).cuda())/task\n",
    "                loss+=0.05*mean_loss[0]\n",
    "        ###################################################################################\n",
    "\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        _, predicted = outputs.max(1)\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "        acc = 100.*correct/total\n",
    "    \n",
    "    print(\"[Train: ], [%d/%d: ], [Accuracy: %f], [Loss: %f] [LossS: %f] [Lr: %f]\" \n",
    "          %(epoch,args.total_epoch,acc, train_loss/batch_idx,Sloss/batch_idx,\n",
    "            optimizer.param_groups[0]['lr']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_task(task,inputs,model,task_model):\n",
    "    joint_entropy=[]\n",
    "    with torch.no_grad():\n",
    "        for m in range(task):\n",
    "            task_model[m].eval()\n",
    "            outputs,_=task_model[m](inputs)\n",
    "\n",
    "            sout=F.softmax(outputs,1)\n",
    "            entropy=-torch.sum(sout*torch.log(sout+0.0001),1)\n",
    "            joint_entropy.append(entropy)\n",
    "\n",
    "        model.eval()\n",
    "        outputs,_=model(inputs)\n",
    "\n",
    "        sout=F.softmax(outputs,1)\n",
    "        entropy=-torch.sum(sout*torch.log(sout+0.0001),1)\n",
    "        joint_entropy.append(entropy)\n",
    "\n",
    "\n",
    "    all_entropy=torch.zeros([joint_entropy[0].shape[0],task+1]).cuda()\n",
    "    for i in range(task+1):\n",
    "        all_entropy[:,i]=joint_entropy[i]\n",
    "    ctask=torch.argmin(all_entropy, axis=1)==task\n",
    "    correct=sum(ctask)\n",
    "    \n",
    "    return ctask, correct,all_entropy\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(test_loader,task,model,task_model):\n",
    "    global best_acc\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    cl_loss=0\n",
    "    tcorrect=0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets) in enumerate(test_loader):\n",
    "            inputs, targets1 = inputs.cuda(), targets.cuda()\n",
    "            targets=targets1-task*args.class_per_task\n",
    "            if task>0:\n",
    "                correct_sample,Ncorrect,_=check_task(task,inputs,model,task_model)\n",
    "                tcorrect+=Ncorrect\n",
    "                inputs=inputs[correct_sample]\n",
    "                targets=targets[correct_sample]\n",
    "            if inputs.shape[0]!=0:\n",
    "                outputs,_ = model(inputs)\n",
    "                loss = criterion(outputs, targets)\n",
    "\n",
    "\n",
    "                test_loss += loss.item()\n",
    "                _, predicted = outputs.max(1)\n",
    "                correct += predicted.eq(targets).sum().item()\n",
    "            total += targets1.size(0)\n",
    "    if task>0:\n",
    "        taskC= tcorrect.item()/total\n",
    "    else:\n",
    "        taskC=1.0 \n",
    "    acc = 100.*correct/total\n",
    "    print(\"[Test Accuracy: %f], [Loss: %f] [Correct: %f]\" %(acc,test_loss/batch_idx,taskC))\n",
    "    \n",
    "    \n",
    "    if acc>=best_acc:\n",
    "        save_model(task,acc,model)\n",
    "        best_acc=acc\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this false the gradient of the global parameter after the 1st task\n",
    "\n",
    "def grad_false(model):\n",
    "    gradf=[0,7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112,119,126,133]\n",
    "    i=0\n",
    "    for p in model.parameters():\n",
    "        if i in gradf:\n",
    "            p.requires_grad=False\n",
    "        i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import incremental_dataloader as data\n",
    "inc_dataset = data.IncrementalDataset(\n",
    "                                dataset_name=args.dataset,\n",
    "                                args = args,\n",
    "                                random_order=args.random_classes,\n",
    "                                shuffle=True,\n",
    "                                seed=1,\n",
    "                                batch_size=args.train_batch,\n",
    "                                workers=args.workers,\n",
    "                                validation_split=args.validation,\n",
    "                                increment=args.class_per_task,\n",
    "                            )\n",
    "task_data=[]\n",
    "for i in range(args.num_task):\n",
    "    task_info, train_loader, val_loader, test_loader = inc_dataset.new_task()\n",
    "\n",
    "    task_data.append([train_loader,test_loader])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# groupwise and pointwise group convolutional size\n",
    "# this cell is for the training of the class incremental \n",
    "# here cifar100 dataset divided into 10 task and each task contains 10 class\n",
    "gp=8\n",
    "pt=16\n",
    "###############################################\n",
    "\n",
    "task_model=[]\n",
    "task_acc=[]\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "for task in range(args.num_task):\n",
    "    if task==0:\n",
    "        best_acc=0\n",
    "        print('Training Task :---'+str(task))\n",
    "\n",
    "        train_loader, test_loader = task_data[task][0],task_data[task][1]\n",
    "        modelm = Net().cuda()\n",
    "\n",
    "        mse_loss=nn.MSELoss()\n",
    "        optimizer = optim.SGD(modelm.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-3)\n",
    "        schedulerG = MultiStepLR(optimizer, milestones=[100, 150, 200], gamma=0.1)\n",
    "\n",
    "        for epoch in range(args.total_epoch):\n",
    "            train(train_loader,epoch,task,modelm,task_model)\n",
    "            test(test_loader,task,modelm,task_model)\n",
    "            schedulerG.step()\n",
    "        acc1=load_model(task,modelm)\n",
    "        task_model.append(copy.deepcopy(modelm))\n",
    "\n",
    "\n",
    "    if task!=0:\n",
    "        print('Training Task :---'+str(task))\n",
    "        best_acc=0\n",
    "        train_loader, test_loader = task_data[task][0],task_data[task][1]\n",
    "        modelm = Net().cuda()\n",
    "\n",
    "        acc1=load_model(task-1,modelm)  \n",
    "        grad_false(modelm) # make the gradient false for the global parameter\n",
    "\n",
    "        mse_loss=nn.MSELoss()\n",
    "        optimizer = optim.SGD(modelm.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-3)\n",
    "        schedulerG = MultiStepLR(optimizer, milestones=[100,150,200],gamma=0.1)\n",
    "        if task>=1:\n",
    "            for epoch in range(args.total_epoch):\n",
    "                train(train_loader,epoch,task,modelm,task_model)\n",
    "                test(test_loader,task,modelm,task_model)\n",
    "                schedulerG.step()\n",
    "\n",
    "        acc1=load_model(task,modelm)\n",
    "        task_model.append(copy.deepcopy(modelm))\n",
    "\n",
    "    task_acc.append(acc1)\n",
    "    print('Task: '+str(task)+'  Test_accuracy: '+ str(acc1))\n",
    "\n",
    "# This evaluation only check the accuracy of the current task with all the previous task.\n",
    "# therefore not correct since if trained the model for the 10 task the 1st task will get \n",
    "# evaluated with 10 task. Next cell provides the model evaluation\n",
    "\n",
    "print(task_acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# previous cell save model for all task with its local and global parameter.\n",
    "# global parameters are repeated in each model\n",
    "# this cell is for evaluation for the class incremental \n",
    "\n",
    "\n",
    "def inference_check_task(dtask,mtask,inputs,task_model):\n",
    "    joint_entropy=[]\n",
    "    with torch.no_grad():\n",
    "        for m in range(mtask+1):\n",
    "            task_model[m].eval()\n",
    "            outputs,_=task_model[m](inputs)\n",
    "\n",
    "            sout=F.softmax(outputs,1)\n",
    "            entropy=-torch.sum(sout*torch.log(sout+0.0001),1)\n",
    "            joint_entropy.append(entropy)\n",
    "\n",
    "\n",
    "    all_entropy=torch.zeros([joint_entropy[0].shape[0],mtask+1]).cuda()\n",
    "    for i in range(mtask+1):\n",
    "        all_entropy[:,i]=joint_entropy[i]\n",
    "    ctask=torch.argmin(all_entropy, axis=1)==dtask\n",
    "    correct=sum(ctask)\n",
    "    \n",
    "    return ctask, correct,all_entropy\n",
    "\n",
    "\n",
    "\n",
    "def inferecne(test_loader,dtask,mtask,task_model):\n",
    "    global best_acc\n",
    "    model=task_model[dtask].eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    cl_loss=0\n",
    "    tcorrect=0\n",
    "    accuracy=[]\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets) in enumerate(test_loader):\n",
    "            inputs, targets1 = inputs.cuda(), targets.cuda()\n",
    "            targets=targets1-dtask*args.class_per_task\n",
    "                        \n",
    "            if mtask>0:\n",
    "                correct_sample,Ncorrect,_=inference_check_task(dtask,mtask,inputs,task_model)\n",
    "                tcorrect+=Ncorrect\n",
    "                inputs=inputs[correct_sample]\n",
    "                targets=targets[correct_sample]\n",
    "                \n",
    "            if inputs.shape[0]!=0:\n",
    "                outputs,_ = model(inputs)\n",
    "                loss = criterion(outputs, targets)\n",
    "\n",
    "                test_loss += loss.item()\n",
    "                _, predicted = outputs.max(1)\n",
    "                correct += predicted.eq(targets).sum().item()\n",
    "            total += targets1.size(0)\n",
    "    if mtask>0:\n",
    "        taskC= tcorrect.item()/total\n",
    "    else:\n",
    "        taskC=1.0 \n",
    "    acc = 100.*correct/total\n",
    "    print(\"[Test Accuracy: %f], [Loss: %f] [Correct: %f]\" %(acc,test_loss/(batch_idx+1),taskC))\n",
    "    \n",
    "    return acc,taskC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# evaluating the ith task we will take the test tata from task 0:i\n",
    "# and evaluate over all the model till i since we dont know the task id\n",
    "# we have to evaluate all model till the ith and choose the task with minimum entropy\n",
    "\n",
    "cl_acc=[]\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "for mtask in range(args.num_task):\n",
    "    accuracy=[]\n",
    "    corr=[]\n",
    "    print('#######################################################')\n",
    "    ttask=mtask+1\n",
    "    for dtask in range(ttask):\n",
    "        print('Testing Task :---'+str(dtask))\n",
    "        test_loader = task_data[dtask][1]\n",
    "        acc,correct=inferecne(test_loader,dtask,mtask,task_model)\n",
    "        accuracy.append(acc)\n",
    "        corr.append(correct)\n",
    "    task_acc=np.mean(accuracy)\n",
    "    task_correct=np.mean(corr)\n",
    "    print('acc: '+str(task_acc)+'  Correct:  '+str(task_correct))\n",
    "    cl_acc.append(task_acc)\n",
    "print(np.array(cl_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}