unet-augmentations-bceweighted.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport wandb\nimport time\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport torchvision\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport albumentations as A\nimport torch.optim as optim\nfrom PIL import Image\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path\nfrom albumentations.pytorch import ToTensorV2\nimport torch.nn.functional as F\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:50:02.294989Z","iopub.execute_input":"2021-05-23T17:50:02.29531Z","iopub.status.idle":"2021-05-23T17:50:02.30223Z","shell.execute_reply.started":"2021-05-23T17:50:02.295281Z","shell.execute_reply":"2021-05-23T17:50:02.300321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.manual_seed(42)\n#42? You know why ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_height=384\noriginal_width =384\n\ntransformations = A.Compose([\n    \n    A.OneOf([\n        A.RandomSizedCrop(min_max_height=(50, 101), height=original_height, width=original_width, p=0.5),\n        A.PadIfNeeded(min_height=original_height, min_width=original_width, p=0.5)\n    ], p=1),    \n    A.VerticalFlip(p=0.5),              \n    A.RandomRotate90(p=0.5),\n    A.OneOf([\n        A.ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03, p=0.5),\n        A.GridDistortion(p=0.5),\n        A.OpticalDistortion(distort_limit=2, shift_limit=0.5, p=1)                  \n        ], p=0.8),  \n    A.RandomGamma(p=0.8),\n    ])\n#Highly Inspired from the Original shared notebook : 38-Cloud-Data preparation by cordmau\nclass Cloud(Dataset):\n    def __init__(self, r_dir, g_dir, b_dir, nir_dir, gt_dir, pytorch=True,transform=False):\n        super().__init__()\n        \n        # Loop through the files in red folder and combine, into a dictionary, the other bands\n        self.files = [self.combine_files(f, g_dir, b_dir, nir_dir, gt_dir) for f in r_dir.iterdir() if not f.is_dir()]\n        self.pytorch = pytorch\n        self.transform = transform\n        \n    def combine_files(self, r_file: Path, g_dir, b_dir,nir_dir, gt_dir):\n        \n        files = {'red': r_file, \n                 'green':g_dir/r_file.name.replace('red', 'green'),\n                 'blue': b_dir/r_file.name.replace('red', 'blue'), \n                 'nir': nir_dir/r_file.name.replace('red', 'nir'),\n                 'gt': gt_dir/r_file.name.replace('red', 'gt')}\n\n        return files\n                                       \n    def __len__(self):\n        \n        return len(self.files)\n     \n    def open_as_array(self, idx, invert=False, include_nir=True):\n\n        raw_rgb = np.stack([np.array(Image.open(self.files[idx]['red'])),\n                            np.array(Image.open(self.files[idx]['green'])),\n                            np.array(Image.open(self.files[idx]['blue'])),\n                           ], axis=2)\n    \n        if include_nir:\n            nir = np.expand_dims(np.array(Image.open(self.files[idx]['nir'])), 2)\n            raw_rgb = np.concatenate([nir,raw_rgb], axis=2)\n    \n        if invert:\n            raw_rgb = raw_rgb.transpose((2,0,1))\n    \n        # normalize\n        return (raw_rgb / np.iinfo(raw_rgb.dtype).max)\n    \n\n    def open_mask(self, idx, add_dims=False):\n        \n        raw_mask = np.array(Image.open(self.files[idx]['gt']))\n        raw_mask = np.where(raw_mask==255, 1, 0)\n        \n        return np.expand_dims(raw_mask, 0) if add_dims else raw_mask\n    \n    def __getitem__(self, idx):\n                    \n        x = self.open_as_array(idx, invert=False, include_nir=True)\n        \n        y = self.open_mask(idx, add_dims=False)\n       \n        if self.transform is not None:\n            augmented = self.transform(image=x,mask=y)\n            x=augmented['image']\n            y=augmented['mask']\n            \n        x=torch.from_numpy(x)\n        y=torch.tensor(y,dtype=torch.float32) \n        x = x.permute(2,0,1)\n        \n        \n        return x.double(), y.unsqueeze(0)\n    \n    def open_as_pil(self, idx):\n        \n        arr = 256*self.open_as_array(idx)\n        \n        return Image.fromarray(arr.astype(np.uint8), 'RGB')\n    \n    def __repr__(self):\n        s = 'Dataset class with {} files'.format(self.__len__())\n\n        return s","metadata":{"execution":{"iopub.status.busy":"2021-05-23T19:20:15.28567Z","iopub.execute_input":"2021-05-23T19:20:15.286024Z","iopub.status.idle":"2021-05-23T19:20:15.306257Z","shell.execute_reply.started":"2021-05-23T19:20:15.285993Z","shell.execute_reply":"2021-05-23T19:20:15.305294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#MY OWN VERSION BUT IT IS TO A BIT SLOW\noriginal_height=384\noriginal_width =384\n\ntransformed = A.Compose([\n    A.OneOf([\n        A.RandomSizedCrop(min_max_height=(50, 101), height=original_height, width=original_width, p=0.5),\n        A.PadIfNeeded(min_height=original_height, min_width=original_width, p=0.5)\n    ], p=1),    \n    A.VerticalFlip(p=0.5),              \n    A.RandomRotate90(p=0.5),\n    A.OneOf([\n        A.ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03, p=0.5),\n        A.GridDistortion(p=0.5),\n        A.OpticalDistortion(distort_limit=2, shift_limit=0.5, p=1)                  \n        ], p=0.8),\n    #A.CLAHE(p=0.8),\n    #A.RandomBrightnessContrast(p=0.8),    \n    A.RandomGamma(p=0.8),\n    ])\n\ndef get_channels(path):\n    red_ch=os.listdir(path+'train_red')\n    blue_ch=os.listdir(path+'train_blue')\n    green_ch=os.listdir(path+'train_green')\n    nir_ch=os.listdir(path+'train_nir')\n    masks=os.listdir(path+'train_gt')\n    return red_ch,blue_ch,green_ch,nir_ch,masks\n\n#f=transforms.ToTensor()\ndef f(image):\n    return np.array(image)\n\ndef get_image(path,r,b,g,n,m):\n    red=f(Image.open(path+'train_red/'+r))\n    blue=f(Image.open(path+'train_blue/'+b))\n    green=f(Image.open(path+'train_green/'+g))\n    nir=f(Image.open(path+'train_nir/'+n))\n    mask=f(Image.open(path+'train_gt/'+m))\n    return red,blue,green,nir,mask\n\npath='../input/38cloud-cloud-segmentation-in-satellite-images/38-Cloud_training/'\n\nclass Clouds(Dataset):\n    \n    def __init__(self,path,transform=None,num_channels=4):\n        \n        self.path=path\n        self.transform=transform\n        self.channels=get_channels(path)\n        self.num_channels=num_channels\n       \n    def process(self,r,g,b,n,m):\n        r,g,b,n,m=get_image(path,r,g,b,n,m)\n        image = np.stack([n,r,g],axis=2)\n        mask=m\n        return image,mask\n        \n    def __getitem__(self,idx):\n        \n        r=self.channels[0][idx]\n        g=self.channels[1][idx]\n        b=self.channels[2][idx]\n        n=self.channels[3][idx]\n        m=self.channels[4][idx]\n        \n        image,mask=self.process(r,g,b,n,m)\n        image = image.astype('uint16')\n        image = image / np.iinfo(image.dtype).max\n        mask  = mask/255.0\n       \n        if self.transform is not None:\n      \n            augmented = self.transform(image=image,mask=mask)\n            image=torch.from_numpy(augmented['image']).permute(2,0,1)\n            mask=torch.from_numpy(augmented['mask']) \n            \n        image=image.double()\n        mask=mask.unsqueeze(0)\n        mask = mask.type(torch.float32)\n        return image,mask\n    \n    def __len__(self):\n        return len(self.channels[0])\n       \n\n#dataset=Clouds(path,transform=transformed)\n#image,mask=dataset[2]\n#plt.imshow(image.permute(1,2,0))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbase_path = Path('../input/38cloud-cloud-segmentation-in-satellite-images/38-Cloud_training')\ndataset = Clouds(base_path/'train_red', \n                    base_path/'train_green', \n                    base_path/'train_blue', \n                    base_path/'train_nir',\n                    base_path/'train_gt',transform=transformations)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-23T19:20:16.609464Z","iopub.execute_input":"2021-05-23T19:20:16.609769Z","iopub.status.idle":"2021-05-23T19:20:27.885488Z","shell.execute_reply.started":"2021-05-23T19:20:16.609739Z","shell.execute_reply":"2021-05-23T19:20:27.884612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image,mask=dataset[100]\nimage.shape,mask.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-23T19:21:07.1056Z","iopub.execute_input":"2021-05-23T19:21:07.106036Z","iopub.status.idle":"2021-05-23T19:21:07.170907Z","shell.execute_reply.started":"2021-05-23T19:21:07.105996Z","shell.execute_reply":"2021-05-23T19:21:07.170092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_length=int(0.712* len(dataset))\n\ntest_length=len(dataset)-train_length\n\n\ntrain_dataset,test_dataset=torch.utils.data.random_split(dataset,(train_length,test_lengt))\n\nbatch_size= 12\n\ntrainloader = DataLoader(train_dataset,\n        batch_size=batch_size, shuffle=True,num_workers= 2)\ntestloader = DataLoader(test_dataset,\n        batch_size=batch_size, shuffle=False,num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:56:36.395033Z","iopub.execute_input":"2021-05-23T17:56:36.395347Z","iopub.status.idle":"2021-05-23T17:56:36.402014Z","shell.execute_reply.started":"2021-05-23T17:56:36.395317Z","shell.execute_reply":"2021-05-23T17:56:36.401088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Metrics(inputs, targets):\n    \n    inputs = torch.sigmoid(inputs)\n    inputs=inputs.round().int()\n    targets=targets.int()\n    smooth = 1.0\n    inputs = inputs.view(-1)\n    targets = targets.view(-1)\n    intersection = (inputs & targets).float().sum()\n    TP=intersection#Both are same\n    FP = ((1-targets) & inputs).float().sum()\n    FN = (targets & (1-inputs)).float().sum()\n    TN = ((1-targets) & (1-inputs)).float().sum()\n    total = (inputs + targets).float().sum()\n    union = total - intersection \n    IoU = ((intersection + smooth)/(union + smooth))\n    dice = (2.0 * intersection + smooth) / (total + smooth)\n    IoU = ((intersection + smooth)/(union + smooth))\n    valid = (targets >= 0)\n    acc_sum = (valid * (inputs == targets)).sum()\n    valid_sum = valid.sum()\n    #acc2=(inputs.argmax(dim=1) == targets.float().mean()\n    acc = (float(acc_sum) / (valid_sum + 1e-10))\n    precision = (intersection/(FP+intersection+1e-5))\n    recall = (intersection/(FN+intersection+1e-5))\n    specificity = (TN/(TN+FP+1e-5))\n    metrics={'IoU':IoU,'Dice':dice, 'Pixel_Acc': acc, 'Precision': precision,'Recall': recall, 'Specificity':specificity}\n    return metrics\n","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:50:12.449486Z","iopub.execute_input":"2021-05-23T17:50:12.449904Z","iopub.status.idle":"2021-05-23T17:50:12.463288Z","shell.execute_reply.started":"2021-05-23T17:50:12.449861Z","shell.execute_reply":"2021-05-23T17:50:12.462523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(network,criterion, optimizer, trainloader):\n    loss_train = 0\n    acc_train = 0\n    network.train()\n    \n    for step in tqdm(range(len(trainloader))):\n\n        images , masks = next(iter(trainloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n      \n\n        pred = network(images)\n        \n      \n        # clear all the gradients before calculating them\n        optimizer.zero_grad()\n        #v_pix=val_metrics['Pixel_Acc']\n        #val_dice = val_metrics['Dice']\n        # find the loss for the current step\n        loss_train_step = criterion(pred , masks)\n        \n        # find accuracy\n        acc_train_ = Metrics(pred,masks)\n        acc_train_step=acc_train_['IoU']\n        # calculate the gradients\n        loss_train_step.backward()\n        \n        # update the parameters\n        optimizer.step()\n        \n        loss_train += loss_train_step.item()\n        acc_train += acc_train_step  \n            \n    loss_train /= len(trainloader)\n    acc_train /= len(trainloader)\n    #gtrain_dice/= len(testloader)\n    #gtrain_pix/= len(testloader)\n    #print(pred.max(),pred.min(),masks.max(),masks.min())\n    return loss_train, acc_train,acc_train_  \n        \ndef validate(network,criterion, testloader): \n    loss_valid = 0\n    acc_valid = 0\n    gval_dice=0\n    gval_pix=0\n    network.eval()  \n\n    for step in tqdm(range(len(testloader))):\n\n        images , masks = next(iter(testloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        \n        pred = network(images)\n        #pred=torch.sigmoid(pred)\n        \n      \n        # clear all the gradients before calculating them\n        #optimizer.zero_grad()\n        \n        # find the loss and acc for the current step\n        \n        loss_valid_step = criterion(pred , masks)\n        \n        # find accuracy\n        val_metrics=Metrics(pred,masks)\n        acc_valid_step=val_metrics['IoU']\n        \n        \n        acc_val = val_metrics\n       # acc_valid_step=acc_val['IoU']\n        loss_valid += loss_valid_step.item()\n        acc_valid += acc_valid_step\n        \n        \n        #print(loss_tvalid_step,masks.shape)\n        #print(acc_val)\n\n    loss_valid /= len(testloader)\n    acc_valid /= len(testloader)\n  \n    return loss_valid, acc_valid,acc_val","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install segmentation_models_pytorch \nimport segmentation_models_pytorch as smp\n\nmodel = smp.Unet(\n    encoder_name=\"resnet34\",        # choose encoder, e.g. mobilenet_v2 or efficientnet-b7,     # use `imagenet` pre-trained weights for encoder initialization\n    in_channels=4,                  # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n    classes=1,                      # model output channels (number of classes in your dataset)\n)\n\n#mask=torch.randn(4,1,384,384)\n#target=torch.randn(4,1,384,384)\n\n\n#z=m(target,mask)\ndevice = \"cpu\"\n\nif torch.cuda.is_available():\n    device = \"cuda\"\n\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:50:12.464579Z","iopub.execute_input":"2021-05-23T17:50:12.465147Z","iopub.status.idle":"2021-05-23T17:50:30.106515Z","shell.execute_reply.started":"2021-05-23T17:50:12.465112Z","shell.execute_reply":"2021-05-23T17:50:30.105471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.init(name='Clouds', \n           project='UNetResnet+WBCE+Augs',\n           notes='RGBNIR', \n           entity='creganstark')\n\n# WandB Configurations (optional)        \nwandb.config.lr = 7e-3\n\nmodel=model.double()\nmodel = model.to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\n#criterion = FocalLoss()\noptimizer = optim.Adam(model.parameters(),lr = wandb.config.lr)\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n  \nnum_epochs = 20\nstart_time = time.time()\nprev_acc=0\nfor epoch in range(1, num_epochs+1):\n    \n    loss_train, acc_train,metric_train = train(model, criterion, optimizer, trainloader)\n    loss_valid, acc_valid,metric_val = validate(model, criterion, testloader)\n    \n    print('Epoch: {}  Train Loss: {:.4f}  Train IoU: {:.4f}  Valid Loss: {:.4f}  Valid IoU: {:.4f}'.\n          format(epoch, loss_train, acc_train, loss_valid, acc_valid))\n    print('Train Metrics',metric_train)\n    print('Val. Metrics',metric_val)\n\n    # Log the loss and accuracy values at the end of each epoch\n    wandb.log({\n        \"Epoch\": epoch,\n        \"Train IoU\": metric_train['IoU'],\n        \"Train Dice\": metric_train['Dice'],\n        \"Train Pixel Acc\": metric_train['Pixel_Acc'],\n        \"Train Precision\":metric_train[\"Precision\"],\n        \"Train Recall\": metric_train['Recall'],\n        \"Train Specificity\":metric_train['Specificity'],\n        \"Train Loss\": loss_train,\n        \"Val IoU\": metric_val['IoU'],\n        \"Val Dice \": metric_val['Dice'],\n        \"Val Pixel Acc\": metric_val['Pixel_Acc'],\n        \"Val Precision\":metric_val[\"Precision\"],\n        \"Val Recall\": metric_val['Recall'],\n        \"Val Specificity\":metric_val['Specificity'],\n        \"Val Loss\" : loss_valid\n       })\n    if acc_train>prev_acc:\n      prev_acc=acc_train\n      paths = \"model\"+str(acc_valid)+\".pt\"\n      print('Saving Model')\n      torch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))\n\nprint(\"Time Elapsed : {:.4f}s\".format(time.time() - start_time))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading last chekpoints\nfilename='../input/finalbce/finalfullbce.pt'\n\ndef load_checkpoint(model,filepath):\n    checkpoint = torch.load(filepath)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    #for parameter in model.parameters():\n        #parameter.requires_grad = True\n    return model\nmodel=load_checkpoint(model,filename)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:50:30.110414Z","iopub.execute_input":"2021-05-23T17:50:30.110761Z","iopub.status.idle":"2021-05-23T17:50:39.633444Z","shell.execute_reply.started":"2021-05-23T17:50:30.110725Z","shell.execute_reply":"2021-05-23T17:50:39.632497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inference","metadata":{}},{"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\nmodel=model.double()\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:51:01.893306Z","iopub.execute_input":"2021-05-23T17:51:01.893639Z","iopub.status.idle":"2021-05-23T17:51:02.067151Z","shell.execute_reply.started":"2021-05-23T17:51:01.893607Z","shell.execute_reply":"2021-05-23T17:51:02.066235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Metrics(inputs, targets):\n    \n    inputs = torch.sigmoid(inputs)\n    inputs=inputs.round().int()\n    targets=targets.int()\n    smooth = 1.0\n    inputs = inputs.view(-1)\n    targets = targets.view(-1)\n    intersection = (inputs & targets).float().sum()\n    TP=intersection\n    FP = ((1-targets) & inputs).float().sum()\n    FN = (targets & (1-inputs)).float().sum()\n    TN = ((1-targets) & (1-inputs)).float().sum()\n    total = (inputs + targets).float().sum()\n    union = total - intersection \n    IoU = ((intersection + smooth)/(union + smooth))\n    # 'Dice':dice, 'Pixel_Acc': acc, 'Precision': precision,'Recall': recall, 'Specificity':specificity}\n    #return metrics\n    \n    dice = (2.0 * intersection + smooth) / (total + smooth)\n    IoU = ((intersection + smooth)/(union + smooth))\n    valid = (targets >= 0)\n    acc_sum = (valid * (inputs == targets)).sum()\n    valid_sum = valid.sum()\n    #acc2=(inputs.argmax(dim=1) == targets.float().mean()\n    acc = (float(acc_sum) / (valid_sum + 1e-10))\n    precision = (intersection/(FP+intersection+1e-5))\n    recall = (intersection/(FN+intersection+1e-5))\n    specificity = (TN/(TN+FP+1e-5))\n    metrics={'IoU':IoU,'Dice':dice, 'Pixel_Acc': acc, 'Precision': precision,'Recall': recall, 'Specificity':specificity}\n    return metrics\ndef validate2(network,criterion, testloader): \n    \n    #train_dataset,test_dataset=torch.utils.data.random_split(dataset,(6000,2400))\n    \n\n    #batch_size= 12\n\n    \n   # testloader = DataLoader(test_dataset,\n        #batch_size=batch_size, shuffle=False,num_workers=2)\n    \n    \n    loss_valid = 0\n    acc_valid = 0    \n    acc_pix = 0\n    acc_dice=0\n    precision=0\n    recall=0\n    specificity=0\n    network.eval()  \n\n    for step in tqdm(range(len(testloader))):\n\n        images , masks = next(iter(testloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        \n        pred = network(images)\n        \n        \n        loss_valid_step = criterion(pred , masks)\n       \n        val_metrics=Metrics(pred,masks)\n        \n        #acc_val = val_metrics\n        acc_valid_step=val_metrics['IoU']\n        acc_valid_dice=val_metrics['Dice']\n        acc_valid_pixel=val_metrics['Pixel_Acc']\n        acc_precision=val_metrics['Precision']\n        acc_recall = val_metrics['Recall']\n        acc_speicificity = val_metrics['Specificity']\n        precision+=acc_precision\n        recall+=acc_recall\n        specificity+=acc_speicificity\n        loss_valid += loss_valid_step.item()\n        acc_valid += acc_valid_step\n        acc_pix+=acc_valid_pixel\n        acc_dice+=acc_valid_dice\n        \n        \n        #print(loss_tvalid_step,masks.shape)\n        #print(acc_val)\n\n    loss_valid /= len(testloader)\n    acc_valid /= len(testloader)\n    acc_pix/= len(testloader)\n    acc_dice/= len(testloader) \n    precision/=len(testloader) \n    recall/=len(testloader) \n    specificity/=len(testloader) \n    \n\n    return loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:51:10.183819Z","iopub.execute_input":"2021-05-23T17:51:10.184167Z","iopub.status.idle":"2021-05-23T17:51:10.199041Z","shell.execute_reply.started":"2021-05-23T17:51:10.184137Z","shell.execute_reply":"2021-05-23T17:51:10.197953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity=validate2(model,criterion, testloader)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:56:43.647919Z","iopub.execute_input":"2021-05-23T17:56:43.648239Z","iopub.status.idle":"2021-05-23T18:00:16.278065Z","shell.execute_reply.started":"2021-05-23T17:56:43.648212Z","shell.execute_reply":"2021-05-23T18:00:16.277174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity","metadata":{"execution":{"iopub.status.busy":"2021-05-23T17:56:03.86269Z","iopub.execute_input":"2021-05-23T17:56:03.863053Z","iopub.status.idle":"2021-05-23T17:56:03.895123Z","shell.execute_reply.started":"2021-05-23T17:56:03.863015Z","shell.execute_reply":"2021-05-23T17:56:03.894185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity","metadata":{"execution":{"iopub.status.busy":"2021-05-23T18:03:03.995423Z","iopub.execute_input":"2021-05-23T18:03:03.995775Z","iopub.status.idle":"2021-05-23T18:03:04.008911Z","shell.execute_reply.started":"2021-05-23T18:03:03.995737Z","shell.execute_reply":"2021-05-23T18:03:04.007963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Clutter","metadata":{}},{"cell_type":"code","source":"def train(network,criterion, optimizer, trainloader):\n    loss_train = 0\n    acc_train = 0\n    network.train()\n    \n    for step in tqdm(range(len(trainloader))):\n\n        images , masks = next(iter(trainloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n      \n\n        pred = network(images)\n        \n      \n        # clear all the gradients before calculating them\n        optimizer.zero_grad()\n        #v_pix=val_metrics['Pixel_Acc']\n        #val_dice = val_metrics['Dice']\n        # find the loss for the current step\n        loss_train_step = criterion(pred , masks)\n        \n        # find accuracy\n        acc_train_ = Metrics(pred,masks)\n        acc_train_step=acc_train_['IoU']\n        # calculate the gradients\n        loss_train_step.backward()\n        \n        # update the parameters\n        optimizer.step()\n        \n        loss_train += loss_train_step.item()\n        acc_train += acc_train_step  \n            \n    loss_train /= len(trainloader)\n    acc_train /= len(trainloader)\n    #gtrain_dice/= len(testloader)\n    #gtrain_pix/= len(testloader)\n    #print(pred.max(),pred.min(),masks.max(),masks.min())\n    return loss_train, acc_train,acc_train_  \n        \ndef validate(network,criterion, testloader): \n    loss_valid = 0\n    acc_valid = 0\n    gval_dice=0\n    gval_pix=0\n    network.eval()  \n\n    for step in tqdm(range(len(testloader))):\n\n        images , masks = next(iter(testloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        \n        pred = network(images)\n        #pred=torch.sigmoid(pred)\n        \n      \n        # clear all the gradients before calculating them\n        #optimizer.zero_grad()\n        \n        # find the loss and acc for the current step\n        \n        loss_valid_step = criterion(pred , masks)\n        \n        # find accuracy\n        val_metrics=Metrics(pred,masks)\n        acc_valid_step=val_metrics['IoU']\n        #val_pix=val_metrics['Pixel_Acc']\n        #val_dice = val_metrics['Dice']\n        # calculate the gradients\n        #loss_train_step.backward()\n        #print(loss_train_step,masks.shape)\n        #print(acc_train_)\n        # update the parameters\n        #optimizer.step()\n        \n        acc_val = val_metrics\n       # acc_valid_step=acc_val['IoU']\n        loss_valid += loss_valid_step.item()\n        acc_valid += acc_valid_step\n        \n        \n        #print(loss_tvalid_step,masks.shape)\n        #print(acc_val)\n\n    loss_valid /= len(testloader)\n    acc_valid /= len(testloader)\n    #gval_dice/= len(testloader)\n    #gval_pix/= len(testloader)\n    #print(pred.max(),pred.min(),masks.max(),masks.min())\n    return loss_valid, acc_valid,acc_val","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"     \nlr = 1e-2\nmodel=model.double()\n\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\n#criterion = FocalLoss()\noptimizer = optim.Adam(model.parameters(),lr  =  lr)# ,\n\nnum_epochs = 2\nstart_time = time.time()\nprev_acc=0\nfor epoch in range(1, num_epochs+1):\n    \n    loss_train, acc_train,metric_train = train(model, criterion, optimizer, trainloader)\n    loss_valid, acc_valid,metric_val = validate(model, criterion, testloader)\n    \n    print('Epoch: {}  Train Loss: {:.4f}  Train IoU: {:.4f}  Valid Loss: {:.4f}  Valid IoU: {:.4f}'.\n          format(epoch, loss_train, acc_train, loss_valid, acc_valid))\n    \n    if acc_train>prev_acc:\n      prev_acc=acc_train\n      paths = \"model\"+str(acc_train)+\".pt\"\n      print('Saving Model')\n      torch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))\n\nprint(\"Time Elapsed : {:.4f}s\".format(time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T15:57:12.546546Z","iopub.execute_input":"2021-05-23T15:57:12.546906Z","iopub.status.idle":"2021-05-23T16:34:53.813799Z","shell.execute_reply.started":"2021-05-23T15:57:12.54687Z","shell.execute_reply":"2021-05-23T16:34:53.810914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp semifinalfullbce2.pt ../input/checkpoint","metadata":{"execution":{"iopub.status.busy":"2021-05-23T15:40:02.631123Z","iopub.execute_input":"2021-05-23T15:40:02.63145Z","iopub.status.idle":"2021-05-23T15:40:03.303249Z","shell.execute_reply.started":"2021-05-23T15:40:02.631418Z","shell.execute_reply":"2021-05-23T15:40:03.302337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths='finalfullbce.pt'\ntorch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          \n          }, str(paths))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:35:07.957964Z","iopub.execute_input":"2021-05-23T16:35:07.958301Z","iopub.status.idle":"2021-05-23T16:35:09.129562Z","shell.execute_reply.started":"2021-05-23T16:35:07.958264Z","shell.execute_reply":"2021-05-23T16:35:09.128742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Metrics(inputs, targets):\n    \n    inputs = torch.sigmoid(inputs)\n    inputs=inputs.round().int()\n    targets=targets.int()\n    smooth = 1.0\n    inputs = inputs.view(-1)\n    targets = targets.view(-1)\n    intersection = (inputs & targets).float().sum()\n    TP=intersection\n    FP = ((1-targets) & inputs).float().sum()\n    FN = (targets & (1-inputs)).float().sum()\n    TN = ((1-targets) & (1-inputs)).float().sum()\n    total = (inputs + targets).float().sum()\n    union = total - intersection \n    IoU = ((intersection + smooth)/(union + smooth))\n    # 'Dice':dice, 'Pixel_Acc': acc, 'Precision': precision,'Recall': recall, 'Specificity':specificity}\n    #return metrics\n    \n    dice = (2.0 * intersection + smooth) / (total + smooth)\n    IoU = ((intersection + smooth)/(union + smooth))\n    valid = (targets >= 0)\n    acc_sum = (valid * (inputs == targets)).sum()\n    valid_sum = valid.sum()\n    #acc2=(inputs.argmax(dim=1) == targets.float().mean()\n    acc = (float(acc_sum) / (valid_sum + 1e-10))\n    precision = (intersection/(FP+intersection+1e-5))\n    recall = (intersection/(FN+intersection+1e-5))\n    specificity = (TN/(TN+FP+1e-5))\n    metrics={'IoU':IoU,'Dice':dice, 'Pixel_Acc': acc, 'Precision': precision,'Recall': recall, 'Specificity':specificity}\n    return metrics\ndef validate2(network,criterion, testloader): \n    model = model.to(device)\n    #train_dataset,test_dataset=torch.utils.data.random_split(dataset,(6000,2400))\n    \n\n    #batch_size= 12\n\n    \n   # testloader = DataLoader(test_dataset,\n        #batch_size=batch_size, shuffle=False,num_workers=2)\n    \n    \n    loss_valid = 0\n    acc_valid = 0    \n    acc_pix = 0\n    acc_dice=0\n    precision=0\n    recall=0\n    specificity=0\n    network.eval()  \n\n    for step in tqdm(range(len(testloader))):\n\n        images , masks = next(iter(testloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        \n        pred = network(images)\n        \n        \n        loss_valid_step = criterion(pred , masks)\n       \n        val_metrics=Metrics(pred,masks)\n        \n        #acc_val = val_metrics\n        acc_valid_step=val_metrics['IoU']\n        acc_valid_dice=val_metrics['Dice']\n        acc_valid_pixel=val_metrics['Pixel_Acc']\n        acc_precision=val_metrics['Precision']\n        acc_recall = val_metrics['Recall']\n        acc_speicificity = val_metrics['Specificity']\n        precision+=acc_precision\n        recall+=acc_recall\n        specificity+=acc_speicificity\n        loss_valid += loss_valid_step.item()\n        acc_valid += acc_valid_step\n        acc_pix+=acc_valid_pixel\n        acc_dice+=acc_valid_dice\n        \n        \n        #print(loss_tvalid_step,masks.shape)\n        #print(acc_val)\n\n    loss_valid /= len(testloader)\n    acc_valid /= len(testloader)\n    acc_pix/= len(testloader)\n    acc_dice/= len(testloader) \n    precision/=len(testloader) \n    recall/=len(testloader) \n    specificty/=len(testloader) \n    \n\n    return loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:39:06.044622Z","iopub.execute_input":"2021-05-23T16:39:06.045002Z","iopub.status.idle":"2021-05-23T16:39:06.055895Z","shell.execute_reply.started":"2021-05-23T16:39:06.044963Z","shell.execute_reply":"2021-05-23T16:39:06.05487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid,acc_valid,acc_pix,acc_dice,precision,recall,specificity=validate2(model,criterion, testloader)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:39:08.181813Z","iopub.execute_input":"2021-05-23T16:39:08.18216Z","iopub.status.idle":"2021-05-23T16:39:11.109705Z","shell.execute_reply.started":"2021-05-23T16:39:08.182129Z","shell.execute_reply":"2021-05-23T16:39:11.107123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#wandb.init(name='Clouds', \n           #project='UNetResnet+WBCE+Augs',\n           #notes='RGBNIR', \n           #tags=['Replay-Attack','Cyclic_LR'],\n           #entity='creganstark')\n\n# WandB Configurations (optional)        \nlr = 7e-3\n#model=model.float()\n#model=model.double()\n#model = model.to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\n#criterion = FocalLoss()\noptimizer = optim.Adam(model.parameters(),lr  =  lr)# ,\n                      #momentum     = 0.9,\n                      #nesterov     = True,\n                      #weight_decay = 5e-4)\n\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n  \n\n\nnum_epochs = 20\nstart_time = time.time()\nprev_acc=0\nfor epoch in range(1, num_epochs+1):\n    \n    loss_train, acc_train,metric_train = train(model, criterion, optimizer, trainloader)\n    loss_valid, acc_valid,metric_val = validate(model, criterion, testloader)\n    \n    print('Epoch: {}  Train Loss: {:.4f}  Train IoU: {:.4f}  Valid Loss: {:.4f}  Valid IoU: {:.4f}'.\n          format(epoch, loss_train, acc_train, loss_valid, acc_valid))\n    print('Train Metrics',metric_train)\n    print('Val. Metrics',metric_val)\n\n    # Log the loss and accuracy values at the end of each epoch\n    wandb.log({\n        \"Epoch\": epoch,\n        \"Train IoU\": metric_train['IoU'],\n        \"Train Dice\": metric_train['Dice'],\n        \"Train Pixel Acc\": metric_train['Pixel_Acc'],\n        \"Train Precision\":metric_train[\"Precision\"],\n        \"Train Recall\": metric_train['Recall'],\n        \"Train Specificity\":metric_train['Specificity'],\n        \"Train Loss\": loss_train,\n        \"Val IoU\": metric_val['IoU'],\n        \"Val Dice \": metric_val['Dice'],\n        \"Val Pixel Acc\": metric_val['Pixel_Acc'],\n        \"Val Precision\":metric_val[\"Precision\"],\n        \"Val Recall\": metric_val['Recall'],\n        \"Val Specificity\":metric_val['Specificity'],\n        \"Val Loss\" : loss_valid\n       })\n    if acc_train>prev_acc:\n      prev_acc=acc_train\n      paths = \"model\"+str(acc_valid)+\".pt\"\n      print('Saving Model')\n      torch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))\n\nprint(\"Time Elapsed : {:.4f}s\".format(time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T05:48:06.280537Z","iopub.execute_input":"2021-05-23T05:48:06.280854Z","iopub.status.idle":"2021-05-23T09:24:31.265612Z","shell.execute_reply.started":"2021-05-23T05:48:06.280824Z","shell.execute_reply":"2021-05-23T09:24:31.262926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(val_set)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:38:13.826756Z","iopub.execute_input":"2021-05-23T09:38:13.82707Z","iopub.status.idle":"2021-05-23T09:38:13.832244Z","shell.execute_reply.started":"2021-05-23T09:38:13.827042Z","shell.execute_reply":"2021-05-23T09:38:13.831411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#wandb.init(name='Clouds', \n           project='UNetResnet+WBCE+Augs',\n           notes='RGBNIR', \n           #tags=['Replay-Attack','Cyclic_LR'],\n           entity='creganstark')\n\n# WandB Configurations (optional)        \n#wandb.config.lr = 9e-3\n#model=model.float()\n#model=model.double()\n#model = model.to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\n#criterion = FocalLoss()\noptimizer = optim.Adam(model.parameters(),lr  =  7e-3)# ,\n#optmizer.load_state_dict(checkpoint['optimizer_state_dict'])\n#torch.load_checkpoint('optimizer_state_dict')\n                      #momentum     = 0.9,\n                      #nesterov     = True,\n                      #weight_decay = 5e-4)\n\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n  \n\n# Log the network weight histograms (optional)\n#wandb.watch(model)\n\nnum_epochs = 5\nstart_time = time.time()\nprev_acc=0\nfor epoch in range(1, num_epochs+1):\n    \n    loss_train, acc_train,metric_train = train(model, criterion, optimizer, trainloader)\n    loss_valid, acc_valid,metric_val = validate(model, criterion, testloader)\n    \n    print('Epoch: {}  Train Loss: {:.4f}  Train IoU: {:.4f}  Valid Loss: {:.4f}  Valid IoU: {:.4f}'.\n          format(epoch, loss_train, acc_train, loss_valid, acc_valid))\n    print('Train Metrics',metric_train)\n    print('Val. Metrics',metric_val)\n\n    # Log the loss and accuracy values at the end of each epoch\n    wandb.log({\n        \"Epoch\": epoch,\n        \"Train IoU\": metric_train['IoU'],\n        \"Train Dice\": metric_train['Dice'],\n        \"Train Pixel Acc\": metric_train['Pixel_Acc'],\n        \"Train Precision\":metric_train[\"Precision\"],\n        \"Train Recall\": metric_train['Recall'],\n        \"Train Specificity\":metric_train['Specificity'],\n        \"Train Loss\": loss_train,\n        \"Val IoU\": metric_val['IoU'],\n        \"Val Dice \": metric_val['Dice'],\n        \"Val Pixel Acc\": metric_val['Pixel_Acc'],\n        \"Val Precision\":metric_val[\"Precision\"],\n        \"Val Recall\": metric_val['Recall'],\n        \"Val Specificity\":metric_val['Specificity'],\n        \"Val Loss\" : loss_valid\n       })\n    if acc_valid>prev_acc:\n      prev_acc=acc_valid\n      paths = \"model\"+str(acc_valid)+\".pt\"\n      print('Saving Model')\n      torch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))\n\nprint(\"Time Elapsed : {:.4f}s\".format(time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:38:35.860969Z","iopub.execute_input":"2021-05-23T09:38:35.861287Z","iopub.status.idle":"2021-05-23T12:32:58.04643Z","shell.execute_reply.started":"2021-05-23T09:38:35.861259Z","shell.execute_reply":"2021-05-23T12:32:58.045639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#wandb.init(name='Clouds', \n           #project='UNetResnet+WBCE+Augs',\n           #notes='RGBNIR', \n           #tags=['Replay-Attack','Cyclic_LR'],\n           #entity='creganstark')\n\n# WandB Configurations (optional)        \n#wandb.config.lr = 9e-3\n#model=model.float()\n#model=model.double()\n#model = model.to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.3053))\n#criterion = FocalLoss()\noptimizer = optim.Adam(model.parameters(),lr  =  7e-3)# ,\n#optmizer.load_state_dict(checkpoint['optimizer_state_dict'])\n#torch.load_checkpoint('optimizer_state_dict')\n                      #momentum     = 0.9,\n                      #nesterov     = True,\n                      #weight_decay = 5e-4)\n\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n#scheduler = optim.lr_scheduler.CyclicLR(optimizer,base_lr=wandb.config.lr,max_lr=1e-3 ,step_size_up=2000)\n  \n\n# Log the network weight histograms (optional)\n#wandb.watch(model)\n\nnum_epochs = 1\nstart_time = time.time()\nprev_acc=0\nfor epoch in range(1, num_epochs+1):\n    \n    loss_train, acc_train,metric_train = train(model, criterion, optimizer, trainloader)\n    loss_valid, acc_valid,metric_val = validate(model, criterion, testloader)\n    \n    print('Epoch: {}  Train Loss: {:.4f}  Train IoU: {:.4f}  Valid Loss: {:.4f}  Valid IoU: {:.4f}'.\n          format(epoch, loss_train, acc_train, loss_valid, acc_valid))\n    print('Train Metrics',metric_train)\n    print('Val. Metrics',metric_val)\n\n    # Log the loss and accuracy values at the end of each epoch\n    wandb.log({\n        \"Epoch\": epoch,\n        \"Train IoU\": metric_train['IoU'],\n        \"Train Dice\": metric_train['Dice'],\n        \"Train Pixel Acc\": metric_train['Pixel_Acc'],\n        \"Train Precision\":metric_train[\"Precision\"],\n        \"Train Recall\": metric_train['Recall'],\n        \"Train Specificity\":metric_train['Specificity'],\n        \"Train Loss\": loss_train,\n        \"Val IoU\": metric_val['IoU'],\n        \"Val Dice \": metric_val['Dice'],\n        \"Val Pixel Acc\": metric_val['Pixel_Acc'],\n        \"Val Precision\":metric_val[\"Precision\"],\n        \"Val Recall\": metric_val['Recall'],\n        \"Val Specificity\":metric_val['Specificity'],\n        \"Val Loss\" : loss_valid\n       })\n    if acc_valid>prev_acc:\n      prev_acc=acc_valid\n      paths = \"model\"+str(acc_valid)+\".pt\"\n      print('Saving Model')\n      torch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))\n\nprint(\"Time Elapsed : {:.4f}s\".format(time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:56:36.63974Z","iopub.execute_input":"2021-05-23T13:56:36.640058Z","iopub.status.idle":"2021-05-23T14:28:57.543046Z","shell.execute_reply.started":"2021-05-23T13:56:36.640023Z","shell.execute_reply":"2021-05-23T14:28:57.542224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink(r'finalfullbce.pt')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:35:55.378618Z","iopub.execute_input":"2021-05-23T16:35:55.378974Z","iopub.status.idle":"2021-05-23T16:35:55.390962Z","shell.execute_reply.started":"2021-05-23T16:35:55.378941Z","shell.execute_reply":"2021-05-23T16:35:55.389382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths='semifinalfullbce.pt'\ntorch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:29:08.660759Z","iopub.execute_input":"2021-05-23T14:29:08.661098Z","iopub.status.idle":"2021-05-23T14:29:10.312287Z","shell.execute_reply.started":"2021-05-23T14:29:08.661065Z","shell.execute_reply":"2021-05-23T14:29:10.311424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink(r'semifinalfullbce.pt')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:29:32.597466Z","iopub.execute_input":"2021-05-23T14:29:32.597883Z","iopub.status.idle":"2021-05-23T14:29:32.610387Z","shell.execute_reply.started":"2021-05-23T14:29:32.59785Z","shell.execute_reply":"2021-05-23T14:29:32.609225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths='5epochsunet100.pt'\ntorch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:25:12.563208Z","iopub.execute_input":"2021-05-23T09:25:12.563567Z","iopub.status.idle":"2021-05-23T09:25:14.348829Z","shell.execute_reply.started":"2021-05-23T09:25:12.563535Z","shell.execute_reply":"2021-05-23T09:25:14.348042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = 'modeltensor(0.8623, device='+'-cuda'+':'+'0-'+').pt'\nfilename=filename.replace('-',\"'\")","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:28:01.40742Z","iopub.execute_input":"2021-05-23T09:28:01.40786Z","iopub.status.idle":"2021-05-23T09:28:01.413003Z","shell.execute_reply.started":"2021-05-23T09:28:01.407823Z","shell.execute_reply":"2021-05-23T09:28:01.412198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '../input/finalcheckpoint/semifinalfullbce2.pt'\ndef load_checkpoint(model,filepath):\n    checkpoint = torch.load(filepath)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    \n    #for parameter in model.parameters():\n        #parameter.requires_grad = True\n    return model\nmodel=load_checkpoint(model,filename)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T15:55:40.176988Z","iopub.execute_input":"2021-05-23T15:55:40.177334Z","iopub.status.idle":"2021-05-23T15:55:50.246608Z","shell.execute_reply.started":"2021-05-23T15:55:40.177303Z","shell.execute_reply":"2021-05-23T15:55:50.245736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.chdir(r'/kaggle/working')\nfrom IPython.display import FileLink\nFileLink(r'iou_0.7884.pt')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:49:41.307154Z","iopub.execute_input":"2021-05-23T13:49:41.307487Z","iopub.status.idle":"2021-05-23T13:49:41.319322Z","shell.execute_reply.started":"2021-05-23T13:49:41.307455Z","shell.execute_reply":"2021-05-23T13:49:41.318203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_length=int(0.7* len(dataset))\n\n#test_length=len(dataset)-train_length\n#val_length=350\n\ntrain_dataset,test_dataset=torch.utils.data.random_split(dataset,(train_length,test_length))\ntest_length=test_length-val_length\nval_set,_=torch.utils.data.random_split(test_dataset,(val_length,test_length))\nbatch_size= 12\n\ntrainloader = DataLoader(train_dataset,\n        batch_size=batch_size, shuffle=True,num_workers= 2)\ntestloader = DataLoader(val_set,\n        batch_size=batch_size, shuffle=False,num_workers=2)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:29:28.614361Z","iopub.execute_input":"2021-05-23T09:29:28.614713Z","iopub.status.idle":"2021-05-23T09:29:28.622429Z","shell.execute_reply.started":"2021-05-23T09:29:28.614684Z","shell.execute_reply":"2021-05-23T09:29:28.621315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate2(network,criterion, testloader): \n    train_dataset,test_dataset=torch.utils.data.random_split(dataset,(6800,1000))\n    \n\n    batch_size= 12\n\n    \n    testloader = DataLoader(test_dataset,\n        batch_size=batch_size, shuffle=False,num_workers=2)\n    \n    loss_valid = 0\n    acc_valid = 0    \n    acc_pix = 0\n    acc_dice=0\n    network.eval()  \n\n    for step in tqdm(range(len(testloader))):\n\n        images , masks = next(iter(testloader))\n        \n        # move the images and labels to GPU\n        images = images.to(device)\n        masks = masks.to(device)\n        \n        \n        pred = network(images)\n        \n        \n        loss_valid_step = criterion(pred , masks)\n       \n        val_metrics=Metrics(pred,masks)\n        \n        #acc_val = val_metrics\n        acc_valid_step=val_metrics['IoU']\n        acc_valid_dice=val_metrics['Dice']\n        acc_valid_pixel=val_metrics['Pixel_Acc']\n        loss_valid += loss_valid_step.item()\n        acc_valid += acc_valid_step\n        acc_pix+=acc_valid_pixel\n        acc_dice+=acc_valid_dice\n        \n        \n        #print(loss_tvalid_step,masks.shape)\n        #print(acc_val)\n\n    loss_valid /= len(testloader)\n    acc_valid /= len(testloader)\n    acc_pix/= len(testloader)\n    acc_dice/= len(testloader)   \n\n    \n\n    return loss_valid,acc_valid,acc_pix,acc_dice","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:37:36.397329Z","iopub.execute_input":"2021-05-23T16:37:36.397662Z","iopub.status.idle":"2021-05-23T16:37:36.405622Z","shell.execute_reply.started":"2021-05-23T16:37:36.397633Z","shell.execute_reply":"2021-05-23T16:37:36.404806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid, acc_valid,acc_pix,acc_dice = validate2(model,criterion, testloader)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:37:38.013535Z","iopub.execute_input":"2021-05-23T16:37:38.013891Z","iopub.status.idle":"2021-05-23T16:37:38.044486Z","shell.execute_reply.started":"2021-05-23T16:37:38.013854Z","shell.execute_reply":"2021-05-23T16:37:38.043005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid, acc_valid,acc_pix,acc_dice","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:17:50.794678Z","iopub.execute_input":"2021-05-23T13:17:50.795021Z","iopub.status.idle":"2021-05-23T13:17:50.809135Z","shell.execute_reply.started":"2021-05-23T13:17:50.794986Z","shell.execute_reply":"2021-05-23T13:17:50.808248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid, acc_valid,acc_pix,acc_dice","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:48:06.481448Z","iopub.execute_input":"2021-05-23T13:48:06.4818Z","iopub.status.idle":"2021-05-23T13:48:06.502996Z","shell.execute_reply.started":"2021-05-23T13:48:06.481766Z","shell.execute_reply":"2021-05-23T13:48:06.502115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_valid, acc_valid,acc_pix,acc_dice","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:33:50.041013Z","iopub.execute_input":"2021-05-23T13:33:50.041407Z","iopub.status.idle":"2021-05-23T13:33:50.057505Z","shell.execute_reply.started":"2021-05-23T13:33:50.04136Z","shell.execute_reply":"2021-05-23T13:33:50.056617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths='iou_0.7884.pt'\ntorch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:18:05.314288Z","iopub.execute_input":"2021-05-23T13:18:05.314631Z","iopub.status.idle":"2021-05-23T13:18:06.86207Z","shell.execute_reply.started":"2021-05-23T13:18:05.314602Z","shell.execute_reply":"2021-05-23T13:18:06.86131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filename = 'modeltensor(0.8413, device='cuda:0').pt\nfilename = 'modeltensor(0.7997, device='+'-cuda'+':'+'0-'+').pt'\nfilename=filename.replace('-',\"'\")\n#filename='epoch20checkpoint.pth'\n\ndef load_checkpoint(model,filepath):\n    checkpoint = torch.load(filepath)\n    model.load_state_dict(checkpoint['model_state_dict'])\n    #for parameter in model.parameters():\n        #parameter.requires_grad = True\n    return model\nmodel=load_checkpoint(model,filename)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T13:34:31.865716Z","iopub.execute_input":"2021-05-23T13:34:31.866049Z","iopub.status.idle":"2021-05-23T13:34:34.298036Z","shell.execute_reply.started":"2021-05-23T13:34:31.86602Z","shell.execute_reply":"2021-05-23T13:34:34.297129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_length=int(0.70* len(dataset))\n\ntest_length=len(dataset)-train_length\n\n\ntrain_dataset,test_dataset=torch.utils.data.random_split(dataset,(train_length,test_length))\n#val_length=300\n#test_length-=300\n#val_set,_=torch.utils.data.random_split(test_dataset,(val_length,test_length))\nbatch_size= 12\n\ntrainloader = DataLoader(train_dataset,\n        batch_size=batch_size, shuffle=True,num_workers= 2)\ntestloader = DataLoader(test_dataset,\n        batch_size=batch_size, shuffle=False,num_workers=2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths='iou_0.6.pt'\ntorch.save({\n          'epoch': epoch,\n          'model_state_dict': model.state_dict(),\n          'optimizer_state_dict': optimizer.state_dict(),\n          'val_loss': loss_valid,\n          'val_acc':acc_valid,\n          'train_acc':acc_train,\n          'loss_acc':loss_train,\n          }, str(paths))","metadata":{},"execution_count":null,"outputs":[]}]}