resize-normalize-cloud-dataset.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport torch\nimport time\nimport os\nimport numpy as np\nfrom pathlib import Path\nfrom PIL import Image\nfrom skimage.transform import resize\nimport helper\nimport matplotlib.pyplot as plt\nfrom matplotlib import pyplot\nfrom matplotlib.image import imread\nfrom torchvision import datasets\nfrom torchvision import datasets, transforms, models\nfrom torch import nn, optim, Tensor\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nfrom torch.utils.data import DataLoader, Dataset, TensorDataset\nimport random\nseed=42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading paths\nbase_path = Path('../input/95cloud-cloud-segmentation-on-satellite-images/95-cloud_training_only_additional_to38-cloud')\nred_dir   = base_path/'train_red_additional_to38cloud'\nblue_dir  = base_path/'train_blue_additional_to38cloud'\ngreen_dir = base_path/'train_green_additional_to38cloud'\nnir_dir   = base_path/'train_nir_additional_to38cloud'\ngt_dir    = base_path/'train_gt_additional_to38cloud'\n\n# Saving paths\n\nos.mkdir('./TrueColor_imgs')\nos.mkdir('./FalseColor_imgs')\nos.mkdir('./GT')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RGB_CloudDataset (Dataset):\n    def __init__(self, red_dir, blue_dir, green_dir, gt_dir, transform= None):\n        \n        \n        self.transform   = transform\n    \n        \n        # Listing subdirectories\n        # Loop through the files in red folder  \n        # and combine, into a dictionary, the other bands\n        \n        self.files = [self.combine_files(f, green_dir, blue_dir, gt_dir) \n                      for f in red_dir.iterdir() if not f.is_dir()]\n        \n        \n        \n    def combine_files(self, red_file: Path, green_dir, blue_dir, gt_dir):\n        \n        files = {'red': red_file, \n                 'green':green_dir/red_file.name.replace('red', 'green'),\n                 'blue': blue_dir/red_file.name.replace('red', 'blue'), \n                 'gt': gt_dir/red_file.name.replace('red', 'gt')}\n\n        return files\n    \n\n    \n    \n    def OpenAsArray(self, idx):\n        \n        TrueColor = np.stack([np.array(Image.open(self.files[idx]['red'])),\n                              np.array(Image.open(self.files[idx]['green'])),\n                              np.array(Image.open(self.files[idx]['blue']))], axis = 2)\n     \n        TrueColor = TrueColor.transpose((2, 0, 1))\n    \n        return TrueColor \n    \n    \n    \n    def OpenMask(self, idx, add_dims=False):\n        raw_mask=np.array(Image.open(self.files[idx]['gt']))\n        raw_mask = np.where(raw_mask==255, 1, 0)\n        \n        return np.expand_dims(raw_mask, 0) if add_dims else raw_mask\n\n\n        \n    def __len__(self):\n        return len(self.files)\n    \n    \n    def __getitem__(self, idx):\n        x = self.OpenAsArray(idx)\n        y = self.OpenMask(idx, add_dims=False)\n        \n        \n        if self.transform is not None:\n            x, y = self.transform((x, y))\n        \n        \n        return x, y","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NirGB_CloudDataset (Dataset):\n    def __init__(self, red_dir, blue_dir, green_dir, nir_dir, gt_dir, transform = None):\n        \n        \n        self.transform = transform\n        \n        # Listing subdirectories\n        # Loop through the files in red folder  \n        # and combine, into a dictionary, the other bands\n        \n        self.files = [self.combine_files(f, green_dir, blue_dir, nir_dir, gt_dir) \n                      for f in red_dir.iterdir() if not f.is_dir()]\n        \n        \n        \n    def combine_files(self, red_file: Path, green_dir, blue_dir, nir_dir, gt_dir):\n        \n        files = {'red': red_file, \n                 'green':green_dir/red_file.name.replace('red', 'green'),\n                 'blue': blue_dir/red_file.name.replace('red', 'blue'), \n                 'nir': nir_dir/red_file.name.replace('red', 'nir'),\n                 'gt': gt_dir/red_file.name.replace('red', 'gt')}\n\n        return files\n    \n    \n    \n    def OpenAsArray(self, idx):\n        \n        FalseColor = np.stack([np.array(Image.open(self.files[idx]['nir'])),\n                               np.array(Image.open(self.files[idx]['green'])),\n                               np.array(Image.open(self.files[idx]['blue']))], axis = 2)\n     \n                    \n        FalseColor      = FalseColor.transpose((2, 0, 1))\n    \n        return FalseColor \n    \n    \n    \n    def OpenMask(self, idx, add_dims=False):\n        raw_mask=np.array(Image.open(self.files[idx]['gt']))\n        raw_mask = np.where(raw_mask==255, 1, 0)\n        \n        return np.expand_dims(raw_mask, 0) if add_dims else raw_mask\n\n\n        \n    def __len__(self):\n        return len(self.files)\n    \n    \n    def __getitem__(self, idx):\n        x = self.OpenAsArray(idx)\n        y = self.OpenMask(idx, add_dims=False)\n        return torch.from_numpy(x), torch.from_numpy(y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Resize(object):\n    def __init__(self, size = 256):\n        self.size = size\n    def __call__(self, sample):\n        x, y = sample\n        return (resize(x, (x.shape[0], self.size, self.size), mode = \"constant\", \n                      preserve_range = True, anti_aliasing = False),\n                resize(y, (self.size, self.size), mode = \"constant\", \n                      preserve_range = True, anti_aliasing = False))\n    \n\nclass Normalize(object):\n    def __init__(self, mean, std):\n        self.mean = mean\n        self.std  = std\n        \n    def __call__(self, sample):\n        x,y =sample\n        x = x.transpose(1,2,0)\n        x=(x-self.mean)/self.std\n        return x.transpose(2,0,1), y\n    \n    \n# TODO: Define transforms for the training data and testing data\ntrain_transforms=transforms.Compose([Resize(256),\n                                     Normalize([0.485, 0.456, 0.406], [0.229, 0.224,0.225])])\n\n                                 \ntest_transforms=transforms.Compose(Normalize([0.485, 0.456, 0.406], [0.229, 0.224,0.225]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NirGB=NirGB_CloudDataset(red_dir, blue_dir, green_dir, nir_dir, gt_dir, transform = None)\nfor i,d in enumerate(NirGB):\n    torch.save(d[0], './FalseColor_imgs/FasleColor_{}'.format(i))\n    torch.save(d[1], './GT/Mask_{}'.format(i))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data   = RGB_CloudDataset(red_dir, blue_dir, green_dir, gt_dir, transform = train_transforms)\nfor i,d in enumerate(data):\n    torch.save(d[0], './TrueColor_imgs/TrueColor_{}'.format(i))\n    torch.save(d[1], './GT/Mask_{}'.format(i))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_dir = sorted(os.listdir(base_path/'train_gt_additional_to38cloud'))\nidx = -1\nfor d in enumerate(data):\n    idx+=1\n    img_name = list_dir[idx]\n    new_img_name = img_name.replace('gt', 'TrueColor')\n    print(img_name)\n    print( './TrueColor_imgs/{}'.format(new_img_name))\n    print( './GT/{}'.format(img_name))\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}