current_make_tfrecord_to_gdrive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8935d13b-43fc-4b4c-8cc9-d8275ff993c5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.16.0\n",
      "3825.0\n",
      "Starting job for year 2018\n",
      "337229\n",
      "337229\n",
      "['IRR_Method']\n",
      "156925\n",
      "1\n",
      "{'type': 'Image', 'bands': [{'id': 'Irrigation', 'data_type': {'type': 'PixelType', 'precision': 'double'}, 'crs': 'EPSG:4326', 'crs_transform': [1, 0, 0, 0, 1, 0]}]}\n",
      "['2001', '2004', '2006', '2008', '2011', '2013', '2016', '2019']\n",
      "2018, closest = 2019\n",
      "Bands: ['landcover', 'impervious', 'impervious_descriptor']\n",
      "[{'type': 'Date', 'value': 1522540800000}, {'type': 'Date', 'value': 1525132800000}, {'type': 'Date', 'value': 1527811200000}, {'type': 'Date', 'value': 1530403200000}]\n",
      "['0_BGR0_median', '0_BGR1_median', '0_BGR2_median', '0_SWIR0_median', '0_SWIR1_median', '0_SWIR2_median', '0_SR_TH_median', '0_BGR0_diff', '0_BGR1_diff', '0_BGR2_diff', '0_SWIR0_diff', '0_SWIR1_diff', '0_SWIR2_diff', '0_SR_TH_diff', '1_BGR0_median', '1_BGR1_median', '1_BGR2_median', '1_SWIR0_median', '1_SWIR1_median', '1_SWIR2_median', '1_SR_TH_median', '1_BGR0_diff', '1_BGR1_diff', '1_BGR2_diff', '1_SWIR0_diff', '1_SWIR1_diff', '1_SWIR2_diff', '1_SR_TH_diff', '2_BGR0_median', '2_BGR1_median', '2_BGR2_median', '2_SWIR0_median', '2_SWIR1_median', '2_SWIR2_median', '2_SR_TH_median', '2_BGR0_diff', '2_BGR1_diff', '2_BGR2_diff', '2_SWIR0_diff', '2_SWIR1_diff', '2_SWIR2_diff', '2_SR_TH_diff', '3_BGR0_median', '3_BGR1_median', '3_BGR2_median', '3_SWIR0_median', '3_SWIR1_median', '3_SWIR2_median', '3_SR_TH_median', '3_BGR0_diff', '3_BGR1_diff', '3_BGR2_diff', '3_SWIR0_diff', '3_SWIR1_diff', '3_SWIR2_diff', '3_SR_TH_diff', 'BGR0_stdDev', 'BGR1_stdDev', 'BGR2_stdDev', 'SWIR0_stdDev', 'SWIR1_stdDev', 'SWIR2_stdDev', 'SR_TH_stdDev']\n",
      "['0_BGR0_median', '0_BGR1_median', '0_BGR2_median', '0_SWIR0_median', '0_SWIR1_median', '0_SWIR2_median', '0_SR_TH_median', '0_BGR0_diff', '0_BGR1_diff', '0_BGR2_diff', '0_SWIR0_diff', '0_SWIR1_diff', '0_SWIR2_diff', '0_SR_TH_diff', '1_BGR0_median', '1_BGR1_median', '1_BGR2_median', '1_SWIR0_median', '1_SWIR1_median', '1_SWIR2_median', '1_SR_TH_median', '1_BGR0_diff', '1_BGR1_diff', '1_BGR2_diff', '1_SWIR0_diff', '1_SWIR1_diff', '1_SWIR2_diff', '1_SR_TH_diff', '2_BGR0_median', '2_BGR1_median', '2_BGR2_median', '2_SWIR0_median', '2_SWIR1_median', '2_SWIR2_median', '2_SR_TH_median', '2_BGR0_diff', '2_BGR1_diff', '2_BGR2_diff', '2_SWIR0_diff', '2_SWIR1_diff', '2_SWIR2_diff', '2_SR_TH_diff', '3_BGR0_median', '3_BGR1_median', '3_BGR2_median', '3_SWIR0_median', '3_SWIR1_median', '3_SWIR2_median', '3_SR_TH_median', '3_BGR0_diff', '3_BGR1_diff', '3_BGR2_diff', '3_SWIR0_diff', '3_SWIR1_diff', '3_SWIR2_diff', '3_SR_TH_diff', 'BGR0_stdDev', 'BGR1_stdDev', 'BGR2_stdDev', 'SWIR0_stdDev', 'SWIR1_stdDev', 'SWIR2_stdDev', 'SR_TH_stdDev']\n",
      "['0_BGR0_median', '0_BGR1_median', '0_BGR2_median', '0_SWIR0_median', '0_SWIR1_median', '0_SWIR2_median', '0_SR_TH_median', '0_BGR0_diff', '0_BGR1_diff', '0_BGR2_diff', '0_SWIR0_diff', '0_SWIR1_diff', '0_SWIR2_diff', '0_SR_TH_diff', '1_BGR0_median', '1_BGR1_median', '1_BGR2_median', '1_SWIR0_median', '1_SWIR1_median', '1_SWIR2_median', '1_SR_TH_median', '1_BGR0_diff', '1_BGR1_diff', '1_BGR2_diff', '1_SWIR0_diff', '1_SWIR1_diff', '1_SWIR2_diff', '1_SR_TH_diff', '2_BGR0_median', '2_BGR1_median', '2_BGR2_median', '2_SWIR0_median', '2_SWIR1_median', '2_SWIR2_median', '2_SR_TH_median', '2_BGR0_diff', '2_BGR1_diff', '2_BGR2_diff', '2_SWIR0_diff', '2_SWIR1_diff', '2_SWIR2_diff', '2_SR_TH_diff', '3_BGR0_median', '3_BGR1_median', '3_BGR2_median', '3_SWIR0_median', '3_SWIR1_median', '3_SWIR2_median', '3_SR_TH_median', '3_BGR0_diff', '3_BGR1_diff', '3_BGR2_diff', '3_SWIR0_diff', '3_SWIR1_diff', '3_SWIR2_diff', '3_SR_TH_diff', 'BGR0_stdDev', 'BGR1_stdDev', 'BGR2_stdDev', 'SWIR0_stdDev', 'SWIR1_stdDev', 'SWIR2_stdDev', 'SR_TH_stdDev', 'flood', 'sprinkler', 'other']\n",
      "Saving training samples for 2018\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "#### This one normalizes based on NLCD ag areas average and standard deviation\n",
    "'''\n",
    "\n",
    "import os\n",
    "import ee\n",
    "ee.Initialize()\n",
    "import folium\n",
    "print(folium.__version__)\n",
    "\n",
    "\n",
    "# # Functions\n",
    "\n",
    "\n",
    "IRRI_METHOD = {'Flood':0, 'F':0, 'S':1, 'Sprinkler':1}\n",
    "##Drip irrigation dropped because very limited\n",
    "\n",
    "IRRI_METHOD_list = [key for key in IRRI_METHOD]\n",
    "\n",
    "RESPONSE = ['flood', 'sprinkler', 'other']\n",
    "\n",
    "def filter_by_property(feat_coll, propert, values):\n",
    "    expression_ = ''\n",
    "    for val in values:\n",
    "        expression_ += '{0} == \\\"{1}\\\" || '.format(propert, val)\n",
    "    expression_ = expression_.strip(' || ')\n",
    "    expression_ += ''\n",
    "    filt = ee.Filter.expression(expression_)\n",
    "    new_feats = feat_coll.filter(filt)\n",
    "    return new_feats\n",
    "\n",
    "\n",
    "def get_irrig_array(feat):\n",
    "    global IRRNAME\n",
    "    feat = ee.Feature(feat)\n",
    "    thisirrg = ee.String(feat.get(IRRNAME))\n",
    "    thisdict = ee.Dictionary(IRRI_METHOD)\n",
    "    thisval = thisdict.get(thisirrg)\n",
    "    feat = feat.set({'Irrigation':thisval})\n",
    "    return feat\n",
    "    \n",
    "def get_wrlu_dataset(year_):\n",
    "    assert (year_ >= 2003 and year_ <= 2021), \"Year out of range\"\n",
    "    all_features = ee.FeatureCollection([])\n",
    "    if year_ in range(2003, 2008):\n",
    "        dname = 'projects/ee-snouwakpo/assets/Water_Related_Land_Use_2002_to_2007'\n",
    "        all_features = all_features.merge(ee.FeatureCollection(dname).filter(ee.Filter.expression(f'SURV_YEAR == \\\"{year_}\\\"')))\n",
    "     \n",
    "    if year_ in range(2005,2011):\n",
    "        dname = 'projects/ee-snouwakpo/assets/Water_Related_Land_Use_2005_to_2010'\n",
    "        all_features = all_features.merge(ee.FeatureCollection(dname).filter(ee.Filter.expression(f'SURV_YEAR == \\\"{year_}\\\"')))\n",
    "    \n",
    "    if year_ in range(2010,2016):\n",
    "        dname = 'projects/ee-snouwakpo/assets/Water_Related_Land_Use_2010_to_2015'\n",
    "        all_features = all_features.merge(ee.FeatureCollection(dname).filter(ee.Filter.expression(f'SURV_YEAR == {year_}')))\n",
    "    if year_ >= 2018:\n",
    "        dname = f'projects/ee-snouwakpo/assets/Water_Related_Land_Use_Statewide_{year_}'\n",
    "        all_features = ee.FeatureCollection(dname)\n",
    "    print(all_features.size().getInfo())\n",
    "    return all_features\n",
    "\n",
    "#needed\n",
    "CRS_ = 'EPSG:3857'\n",
    "SCALE_ = 30\n",
    "KERNEL_SIZE = 256\n",
    "KERNEL_DIM = SCALE_*(KERNEL_SIZE-1)/2 ##30m x kernel size (30*(32-1)/2)\n",
    "print(KERNEL_DIM)\n",
    "\n",
    "BOUND = ee.FeatureCollection(\"users/snouwakpo/ML_Training_Area\").filter(\"NAME == 'UTAH'\").first().geometry().bounds()\n",
    "\n",
    "\n",
    "\n",
    "# Period to use to grab images\n",
    "# 0 = Apr-Aug, 1 = Apr-June, 2 = July-Aug\n",
    "IMPERIOD = 0 \n",
    "PERIODDATA = [{'folder':'Landsat_April_August','months':[3,8]},\n",
    "              {'folder':'Landsat_April_June','months':[3,6]},\n",
    "              {'folder':'Landsat_July_August','months':[6,8]}]\n",
    "FOLDER = PERIODDATA[IMPERIOD]['folder']\n",
    "MONTHS = PERIODDATA[IMPERIOD]['months']\n",
    "\n",
    "# import sys\n",
    "for year in range(2018,2019):\n",
    "    if year in [2012,2016,2017]:\n",
    "        continue\n",
    "    YEAR_SEL = year ##Takes year as argument\n",
    "\n",
    "    print(f'Starting job for year {YEAR_SEL}')\n",
    "\n",
    "    TILESCALE = 16\n",
    "    #UTAH_WATER = ee.FeatureCollection('users/snouwakpo/Utah_Water_Related_Land_Use-shp')\n",
    "    UTAH_WATER = get_wrlu_dataset(YEAR_SEL)\n",
    "    print(UTAH_WATER.size().getInfo())\n",
    "    IRRNAME = ee.Feature(UTAH_WATER.first()).select(['IRR_.*']).propertyNames().getInfo()\n",
    "    print(IRRNAME)\n",
    "    assert len(IRRNAME) == 1, \"Check irrigation methods column name\"\n",
    "    IRRNAME = IRRNAME[0]\n",
    "\n",
    "\n",
    "    # In[294]:\n",
    "\n",
    "\n",
    "    #UTAH_AG_WATER = filter_by_property(UTAH_WATER, 'Landuse', ['Agricultural']) #Select only ag areas\n",
    "    UTAH_AG_WATER = filter_by_property(UTAH_WATER, IRRNAME, IRRI_METHOD_list) #Select only ag areas\n",
    "\n",
    "\n",
    "    # In[295]:\n",
    "\n",
    "\n",
    "    nFields_surveyed = UTAH_AG_WATER.size().getInfo()\n",
    "    print(nFields_surveyed)\n",
    "\n",
    "\n",
    "    # In[296]:\n",
    "\n",
    "\n",
    "    #Create response data\n",
    "    #UTAH_IRR_DICT = UTAH_AG_WATER.map(get_irrig_array, True) ##This to only run on agricultural fields\n",
    "    UTAH_IRR_DICT = UTAH_AG_WATER.map(get_irrig_array, True) ##Run on ag and non-ag features\n",
    "\n",
    "\n",
    "\n",
    "    UTAH_IRR_IMG = UTAH_IRR_DICT.reduceToImage(properties = ['Irrigation'], reducer = ee.Reducer.first()).rename('Irrigation')\n",
    "    #UNIFIED_IRR_BOUND = UTAH_IRR_DICT.union().first().geometry().simplify(maxError= 100)\n",
    "    UNIFIED_IRR_BOUND = UTAH_IRR_IMG.gte(0).selfMask().reduceToVectors(geometry=BOUND, crs=CRS_, scale=SCALE_, bestEffort=True)\n",
    "    FLOOD_IRR_BOUND = UTAH_IRR_IMG.eq(0).reduceToVectors(geometry=BOUND, crs=CRS_, scale=SCALE_, bestEffort=True)\n",
    "    SPRINKLER_IRR_BOUND = UTAH_IRR_IMG.eq(1).reduceToVectors(geometry=BOUND, crs=CRS_, scale=SCALE_, bestEffort=True)\n",
    "\n",
    "    print(UTAH_IRR_DICT.first().getInfo().get('properties').get('Irrigation'))\n",
    "    print(UTAH_IRR_IMG.getInfo())\n",
    "\n",
    "\n",
    "    # In[297]:\n",
    "\n",
    "    # need to put gza5dr\n",
    "    training_squares_ = ee.FeatureCollection(f'projects/ee-gza5dr/assets/Training_Samples_{KERNEL_SIZE}_{YEAR_SEL}')\n",
    "\n",
    "    evaluation_squares_ = ee.FeatureCollection(f'projects/ee-gza5dr/assets/Evaluation_Samples_{KERNEL_SIZE}_{YEAR_SEL}')\n",
    "\n",
    "    MASK = UTAH_IRR_IMG.mask()\n",
    "\n",
    "    ##Fill with value 2 for the other pixels\n",
    "    two = ee.Image(2).rename('Irrigation')\n",
    "    UTAH_IRR_IMG = two.where(MASK, UTAH_IRR_IMG).toUint8()\n",
    "\n",
    "\n",
    "    # Turn irrigation image into multiband\n",
    "\n",
    "    # In[300]:\n",
    "\n",
    "\n",
    "    expression_flood = 'b == 0 ? 1 : 0'\n",
    "    expression_sprinkler = 'b == 1 ? 1 : 0'\n",
    "    expression_other = 'b == 2 ? 1 : 0'\n",
    "\n",
    "    flood = ee.Image().expression(expression_flood,\n",
    "                                          {'b':UTAH_IRR_IMG.select(\"Irrigation\")}).rename(RESPONSE[0])\n",
    "    sprinkler = ee.Image().expression(expression_sprinkler,\n",
    "                                          {'b':UTAH_IRR_IMG.select(\"Irrigation\")}).rename(RESPONSE[1])\n",
    "    otherirrig = ee.Image().expression(expression_other,\n",
    "                                          {'b':UTAH_IRR_IMG.select(\"Irrigation\")}).rename(RESPONSE[2])\n",
    "    IRRIGATION_IMG = ee.Image.cat([flood, sprinkler, otherirrig])\n",
    "\n",
    "    other_threshold = KERNEL_SIZE*KERNEL_SIZE*0.9 #At least 10% of the area has to be flood or sprinkler\n",
    "\n",
    "\n",
    "\n",
    "    # # Imagery\n",
    "    # \n",
    "    # Gather and setup the imagery to use for inputs (predictors).  This is a three-year, cloud-free, Landsat 8 composite.  Display it in the notebook for a sanity check.\n",
    "\n",
    "    # In[301]:\n",
    "\n",
    "\n",
    "\n",
    "    ##########################################################################\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    training_year = YEAR_SEL\n",
    "\n",
    "    import ml_input_maker7_3 as ml_input_maker\n",
    "\n",
    "    thisInputClass = ml_input_maker.input_maker(training_year, BOUND)\n",
    "\n",
    "\n",
    "    AgAreas = thisInputClass.AGAREAS\n",
    "\n",
    "    landsat_col = thisInputClass.landsat_col\n",
    "    #otherbands = thisInputClass.otherbands\n",
    "    inputbands = thisInputClass.inputbands\n",
    "\n",
    "\n",
    "    GOTBANDS=False\n",
    "\n",
    "    while not GOTBANDS:\n",
    "        try:\n",
    "            BANDS = inputbands.bandNames().getInfo()\n",
    "            print(BANDS)\n",
    "            GOTBANDS = True\n",
    "        except ee.ee_exception.EEException:\n",
    "            print('Issue getting Band names from EE')\n",
    "        except:\n",
    "            print('Another error occured')\n",
    "            break\n",
    "\n",
    "    assert GOTBANDS, 'BAND NAMES COULD NOT BE RETRIEVED. STOPPING!!!'\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    FEATURES = BANDS+RESPONSE\n",
    "    print(FEATURES)\n",
    "\n",
    "\n",
    "\n",
    "    # In[302]:\n",
    "\n",
    "\n",
    "    #print(YEAR_SEL)\n",
    "\n",
    "    # ## Build feature stack\n",
    "\n",
    "    # In[303]:\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    featureStack = ee.Image.cat([\n",
    "      inputbands.select(BANDS),\n",
    "      IRRIGATION_IMG\n",
    "    ]).float()\n",
    "\n",
    "    # In[304]:\n",
    "\n",
    "\n",
    "    ## Create the arrays now\n",
    "    projected_stack = featureStack.reproject(crs = CRS_, scale=SCALE_)\n",
    "    def create_array_mapping(feat_in):\n",
    "        centroid = feat_in.centroid().transform(CRS_)\n",
    "        coords = centroid.geometry().coordinates()\n",
    "        new_coords = ee.List([[ee.Number(coords.get(0)).add(0-KERNEL_DIM), ee.Number(coords.get(1)).add(0-KERNEL_DIM)], \n",
    "                      [ee.Number(coords.get(0)).add(KERNEL_DIM),ee.Number(coords.get(1)).add(KERNEL_DIM)]])\n",
    "        bbox = ee.Geometry.Rectangle(coords=new_coords, \n",
    "                                     proj=CRS_, geodesic=False)\n",
    "        sampled_arr = projected_stack.sampleRectangle(bbox, defaultValue=0)\n",
    "        sampled_arr = sampled_arr.copyProperties(feat_in)\n",
    "        other_count = sampled_arr.getArray('other').reshape([-1]).reduce(reducer = ee.Reducer.sum(), axes=[0]).get([0])\n",
    "        flood_count = sampled_arr.getArray('flood').reshape([-1]).reduce(reducer = ee.Reducer.sum(), axes=[0]).get([0])\n",
    "        sprinkler_count = sampled_arr.getArray('sprinkler').reshape([-1]).reduce(reducer = ee.Reducer.sum(), axes=[0]).get([0])\n",
    "        test_1 = ee.Number(other_count).lte(other_threshold)\n",
    "        test_2 = ee.Number(other_count).add(ee.Number(flood_count)).add(ee.Number(sprinkler_count))\n",
    "        test_2 = test_2.neq(0)\n",
    "        test_ = test_1.And(test_2)\n",
    "        #test_ = test_2\n",
    "\n",
    "        sampled_arr = ee.Algorithms.If(test_, sampled_arr)\n",
    "\n",
    "        return sampled_arr\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    training_List = training_squares_.toList(15000)\n",
    "    training_List = training_List.shuffle()\n",
    "    training_List = training_List.slice(0,1000)\n",
    "    training_arrays = ee.FeatureCollection(training_List).map(create_array_mapping, True) #Takes the whole set but divides by 2\n",
    "    evaluation_List = evaluation_squares_.toList(30000)\n",
    "    evaluation_List = evaluation_List.shuffle()\n",
    "    evaluation_List = evaluation_List.slice(0,500)\n",
    "    evaluation_arrays = ee.FeatureCollection(evaluation_List).map(create_array_mapping, True) #Takes the whole set but divides by 2\n",
    "\n",
    "    #training_vals = training_arrays.first().getInfo()\n",
    "\n",
    "    import numpy as np\n",
    "\n",
    "    #img_size = np.array(training_vals['properties']['other'])\n",
    "    #print(f'IMGSIZE = {img_size.shape}')\n",
    "    #print([training_attr for training_attr in training_vals])\n",
    "\n",
    "    # ## Functions to save and load data as TFRecords\n",
    "\n",
    "    # In[279]:\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    try:\n",
    "        ntraining = training_arrays.size().getInfo()\n",
    "        neval = evaluation_arrays.size().getInfo()\n",
    "    except:\n",
    "        ntraining = 1000\n",
    "        neval = 250\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    def sample_AOC_to_tfrecord_squares(fname):\n",
    "        desc1 = fname + '_g_train'\n",
    "        if ntraining > 0:\n",
    "            print(f'Saving training samples for {YEAR_SEL}')\n",
    "            task = ee.batch.Export.table.toDrive(\n",
    "                collection = training_arrays,\n",
    "                description = desc1,\n",
    "                folder = f'Irrigation_detection_WRLU_NoCDL_Balanced_Normalized_Unique_UT_lib7_3_new',\n",
    "                fileFormat = 'TFRecord',\n",
    "                selectors = FEATURES\n",
    "            )\n",
    "            task.start()\n",
    "        desc2 = fname + '_g_eval'\n",
    "        # if neval > 0:\n",
    "        #     print(f'Saving evaluation samples for {YEAR_SEL}')\n",
    "        #     task = ee.batch.Export.table.toDrive(\n",
    "        #         collection = evaluation_arrays,\n",
    "        #         description = desc2,\n",
    "        #         folder = f'Irrigation_detection_WRLU_NoCDL_Balanced_Normalized_Unique_UT_lib7_3_new',\n",
    "        #         fileFormat = 'TFRecord',\n",
    "        #         selectors = FEATURES\n",
    "        #     )\n",
    "        #     task.start()\n",
    "            # TASKS.append(task)\n",
    "        return\n",
    "\n",
    "\n",
    "    # ### Write training and evaluation sets\n",
    "\n",
    "    # In[280]:\n",
    "\n",
    "\n",
    "\n",
    "    sample_AOC_to_tfrecord_squares(f'Samples_{YEAR_SEL}_{KERNEL_SIZE}x{KERNEL_SIZE}')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    # In[ ]:\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b4184d2-5da1-4045-8ead-590b28b0e3cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Tensorflow 2.13.0",
   "language": "python",
   "name": "tensorflow-2.13.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}