gradient-descent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2e9d7e29",
   "metadata": {},
   "source": [
    "# Gradient Descent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b56fea23",
   "metadata": {},
   "source": [
    "$$\n",
    "\\min z = f(x, y) = (x - \\pi)^2 + (y - e)^2\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "158cd9c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "f (generic function with 1 method)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function f(w)\n",
    "    return (w[1] - 3.14159265)^2 + (w[2] - exp(1))^2\n",
    "end "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "beebe076",
   "metadata": {},
   "source": [
    "### Load the required package for gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4d31b405",
   "metadata": {},
   "outputs": [],
   "source": [
    "using ForwardDiff"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47c56db",
   "metadata": {},
   "source": [
    "### Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b132299e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convergen in 74 iters. Breaking\n",
      "[3.141592385321343, 2.718281599444254]\n"
     ]
    }
   ],
   "source": [
    "eta = 0.1\n",
    "epsilon = 10^(-7)\n",
    "initial = [0.0, 0.0]\n",
    "for i in 1:100\n",
    "    newinitial = initial - ForwardDiff.gradient(f, initial) * eta\n",
    "    if sum(abs.(newinitial - initial)) < epsilon\n",
    "        println(\"Convergen in $i iters. Breaking\")\n",
    "        break\n",
    "    end \n",
    "    initial = newinitial\n",
    "end \n",
    "println(initial)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7dde07f",
   "metadata": {},
   "source": [
    "### Packing calculations using a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f3ad96c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gradientdescent (generic function with 1 method)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function gradientdescent(f, initialpoint; eta = 0.1, epsilon = 10^(-7), maxiter = 100)\n",
    "    for i in 1:maxiter\n",
    "        newinitial = initialpoint - ForwardDiff.gradient(f, initialpoint) * eta\n",
    "        if sum(abs.(newinitial - initialpoint)) < epsilon\n",
    "            println(\"Convergen in $i iters. Breaking\")\n",
    "            break\n",
    "        end \n",
    "        initialpoint = newinitial\n",
    "    end\n",
    "    return initialpoint\n",
    "end "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4663a4c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convergen in 74 iters. Breaking\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2-element Vector{Float64}:\n",
       " 3.141592385321343\n",
       " 2.718281599444254"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradientdescent(f, [0.0, 0.0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0d27273",
   "metadata": {},
   "source": [
    "### A more complex function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca982df3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "myf (generic function with 1 method)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myf(w) = (w[1] - 7.0)^2 + (w[2] - 77.0)^2 + (w[3] - 777.0)^2 + (w[4] - 7777.0)^2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbb8e427",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element Vector{Float64}:\n",
       " 100.0\n",
       " 100.0\n",
       " 100.0\n",
       " 100.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "initial = [100.0 for i in 1:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "caee73f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convergen in 107 iters. Breaking\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4-element Vector{Float64}:\n",
       "    7.00000000496617\n",
       "   77.0000000012282\n",
       "  776.9999999638484\n",
       " 7776.999999590051"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradientdescent(myf, initial, maxiter = 1000, eta = 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f893c890",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.9Beta4 1.9.0-beta4",
   "language": "julia",
   "name": "julia-1.9beta4-1.9"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}