scrape-ireland.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from  util.spider import Spider\n",
    "from util.user_agent import get_random_ua"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "attrs = {\n",
    "        'name'       :  'name',\n",
    "        'ingredients':  'recipeIngredient',\n",
    "        'total_time':   'totalTime',\n",
    "        'instructions': 'recipeInstructions',\n",
    "        'servings':     'recipeYield',\n",
    "        'category':     'recipeCategory',\n",
    "        'prep_time':    'prepTime',\n",
    "        'cook_time':    'cookTime',\n",
    "}\n",
    "listing= {'items': \"//a[contains(@data-vars-ga-outbound-link, 'cooking/recipe-ideas/recipes')]/@href\", 'next': { '': '', 'type': 'one_page'}}\n",
    "seeds = ['https://www.delish.com/holiday-recipes/g2009/traditional-irish-food/']\n",
    "available_json = {'xpath' : \"normalize-space(//script[@type='application/ld+json'][contains(text(), 'recipeIngredient')])\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "delish_spider= Spider('https://www.delish.com', seeds= seeds, listing =listing,attrs= attrs, available_json=available_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'Irish Beef Stew',\n",
       " 'total_time': 'PT1H10M',\n",
       " 'ingredients': ['3 tbsp. extra-virgin olive oil, divided',\n",
       "  '2 lb. beef chuck stew meat, cubed into 1\" pieces&nbsp;',\n",
       "  'Kosher salt',\n",
       "  'Freshly ground black pepper',\n",
       "  '1 yellow onion, chopped',\n",
       "  '2 medium carrots, peeled and cut into rounds',\n",
       "  '2 stalks celery, chopped',\n",
       "  '3 cloves garlic, minced',\n",
       "  '3 medium russet potatoes, peeled and cut into large chunks',\n",
       "  '4 c. low-sodium beef broth',\n",
       "  '1 (16-oz.) bottle Guinness&nbsp;',\n",
       "  '2 tsp. fresh thyme',\n",
       "  'Freshly chopped parsley, for serving'],\n",
       " 'instructions': ['In a large Dutch oven over medium heat, heat 2 tablespoons oil. Season beef with salt and pepper, then add to pot and cook on all sides until seared, 10 minutes, working in batches if necessary. Transfer beef to a plate.&nbsp;',\n",
       "  'In same pot, add remaining 1 tablespoon oil and cook onion, carrots, and celery until soft, 5 minutes. Season with salt and pepper. Add garlic and cook until fragrant, about 1 minute.',\n",
       "  'Add beef back to Dutch oven, then add potatoes, broth, beer, and thyme, then scrape the bottom of the pot to release the fond (browned bits at the bottom). Bring to a boil, then immediately reduce heat to a simmer. Season with salt and pepper. Cover and let simmer until beef and potatoes are tender, 30 minutes.',\n",
       "  'Garnish with parsley before serving.'],\n",
       " 'servings': '6 servings',\n",
       " 'category': ['gluten-free',\n",
       "  'nut-free',\n",
       "  'weeknight meals',\n",
       "  'winter',\n",
       "  'dinner',\n",
       "  'main dish',\n",
       "  'meat'],\n",
       " 'prep_time': 'PT15M',\n",
       " 'cook_time': 'PT0S'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delish_spider.scrape_one_item('https://www.delish.com/cooking/recipe-ideas/a26258692/irish-stew-recipe/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_list = delish_spider.start_scrape() #max page is 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df = pd.DataFrame(result_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>total_time</th>\n",
       "      <th>ingredients</th>\n",
       "      <th>instructions</th>\n",
       "      <th>servings</th>\n",
       "      <th>category</th>\n",
       "      <th>prep_time</th>\n",
       "      <th>cook_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Shepherd's Pie</td>\n",
       "      <td>PT1H25M</td>\n",
       "      <td>[1 1/2 lb. potatoes, peeled, Kosher salt, 4 tb...</td>\n",
       "      <td>[Preheat oven to 400°. Make mashed potatoes:&amp;n...</td>\n",
       "      <td>6-8 servings</td>\n",
       "      <td>[low sugar, nut-free, weeknight meals, winter,...</td>\n",
       "      <td>PT20M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Barbecue Pulled Pork Shepherd's Pie</td>\n",
       "      <td>PT35M</td>\n",
       "      <td>[2 lb. red potatoes, 1/2 packet ranch seasonin...</td>\n",
       "      <td>[Preheat oven to 400°., Make the ranch mashed ...</td>\n",
       "      <td>8-10</td>\n",
       "      <td>[low-cost, St. Patrick's Day, weeknight meals,...</td>\n",
       "      <td>PT20M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Best Mashed Potatoes</td>\n",
       "      <td>PT30M</td>\n",
       "      <td>[3 lb. mixed potatoes, such as russets &amp; Yukon...</td>\n",
       "      <td>[In a large pot, cover potatoes with water and...</td>\n",
       "      <td>6-8 servings</td>\n",
       "      <td>[vegetarian, Christmas, Thanksgiving, side dish]</td>\n",
       "      <td>PT25M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vegetarian Mushroom Gravy</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>[2 tbsp. extra-virgin olive oil, 1/2 Onion, fi...</td>\n",
       "      <td>[Heat olive oil in a small saucepan over mediu...</td>\n",
       "      <td>8</td>\n",
       "      <td>[vegetarian, Thanksgiving, side dish]</td>\n",
       "      <td>PT0S</td>\n",
       "      <td>PT10M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cauli Shepherd's Pie</td>\n",
       "      <td>PT1H15M</td>\n",
       "      <td>[1 medium head cauliflower, cut into florets, ...</td>\n",
       "      <td>[Preheat oven to 400°. Bring a large pot of wa...</td>\n",
       "      <td>6-8 servings</td>\n",
       "      <td>[low sugar, nut-free, autumn, weeknight meals,...</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Skillet Beer Cheese</td>\n",
       "      <td>PT20M</td>\n",
       "      <td>[3 c. shredded cheddar, 1 (8-oz. block) cream ...</td>\n",
       "      <td>[Place a grate over a campfire. In a large cas...</td>\n",
       "      <td>8</td>\n",
       "      <td>[side dish, snack]</td>\n",
       "      <td>PT10M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Bacon Cheddar Beer Bread</td>\n",
       "      <td>PT1H25M</td>\n",
       "      <td>[3 c. flour, 3 tsp. baking powder, 1 tsp. kosh...</td>\n",
       "      <td>[Preheat oven to 375° and line a loaf pan with...</td>\n",
       "      <td>6-8 servings</td>\n",
       "      <td>[appetizers, side dish]</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Potatoes Au Gratin</td>\n",
       "      <td>PT1H30M</td>\n",
       "      <td>[6 russet potatoes, cleaned and peeled, 2 tbsp...</td>\n",
       "      <td>[Preheat oven to 375° and butter a large 3 qt ...</td>\n",
       "      <td>8 servings</td>\n",
       "      <td>[side dish]</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Slow-Cooker Creamed Corn</td>\n",
       "      <td>PT2H10M</td>\n",
       "      <td>[8 slices bacon, 5 (15-oz.) cans corn kernels,...</td>\n",
       "      <td>[In a nonstick skillet over medium heat, cook ...</td>\n",
       "      <td>6-8</td>\n",
       "      <td>[Christmas, Thanksgiving, side dish]</td>\n",
       "      <td>PT10M</td>\n",
       "      <td>PT2H</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Nachos Supreme</td>\n",
       "      <td>PT50M</td>\n",
       "      <td>[1 tbsp. extra-virgin olive oil, 1 large onion...</td>\n",
       "      <td>[Preheat oven to 425º and line a large baking ...</td>\n",
       "      <td>6-8</td>\n",
       "      <td>[feed a crowd, Super Bowl, appetizers, dinner,...</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Pork Cabbage Rolls</td>\n",
       "      <td>PT2H40M</td>\n",
       "      <td>[18 large cabbage leaves, 1/2 c. uncooked whit...</td>\n",
       "      <td>[Discard thick stems from 12 cabbage leaves; r...</td>\n",
       "      <td>4</td>\n",
       "      <td>[low-calorie, low-carb, dinner party, dinner, ...</td>\n",
       "      <td>PT0S</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Slow-Cooker Corned Beef Hash</td>\n",
       "      <td>PT5H10M</td>\n",
       "      <td>[1 lb. corned beef, chopped, 1 lb. russet pota...</td>\n",
       "      <td>[Line a slow cooker with foil and spray with c...</td>\n",
       "      <td>4</td>\n",
       "      <td>[dinner]</td>\n",
       "      <td>PT10M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Pretzel Ring Beer Cheese Dip</td>\n",
       "      <td>PT45M</td>\n",
       "      <td>[1 3/4 c. shredded cheddar, divided, 1/2 c. sh...</td>\n",
       "      <td>[Garnish with chives before serving.]</td>\n",
       "      <td>8 servings</td>\n",
       "      <td>[nut-free, cocktail party, Super Bowl, appetiz...</td>\n",
       "      <td>PT15M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Slow-Cooker Corned Beef &amp; Cabbage</td>\n",
       "      <td>PT9H</td>\n",
       "      <td>[1 medium yellow onion, cut into large wedges,...</td>\n",
       "      <td>[Add onion, carrots, celery and potatoes to bo...</td>\n",
       "      <td>6</td>\n",
       "      <td>[dairy-free, gluten-free, St. Patrick's Day, d...</td>\n",
       "      <td>PT30M</td>\n",
       "      <td>PT0S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   name total_time  \\\n",
       "0                        Shepherd's Pie    PT1H25M   \n",
       "1   Barbecue Pulled Pork Shepherd's Pie      PT35M   \n",
       "2                  Best Mashed Potatoes      PT30M   \n",
       "3             Vegetarian Mushroom Gravy      PT15M   \n",
       "4                  Cauli Shepherd's Pie    PT1H15M   \n",
       "5                   Skillet Beer Cheese      PT20M   \n",
       "6              Bacon Cheddar Beer Bread    PT1H25M   \n",
       "7                    Potatoes Au Gratin    PT1H30M   \n",
       "8              Slow-Cooker Creamed Corn    PT2H10M   \n",
       "9                        Nachos Supreme      PT50M   \n",
       "10                   Pork Cabbage Rolls    PT2H40M   \n",
       "11         Slow-Cooker Corned Beef Hash    PT5H10M   \n",
       "12         Pretzel Ring Beer Cheese Dip      PT45M   \n",
       "13    Slow-Cooker Corned Beef & Cabbage       PT9H   \n",
       "\n",
       "                                          ingredients  \\\n",
       "0   [1 1/2 lb. potatoes, peeled, Kosher salt, 4 tb...   \n",
       "1   [2 lb. red potatoes, 1/2 packet ranch seasonin...   \n",
       "2   [3 lb. mixed potatoes, such as russets & Yukon...   \n",
       "3   [2 tbsp. extra-virgin olive oil, 1/2 Onion, fi...   \n",
       "4   [1 medium head cauliflower, cut into florets, ...   \n",
       "5   [3 c. shredded cheddar, 1 (8-oz. block) cream ...   \n",
       "6   [3 c. flour, 3 tsp. baking powder, 1 tsp. kosh...   \n",
       "7   [6 russet potatoes, cleaned and peeled, 2 tbsp...   \n",
       "8   [8 slices bacon, 5 (15-oz.) cans corn kernels,...   \n",
       "9   [1 tbsp. extra-virgin olive oil, 1 large onion...   \n",
       "10  [18 large cabbage leaves, 1/2 c. uncooked whit...   \n",
       "11  [1 lb. corned beef, chopped, 1 lb. russet pota...   \n",
       "12  [1 3/4 c. shredded cheddar, divided, 1/2 c. sh...   \n",
       "13  [1 medium yellow onion, cut into large wedges,...   \n",
       "\n",
       "                                         instructions      servings  \\\n",
       "0   [Preheat oven to 400°. Make mashed potatoes:&n...  6-8 servings   \n",
       "1   [Preheat oven to 400°., Make the ranch mashed ...          8-10   \n",
       "2   [In a large pot, cover potatoes with water and...  6-8 servings   \n",
       "3   [Heat olive oil in a small saucepan over mediu...             8   \n",
       "4   [Preheat oven to 400°. Bring a large pot of wa...  6-8 servings   \n",
       "5   [Place a grate over a campfire. In a large cas...             8   \n",
       "6   [Preheat oven to 375° and line a loaf pan with...  6-8 servings   \n",
       "7   [Preheat oven to 375° and butter a large 3 qt ...    8 servings   \n",
       "8   [In a nonstick skillet over medium heat, cook ...           6-8   \n",
       "9   [Preheat oven to 425º and line a large baking ...           6-8   \n",
       "10  [Discard thick stems from 12 cabbage leaves; r...             4   \n",
       "11  [Line a slow cooker with foil and spray with c...             4   \n",
       "12              [Garnish with chives before serving.]    8 servings   \n",
       "13  [Add onion, carrots, celery and potatoes to bo...             6   \n",
       "\n",
       "                                             category prep_time cook_time  \n",
       "0   [low sugar, nut-free, weeknight meals, winter,...     PT20M      PT0S  \n",
       "1   [low-cost, St. Patrick's Day, weeknight meals,...     PT20M      PT0S  \n",
       "2    [vegetarian, Christmas, Thanksgiving, side dish]     PT25M      PT0S  \n",
       "3               [vegetarian, Thanksgiving, side dish]      PT0S     PT10M  \n",
       "4   [low sugar, nut-free, autumn, weeknight meals,...     PT15M      PT0S  \n",
       "5                                  [side dish, snack]     PT10M      PT0S  \n",
       "6                             [appetizers, side dish]     PT15M      PT0S  \n",
       "7                                         [side dish]     PT15M      PT0S  \n",
       "8                [Christmas, Thanksgiving, side dish]     PT10M      PT2H  \n",
       "9   [feed a crowd, Super Bowl, appetizers, dinner,...     PT15M      PT0S  \n",
       "10  [low-calorie, low-carb, dinner party, dinner, ...      PT0S      PT0S  \n",
       "11                                           [dinner]     PT10M      PT0S  \n",
       "12  [nut-free, cocktail party, Super Bowl, appetiz...     PT15M      PT0S  \n",
       "13  [dairy-free, gluten-free, St. Patrick's Day, d...     PT30M      PT0S  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df.to_csv('data/ireland/ireland_delish.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7f97af7b90ec3405e1f66aa9bfad4ba40234615df6d29df3ef268808d6a9feec"
  },
  "kernelspec": {
   "display_name": "Python 3.9.1 ('wb-scraping': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}