1 Data Acquisition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Collect food images and other information from websites\n",
    "including url, title, nutrients, ingredients, serving, and images from two food websites."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape from www.bbcgoodfood.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import shutil\n",
    "import requests\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from recipe_scrapers import scrape_me\n",
    "import sys\n",
    "sys.setrecursionlimit(50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get urls of recipes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def category_information(name, pages):\n",
    "    dic = {}\n",
    "    dic['name'] = name\n",
    "    dic['pages'] = pages\n",
    "    #dic['number'] = recipe_number\n",
    "\n",
    "    return dic\n",
    "\n",
    "\n",
    "def define_required_data():\n",
    "    raw_data = []\n",
    "    categories = ['burger', 'salad', 'cake', 'steak', 'rice', 'sandwich', 'pizza', 'cookies', 'soup', 'pasta', 'pie', 'bread']\n",
    "    pages = [18, 53, 36, 12, 23, 9, 7, 10, 21, 20, 20, 20]\n",
    "    total_numbers = [421, 1271, 857, 269, 531, 193, 146, 221, 490, 474, 471, 459]\n",
    "    \n",
    "    for i in range(len(categories)):\n",
    "        raw_data.append(category_information(categories[i], pages[i]))\n",
    "\n",
    "    return raw_data\n",
    "\n",
    "\n",
    "def get_search_urls(dic):\n",
    "    urls = []\n",
    "    for i in range(1, dic['pages']+1):\n",
    "        url = 'https://www.bbcgoodfood.com/search/recipes/page/' + str(i) + '/?q=' + dic['name'] + '&sort=-relevance'\n",
    "        urls.append(url)\n",
    "    \n",
    "    return urls\n",
    "\n",
    "\n",
    "def get_recipe_urls(page_urls):\n",
    "    # loop through pages\n",
    "    recipe_links = []\n",
    "    for page_url in page_urls:\n",
    "        scraper = scrape_me(page_url)\n",
    "        links = scraper.links()\n",
    "        recipe_links = extract_recipe_links(recipe_links, links)\n",
    "    \n",
    "    return recipe_links\n",
    "\n",
    "\n",
    "def extract_recipe_links(recipe_links, links):\n",
    "    for link in links:\n",
    "        try:\n",
    "            if link['class'] == ['standard-card-new__article-title', 'qa-card-link']:\n",
    "                recipe_links.append('https://www.bbcgoodfood.com'+link['href'])\n",
    "        except:\n",
    "            continue\n",
    "    \n",
    "    return recipe_links"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scrape information from recipe urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_from_web(categories):\n",
    "    # loop through each category\n",
    "    for category in categories:\n",
    "        name = category['name']\n",
    "        page_urls = get_search_urls(category)\n",
    "        print(\"Scraping \" + name + \"...\")\n",
    "        \n",
    "        if not os.path.exists('./extracted_data/bbc/'):\n",
    "            os.mkdir('./extracted_data/bbc/')\n",
    "        \n",
    "        path = './extracted_data/bbc/' + name + '/'\n",
    "        if not os.path.exists(path):\n",
    "            os.mkdir(path)\n",
    "        \n",
    "        # get structured information\n",
    "        recipe_links = get_recipe_urls(page_urls)\n",
    "        structured_information = extract_from_recipes(recipe_links)\n",
    "        download_images(path, structured_information, category)\n",
    "\n",
    "        df = pd.DataFrame(structured_information)\n",
    "        df.to_csv('./csv_files/bbc/' + name + '.csv', index=False)\n",
    "\n",
    "\n",
    "def extract_from_recipes(recipe_links):\n",
    "    # combine structured information of each recipe\n",
    "    structured_information = []\n",
    "    for recipe_link in recipe_links:\n",
    "        temp = structured_information_in_recipe(recipe_link)\n",
    "        # remove recipes that contain incomplete information\n",
    "        if temp == {} or temp['nutrients'] == {} or temp['ingredients'] == {} or temp['image'] == {}:\n",
    "            continue\n",
    "        else:\n",
    "            structured_information.append(temp)\n",
    "    return structured_information\n",
    "\n",
    "\n",
    "def structured_information_in_recipe(recipe_link):\n",
    "    print(recipe_link)\n",
    "    recipe = {}\n",
    "    scraper = scrape_me(recipe_link)\n",
    "    recipe['title'] = scraper.title()\n",
    "    recipe['image'] = scraper.image()\n",
    "    recipe['ingredients'] = scraper.ingredients()\n",
    "    recipe['nutrients'] = scraper.nutrients()\n",
    "    recipe['serving'] = scraper.yields()\n",
    "    recipe['url'] = scraper.url\n",
    "    \n",
    "    return recipe\n",
    "\n",
    "    \n",
    "def download_images(path, structured_information, category):\n",
    "    # download images from image links\n",
    "    print(\"Downloading images...\")\n",
    "    index = 0\n",
    "    for recipe in structured_information:\n",
    "        image = requests.get(recipe['image'])\n",
    "        with open(path + category + str(index)+'.jpg', 'wb') as f:\n",
    "            f.write(image.content)\n",
    "        index += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = define_required_data()\n",
    "scrape_from_web(categories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rename files to class+index.jpg (e.g., burger0.jpg)\n",
    "add filename and category as two new columns (file_name, category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort by 0,1,2,3..., instead of 1, 10, 11... \n",
    "def sort_key(s):\n",
    "    try:\n",
    "        c = re.findall('\\d+', s)[0]\n",
    "    except:\n",
    "        c = -1\n",
    "    return int(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "# rename images from 0.jpg to burger0.jpg\n",
    "for category in categories:\n",
    "    path = './extracted_data/bbc/' + category + '/'\n",
    "    names = os.listdir(path)\n",
    "    for name in names:\n",
    "        os.rename(path+name, path+category+name)\n",
    "\n",
    "        \n",
    "# rename csv files\n",
    "for category in categories:\n",
    "    print(category)\n",
    "    names = os.listdir('./extracted_data/bbc/' + category + '/')\n",
    "    names.sort(key=sort_key)\n",
    "    \n",
    "    #delete '.DS_Store' and 'cake.DS_Store'\n",
    "    if names[1].endswith('DS_Store'):\n",
    "        del names[1]\n",
    "    if names[0].endswith('DS_Store'):\n",
    "        del names[0]\n",
    "    \n",
    "    csv_file = pd.read_csv('./csv_files/bbc/' + category + '.csv')\n",
    "    csv_file['file_name'] = names\n",
    "    csv_file['category'] = [category]*len(csv_file)\n",
    "    csv_file.to_csv('./csv_files/bbc/' + category + '.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### move 85% images of each category to training folder, 15% to test folder\n",
    "Make sure that each class is included in the test set.  Add \"source\" as a new column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "# make sure the csv files are changed too\n",
    "train_data = pd.DataFrame()\n",
    "test_data = pd.DataFrame()\n",
    "\n",
    "for category in categories:\n",
    "    print(category)\n",
    "    path = './csv_files/bbc/'\n",
    "    for path, subpath, files in os.walk(path):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.endswith(category + \".csv\"):\n",
    "                data = pd.read_csv(path + i)\n",
    "                train_temp = data[:-round(len(data)*0.15)]\n",
    "                test_temp = data[-round(len(data)*0.15):]\n",
    "                \n",
    "                train_data = train_data.append(train_temp,ignore_index=True)\n",
    "                test_data = test_data.append(test_temp, ignore_index=True)\n",
    "                            \n",
    "                # move training images \n",
    "                for i in range(len(train_temp)):\n",
    "                    src_path = './extracted_data/bbc/' + category + '/' + category + str(i)\n",
    "                    try:\n",
    "                        shutil.move(src_path + '.jpg', './data/all_data/')\n",
    "                    except:\n",
    "                        shutil.move(src_path + '.png', './data/all_data/')\n",
    "                \n",
    "                # move test images\n",
    "                for i in range(len(test_temp)):\n",
    "                    src_path = './extracted_data/bbc/' + category + '/' + category + str(len(data)-i-1)\n",
    "                    try:\n",
    "                        shutil.move(src_path + '.jpg', './data/all_data/')\n",
    "                    except:\n",
    "                        shutil.move(src_path + '.png', './data/all_data/')\n",
    "\n",
    "                        \n",
    "train_data['source'] = ['bbc']*len(train_data)\n",
    "test_data['source'] = ['bbc']*len(test_data)\n",
    "train_data.to_csv('./data/bbc_train.csv', index=False)\n",
    "test_data.to_csv('./data/bbc_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Integrate images from train/test folder back to original folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "try:\n",
    "    os.remove('./data/bbc_train.csv')\n",
    "    os.remove('./data/bbc_test.csv')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "for category in categories:\n",
    "    src1 = './data/all_data/'\n",
    "    src2 = './data/all_data/'\n",
    "    dst = './extracted_data/bbc/' + category + '/'\n",
    "    \n",
    "    for path, subpath, files in os.walk(src1):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.startswith(category):\n",
    "                shutil.move(src1 + i, dst)   \n",
    "                \n",
    "    for path, subpath, files in os.walk(src2):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.startswith(category):\n",
    "                shutil.move(src2 + i, dst)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape from www.thekitchn.com "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument('User-Agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get urls of search results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get urls of recipes\n",
    "\n",
    "def category_information(name, pages):\n",
    "    dic = {}\n",
    "    dic['name'] = name\n",
    "    dic['pages'] = pages\n",
    "    #dic['number'] = recipe_number\n",
    "\n",
    "    return dic\n",
    "\n",
    "\n",
    "def define_required_data():\n",
    "    raw_data = []\n",
    "    categories = ['burger', 'salad', 'cake', 'steak', 'rice', 'sandwich', 'pizza', 'cookies', 'soup', 'pasta', 'pie', 'bread']\n",
    "    pages = [2, 23, 50, 8, 25, 7, 6, 11, 12, 16, 13, 12]\n",
    "    total_numbers = [35, 452, 1687, 151, 500, 136, 113, 213, 240, 320, 260, 240]\n",
    "    \n",
    "    for i in range(len(categories)):\n",
    "        raw_data.append(category_information(categories[i], pages[i]))\n",
    "\n",
    "    return raw_data\n",
    "\n",
    "\n",
    "def get_search_urls(dic):\n",
    "    urls = []\n",
    "    for i in range(1, dic['pages']+1):\n",
    "        url = 'https://www.thekitchn.com/search?filter=recipes&page=' + str(i) + '&q=' + dic['name']\n",
    "        urls.append(url)\n",
    "    \n",
    "    return urls\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get urls of recipes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_recipe_links(search_results):\n",
    "    # input: urls of search result  of a specific category\n",
    "    # output: urls of recipes of a specific category\n",
    "\n",
    "    # initialize selenium instance, wait for a while to load the content\n",
    "    driver = webdriver.Chrome(executable_path='./chromedriver', options=chrome_options)\n",
    "    driver.implicitly_wait(10)\n",
    "\n",
    "    recipe_links = []\n",
    "    last_links = 0\n",
    "    \n",
    "    for search_result in search_results:\n",
    "        recipe_links = parse_search_results(driver, search_result, recipe_links)\n",
    "        # if it fails at extracting this page (e.g., robot test), then keep trying...\n",
    "        while last_links == len(recipe_links):\n",
    "            recipe_links = parse_search_results(driver, search_result, recipe_links)\n",
    "            time.sleep(5)\n",
    "        time.sleep(5)\n",
    "        last_links = len(recipe_links)\n",
    "        print(last_links)\n",
    "    driver.quit()\n",
    "    \n",
    "    return recipe_links\n",
    "\n",
    "\n",
    "def parse_search_results(driver, search_result, recipe_links):\n",
    "    driver.get(search_result)\n",
    "    html = driver.page_source\n",
    "    soup = BeautifulSoup(html)\n",
    "    print(search_result)\n",
    "\n",
    "    page_links = soup.find_all('a', attrs={'class', 'Teaser__headline Teaser__link'})\n",
    "    for i in range(len(page_links)):\n",
    "        recipe_link = page_links[i]['data-gtm-search-url']\n",
    "        recipe_links.append(recipe_link)\n",
    "    return recipe_links\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scrape information from recipe urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extrct_structured_information(recipe_links):\n",
    "    # input: recipe links of a specific category\n",
    "    # output: list of dictionaries of nutrients, ingredients etc. contained in each recipe\n",
    "    \n",
    "    structured_information = []\n",
    "    \n",
    "    for recipe_link in recipe_links:\n",
    "        temp = get_structured_information(recipe_link)\n",
    "        if temp == {} or temp['Calories '] == 'NaN' or temp['image'].endswith('gif'):\n",
    "            continue\n",
    "        else:\n",
    "            structured_information.append(temp)\n",
    "    \n",
    "    return structured_information\n",
    "    \n",
    "    \n",
    "def get_structured_information(recipe_link):\n",
    "    # input: recipe url\n",
    "    # output: structured information including title etc.\n",
    "    print(recipe_link)\n",
    "    recipe = {}\n",
    "    try:\n",
    "        scraper = scrape_me(recipe_link)\n",
    "        recipe['title'] = scraper.title()\n",
    "        recipe['link'] = scraper.url()\n",
    "        recipe['image'] = scraper.image()\n",
    "        recipe['ingredients'] = scraper.ingredients()\n",
    "        recipe['nutrients'] = get_nutrients(scraper)\n",
    "        recipe['serving'] = get_serving(scraper)\n",
    "        return recipe\n",
    "    except:\n",
    "        print('something went wrong...↑')\n",
    "        return recipe\n",
    "\n",
    "\n",
    "def get_nutrients(scraper):\n",
    "    # scrape nutrients from soup\n",
    "    \n",
    "    information = scraper.soup()[0]\n",
    "    nutrient = {}\n",
    "\n",
    "    nutrient_names = information.find_all('span', attrs={'class':'jsx-1041931414 NutritionalGuide__nutrient-name'})\n",
    "    nutrient_values = information.find_all('span', attrs={'class':'jsx-1041931414 NutritionalGuide__nutrient-quantity'})\n",
    "\n",
    "    for i in range(len(nutrient_names)):\n",
    "        nutrient_name = nutrient_names[i].text\n",
    "        nutrient_value = nutrient_values[i].text\n",
    "        nutrient[nutrient_name] = nutrient_value\n",
    "    \n",
    "    return nutrient\n",
    "    \n",
    "\n",
    "def get_serving(scraper):\n",
    "    \n",
    "    information = scraper.soup()[0]\n",
    "    try:\n",
    "        yield_serving = information.find_all('p', attrs={'class':'jsx-2401602051 Recipe__yield'})[0].text\n",
    "        serving = int(re.findall('\\d+', yield_serving)[0])\n",
    "    except:\n",
    "        serving = None\n",
    "    return serving\n",
    "\n",
    "\n",
    "def download_images(path, structured_information):\n",
    "    index = 0\n",
    "    for recipe in structured_information:\n",
    "        image = requests.get(recipe['image'])\n",
    "        with open(path + str(index)+'.jpg', 'wb') as f:\n",
    "            f.write(image.content)\n",
    "        index += 1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = define_required_data()\n",
    "\n",
    "for category in categories:\n",
    "    name = category['name']\n",
    "    search_results = get_search_urls(category)\n",
    "    recipe_links = get_recipe_links(search_results)\n",
    "    structured_information = extrct_structured_information(recipe_links)\n",
    "    \n",
    "    if not os.path.exists('./extracted_data/thekitchen/'):\n",
    "        os.mkdir('./extracted_data/thekitchen/')\n",
    "        \n",
    "    path = './extracted_data/thekitchen/' +  name + '/'\n",
    "    if not os.path.exists(path):\n",
    "        os.mkdir(path)\n",
    "\n",
    "    df = pd.DataFrame(structured_information)\n",
    "    df.to_csv('./csv_files/thekitchen/' + name + '.csv', index=False)\n",
    "    \n",
    "    # download images from extracted image links\n",
    "    download_images(path, structured_information)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rename files to kitchen_class+index.jpg (e.g., kitchen_burger0.jpg)\n",
    "add filename and category as two new columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort by 0,1,2,3..., instead of 1, 10, 11... \n",
    "def sort_key(s):\n",
    "    try:\n",
    "        c = re.findall('\\d+', s)[0]\n",
    "    except:\n",
    "        c = -1\n",
    "    return int(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "for category in categories:\n",
    "    path = './extracted_data/thekitchen/' + category + '/'\n",
    "    names = os.listdir(path)\n",
    "    for name in names:\n",
    "        os.rename(path+name, path+'kitchen_'+category+name)\n",
    "        \n",
    "        \n",
    "# rename csv files\n",
    "for category in categories:\n",
    "    print(category)\n",
    "    names = os.listdir('./extracted_data/thekitchen/' + category + '/')\n",
    "    names.sort(key=sort_key)\n",
    "    \n",
    "    #delete '.DS_Store' and 'cake.DS_Store'\n",
    "    if names[1].endswith('DS_Store'):\n",
    "        del names[1]\n",
    "    if names[0].endswith('DS_Store'):\n",
    "        del names[0]\n",
    "    \n",
    "    csv_file = pd.read_csv('./csv_files/thekitchen/' + category + '.csv')\n",
    "    csv_file['file_name'] = names\n",
    "    csv_file['category'] = [category]*len(csv_file)\n",
    "    csv_file.to_csv('./csv_files/thekitchen/' + category + '.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### move 85% images of each category to training folder, 15% to test folder\n",
    "Make sure that each class is included in the test set.  Add \"source\" as a new column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "train_data = pd.DataFrame()\n",
    "test_data = pd.DataFrame()\n",
    "\n",
    "for category in categories:\n",
    "    print(category)\n",
    "    path = './csv_files/thekitchen/'\n",
    "    for path, subpath, files in os.walk(path):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.endswith(category + \".csv\"):\n",
    "                data = pd.read_csv(path + i)\n",
    "                train_temp = data[:-round(len(data)*0.15)]\n",
    "                test_temp = data[-round(len(data)*0.15):]\n",
    "                \n",
    "                train_data = train_data.append(train_temp, ignore_index=True)\n",
    "                test_data = test_data.append(test_temp, ignore_index=True)\n",
    "            \n",
    "                # Move training images\n",
    "                for i in range(len(train_temp)):\n",
    "                    src_path = './extracted_data/thekitchen/' + category + '/kitchen_' + category + str(i)\n",
    "                    try:\n",
    "                        shutil.move(src_path + '.jpg', './data/all_data/')\n",
    "                    except:\n",
    "                        if os.path.exists(src_path + '.jp2'):\n",
    "                            shutil.move(src_path + '.jp2', './data/all_data/')\n",
    "                        else:\n",
    "                            continue\n",
    "                \n",
    "                # Move test images\n",
    "                for i in range(len(test_temp)):\n",
    "                    src_path = './extracted_data/thekitchen/' + category + '/kitchen_' + category + str(len(data)-i-1)\n",
    "                    try:\n",
    "                        shutil.move(src_path + '.jpg', './data/all_data/')\n",
    "                    except:\n",
    "                        if os.path.exists(src_path + '.jp2'):\n",
    "                            shutil.move(src_path + '.jp2', './data/all_data/')\n",
    "                        else:\n",
    "                            continue\n",
    "        \n",
    "train_data['source'] = ['kitchen']*len(train_data)\n",
    "test_data['source'] = ['kitchen']*len(test_data)\n",
    "\n",
    "train_data.to_csv('./data/kitchen_train.csv', index=False)\n",
    "test_data.to_csv('./data/kitchen_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Integrate images from train/test folder back to original folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['bread', 'burger', 'cake', 'cookies', 'pasta', 'pie', \n",
    "              'pizza', 'rice', 'salad', 'sandwich', 'soup', 'steak']\n",
    "\n",
    "try:\n",
    "    os.remove('./data/kitchen_train.csv')\n",
    "    os.remove('./data/kitchen_test.csv')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "for category in categories:\n",
    "    src1 = './data/all_data/'\n",
    "    src2 = './data/all_data/'\n",
    "    dst = './extracted_data/thekitchen/' + category + '/'\n",
    "    \n",
    "    for path, subpath, files in os.walk(src1):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.startswith('kitchen_'+ category):\n",
    "                shutil.move(src1 + i, dst)   \n",
    "                \n",
    "    for path, subpath, files in os.walk(src2):\n",
    "        files.sort()\n",
    "        for i in files:\n",
    "            if i.startswith('kitchen_'+ category):\n",
    "                shutil.move(src2 + i, dst)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}