고양이_사료_크롤링.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "고양이 사료 크롤링.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNX7AM5gllPpNxAw8P0GIf5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/da-head0/mypetfood/blob/main/%EA%B3%A0%EC%96%91%EC%9D%B4_%EC%82%AC%EB%A3%8C_%ED%81%AC%EB%A1%A4%EB%A7%81.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MAp7k_uXOZ8l",
        "outputId": "93c39e45-fcea-476f-acc4-a01e5c09a88a"
      },
      "source": [
        "!pip install requests\n",
        "!pip install bs4"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (2.23.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests) (1.25.11)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests) (2.10)\n",
            "Requirement already satisfied: bs4 in /usr/local/lib/python3.7/dist-packages (0.0.1)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from bs4) (4.6.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KpYb09yZZqFR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a239a9e-eb60-4327-de8c-611c664f778f"
      },
      "source": [
        "import requests\n",
        "\n",
        "def crawl():\n",
        "  # 클래스가 증가할 때 마다 hidden body가 나타난다?\n",
        "  url = f\"https://www.purplesto.re/products/sales/list/?type=category&value=18\"\n",
        "  data = requests.get(url)\n",
        "  html = data.text\n",
        "  soup = BeautifulSoup(html, 'html.parser')\n",
        "\n",
        "  titles = soup.select('li')\n",
        "  for i in titles:\n",
        "    print(i.text)\n",
        "\n",
        "  # prices = soup.find('strong', {'class':'jss227'})\n",
        "  # #for pr in prices:\n",
        "  # print(prices.text)\n",
        "  \n",
        "crawl()\n",
        "#def parse(pageString):"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 고객센터\n",
            " 주문/배송\n",
            " 정기 배송\n",
            "전체보기\n",
            "주식\n",
            "간식\n",
            "용품\n",
            "회사소개\n",
            "채용 안내\n",
            "입점 문의\n",
            "도매 문의\n",
            "어플라우즈\n",
            "폴카독베이커리\n",
            "퍼플네스트\n",
            "퍼플잼 \n",
            "고객센터/FAQ\n",
            "공지사항\n",
            "이용약관\n",
            "개인정보 처리방침\n",
            "사업자정보 확인\n",
            "터키츄\n",
            "열빙어\n",
            "네스트매트\n",
            "네스트보울\n",
            "폴카독\n",
            "어플라우즈\n",
            "파미나\n",
            "지위픽\n",
            "펫밀크\n",
            "닥터링크\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJ3mT-S8CB3Q"
      },
      "source": [
        "database = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EI2kIWWNrOaM",
        "outputId": "28dccb85-c1a1-4611-f9f0-8c2b014e04b6"
      },
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "\n",
        "database = {}\n",
        "\n",
        "def crawl(num):\n",
        "  # 클래스가 증가할 때 마다 hidden body가 나타난다?\n",
        "  url = f\"https://www.purplesto.re/products/sales/{num}\"\n",
        "  data = requests.get(url)\n",
        "  html = data.text\n",
        "  soup = BeautifulSoup(html, 'html.parser')\n",
        "\n",
        "  category = soup.find('p', {'class':'breadcrumb__path'}).getText(separator=u' ') # 고양이 만 나옴\n",
        "  #script = soup.find('script')\n",
        "  #database['category'] = script\n",
        "  database['category'] = category\n",
        "\n",
        "  titles = soup.find('p', {'class':'productInfo__name'})\n",
        "  database['title'] = titles.text\n",
        "\n",
        "  brand = soup.select_one('section > div > section.productInfo > div.productInfo__content > div.productInfo__title > span > a')\n",
        "  database['brand'] = brand.text\n",
        "\n",
        "  detail = soup.find('div', {'class':'productSubInfo__detail-table'}).getText(separator=u' ')\n",
        "  age = detail.split('연령구분 ')[1].split(' 푸드타입 ')  # 정보 나옴 - 정규식으로 처리 해서 딕셔너리 넣을 수 있을듯\n",
        "  database['age'] = age[0]\n",
        "  classification = age[1].split(' 중량 ')\n",
        "  database['classification'] = classification[0]\n",
        "  gram = classification[1].split(' 주재료 ')\n",
        "  database['gram'] = gram[0]\n",
        "  ingredient = gram[1].split(' 식단정보 ')\n",
        "  database['ingredient'] = ingredient[0]\n",
        "  info = ingredient[1].split(' 칼로리 ')\n",
        "  database['info'] = info[0]\n",
        "  from_company = info[1].split(' 원산지/제조사 ')\n",
        "  database['calory'] = from_company[0]\n",
        "  database['from_company'] = from_company[1]\n",
        "\n",
        "  img = soup.find('img'>'src', {'alt':'상품 이미지'})\n",
        "  database['img'] = img['src']\n",
        "\n",
        "  content = soup.find('div', {'class':'content'})\n",
        "  database['content'] = content.text\n",
        "\n",
        "  try:\n",
        "    registeredIngredient__grid = soup.find('div', {'class':'registeredIngredient__grid'}).getText(separator=u' ')\n",
        "    regi_num = registeredIngredient__grid.split('성분등록번호 ')[1].split(' 조단백 ')  # 정보 나옴 - 정규식으로 처리 해서 딕셔너리 넣을 수 있을듯\n",
        "    database['성분등록번호'] = regi_num[0]\n",
        "    protein = regi_num[1].split(' 조지방 ')\n",
        "    database['조단백'] = protein[0]\n",
        "    fat = protein[1].split(' 조섬유 ')\n",
        "    database['조지방'] = fat[0]\n",
        "    fiber = fat[1].split(' 조회분 ')\n",
        "    database['조섬유'] = fiber[0]\n",
        "    ash = fiber[1].split(' 수분 ')\n",
        "    database['조회분'] = ash[0]\n",
        "    database['수분'] = ash[1]\n",
        "  except:\n",
        "    nutritionCheck__grid = soup.find('div', {'class':'nutritionCheck__grid'}).getText(separator=u' ')\n",
        "    database['nutrient'] = nutritionCheck__grid\n",
        "\n",
        "  # 이걸 바로 클라우드 db에 저장??\n",
        "  return database\n",
        "\n",
        "crawl(899)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'age': '키튼,어덜트,시니어,전연령',\n",
              " 'brand': 'K9 내추럴 (K9 NATURAL)',\n",
              " 'calory': '895 kcal/kg (76 kcal/캔)',\n",
              " 'category': '고양이',\n",
              " 'classification': '습식사료',\n",
              " 'content': '닭고기, 양의 심장, 양의 신장, 양의 간, 혈장 단백질(양의 피), 아마씨, 뉴질랜드 녹색 입 홍합, 켈프, 타우린, 비타민E, 산화마그네슘, 구리, 망간, 티아민, 비타민D3, 엽산',\n",
              " 'from_company': '뉴질랜드/K9 Natural Foods Limited',\n",
              " 'gram': '85g',\n",
              " 'img': 'https://cdn.purplesto.re/media/store/sale/main_image/k9natural_cat_B008CF02_thumb01.png',\n",
              " 'info': '검류미함유,그레인프리,글루텐프리',\n",
              " 'ingredient': '닭고기, 양의 심장, 양의 신장',\n",
              " 'nutrient': '조단백 9.6% (성묘 최소 6%, 자묘 최소 7%) 조지방 5% (성묘 최소 2%, 자묘 최소 2%) 칼슘 0.3% (성묘 최소 0.1%, 자묘 최소 0.2%) 인 0.27% (성묘 최소 0.1%, 자묘 최소 0.2%) 수분 82.5%',\n",
              " 'title': 'CAT 피스트 캔 닭고기와 양고기 85g'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 540
        },
        "id": "1uTdf7XHExP2",
        "outputId": "04e94bf6-e33f-4ce5-b121-4148736ccbe6"
      },
      "source": [
        "!pip install boto3"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/65/fb/695bde21f34b6bd52db30278d6fdc63b9ce54b27d6ef3369444f621c98e2/boto3-1.17.83-py2.py3-none-any.whl (131kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 4.0MB/s \n",
            "\u001b[?25hCollecting s3transfer<0.5.0,>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/63/d0/693477c688348654ddc21dcdce0817653a294aa43f41771084c25e7ff9c7/s3transfer-0.4.2-py2.py3-none-any.whl (79kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 7.1MB/s \n",
            "\u001b[?25hCollecting botocore<1.21.0,>=1.20.83\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cb/46/8a9db3126e2390cb0a551344a237b5bc5162478b04f56a96cbe1cac5d5a8/botocore-1.20.83-py2.py3-none-any.whl (7.6MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6MB 18.0MB/s \n",
            "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore<1.21.0,>=1.20.83->boto3) (2.8.1)\n",
            "Collecting urllib3<1.27,>=1.25.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0c/cd/1e2ec680ec7b09846dc6e605f5a7709dfb9d7128e51a026e7154e18a234e/urllib3-1.26.5-py2.py3-none-any.whl (138kB)\n",
            "\u001b[K     |████████████████████████████████| 143kB 50.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.21.0,>=1.20.83->boto3) (1.15.0)\n",
            "\u001b[31mERROR: requests 2.23.0 has requirement urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1, but you'll have urllib3 1.26.5 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: jmespath, urllib3, botocore, s3transfer, boto3\n",
            "  Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed boto3-1.17.83 botocore-1.20.83 jmespath-0.10.0 s3transfer-0.4.2 urllib3-1.26.5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Ylx2yGXEuDe"
      },
      "source": [
        "import boto3\n",
        "\n",
        "dynamodb = boto3.resource('dynamodb')\n",
        "\n",
        "table = dynamodb.Table('petfood')\n",
        "for i in range(1,10):\n",
        "  data = crawl(i)\n",
        "  table.put_item(Item=data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P-ihGx5ftwR5",
        "outputId": "10107bfd-6f2f-42a3-acd6-871a4faa6379"
      },
      "source": [
        "s = \"\"\"<div class=\"productSubInfo__detail-table__row\"><div class=\"title\">연령구분</div><div class=\"text\">키튼,어덜트,시니어,전연령</div></div>\n",
        "\"\"\"\n",
        "\n",
        "import re\n",
        "\n",
        "p = re.compile(\"\\\"title\\\">.+$</div>\")\n",
        "p.findall(s)"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 88
        }
      ]
    }
  ]
}