分析贝壳网页.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://bj.lianjia.com/chengjiao/101103972186.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101103247046.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101103250764.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105254947.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101104095322.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105418786.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101104726856.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105507513.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105033914.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105827437.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101103603467.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101101858150.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105178638.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105766869.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101104475794.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105620892.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101101651125.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105955519.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101103451504.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101106149418.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105865369.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105677720.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105660785.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105631011.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105360441.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101104888160.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105055874.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105945821.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105373930.html',\n",
       " 'https://bj.lianjia.com/chengjiao/101105946945.html']"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "bjhtml = open('data/bj.html').read()\n",
    "re_set = re.compile('<li.*?<a.*?class=\"img.*?\".*?href=\"(.*?)\"')\n",
    "re_get = re.findall(re_set, bjhtml)\n",
    "re_get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/chengjiao/mw1',\n",
       " 'https://tj.ke.com/chengjiao/101104644501.html',\n",
       " 'https://tj.ke.com/chengjiao/101106026458.html',\n",
       " 'https://tj.ke.com/chengjiao/101105704170.html',\n",
       " 'https://tj.ke.com/chengjiao/101104525757.html',\n",
       " 'https://tj.ke.com/chengjiao/101105906154.html',\n",
       " 'https://tj.ke.com/chengjiao/101106049669.html',\n",
       " 'https://tj.ke.com/chengjiao/101105646336.html',\n",
       " 'https://tj.ke.com/chengjiao/101105831270.html',\n",
       " 'https://tj.ke.com/chengjiao/101105575688.html',\n",
       " 'https://tj.ke.com/chengjiao/101105255415.html',\n",
       " 'https://tj.ke.com/chengjiao/101104398604.html',\n",
       " 'https://tj.ke.com/chengjiao/101104591866.html',\n",
       " 'https://tj.ke.com/chengjiao/101105528713.html',\n",
       " 'https://tj.ke.com/chengjiao/101105563466.html',\n",
       " 'https://tj.ke.com/chengjiao/101105936484.html',\n",
       " 'https://tj.ke.com/chengjiao/101106134800.html',\n",
       " 'https://tj.ke.com/chengjiao/101106080872.html',\n",
       " 'https://tj.ke.com/chengjiao/101104471191.html',\n",
       " 'https://tj.ke.com/chengjiao/101105529733.html',\n",
       " 'https://tj.ke.com/chengjiao/101104151508.html',\n",
       " 'https://tj.ke.com/chengjiao/101105736354.html',\n",
       " 'https://tj.ke.com/chengjiao/101105108100.html',\n",
       " 'https://tj.ke.com/chengjiao/101105229519.html',\n",
       " 'https://tj.ke.com/chengjiao/101106086931.html',\n",
       " 'https://tj.ke.com/chengjiao/101104714003.html',\n",
       " 'https://tj.ke.com/chengjiao/101105117338.html',\n",
       " 'https://tj.ke.com/chengjiao/101105339477.html',\n",
       " 'https://tj.ke.com/chengjiao/101105346949.html',\n",
       " 'https://tj.ke.com/chengjiao/101105950779.html',\n",
       " 'https://tj.ke.com/chengjiao/101104868428.html',\n",
       " '<%=fangjia_url%>']"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "tjhtml = open('data/tmp.html').read().replace('\\n', '')\n",
    "re_set = re.compile('''<div.*?class=\"title\">.*?<a.*?href=\"(.*?)\".*?</div>''')\n",
    "re_get = re.findall(re_set, tjhtml)\n",
    "re_get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import bs4\n",
    "index = 0\n",
    "page = 0\n",
    "city = '天津'\n",
    "page_text = open('page.html').read()\n",
    "soup = BeautifulSoup(page_text, 'lxml')\n",
    "infos = {}\n",
    "infos['网址'] = re_get\n",
    "infos['标题'] = soup.title.text.split('_')[0]\n",
    "infos['售价(万)'] = soup.find(class_='dealTotalPrice').text.strip()\n",
    "infos['每平方售价'] = soup.find(class_='record_detail').text.split('元')[0].strip()[2:]\n",
    "partent = re.compile('<li><span class=\"label\">(.*?)</span>(.*?)</li>')\n",
    "result = re.findall(partent, page_text)\n",
    "for item in result:\n",
    "    if item[0] == '建成年代':\n",
    "        infos['建成时间：年'] = item[1].strip()\n",
    "    else:\n",
    "        infos[item[0]] = item[1].strip()\n",
    "\n",
    "ele=soup.find(class_='fl l-txt')\n",
    "result = [a.text for a in ele.find_all('a')]\n",
    "infos['所属下辖区']=result[1].split('二手房')[0]\n",
    "infos['所属商圈']=result[2].split('二手房')[0]\n",
    "infos['所属小区']=result[3].split('二手房')[0]\n",
    "msg = soup.find(class_='msg')\n",
    "result = []\n",
    "for x in msg.find_all('span'):\n",
    "    fields = x.text.replace(\" \",\"\").split('\\n')\n",
    "    con = []\n",
    "    for c in fields:\n",
    "        if c != '':\n",
    "            con.append(c)\n",
    "    result.append(con)\n",
    "for item in result:\n",
    "    if item[1] == '建成年代':\n",
    "        infos['建成时间：年'] = item[0].strip()\n",
    "    else:\n",
    "        infos[item[1]] = item[0].strip()\n",
    "infos['成交时间'] = '-'.join(soup.find('strong').text.strip().split('.'))\n",
    "row = index + (page - 1) * 30\n",
    "#infos['序号'] = row + 1\n",
    "infos['城市'] = city"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}