xpath (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from parsel import Selector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample1 = \"\"\"\n",
    "<html>\n",
    "  <head>\n",
    "    <title>My page</title>\n",
    "  </head>\n",
    "  <body>\n",
    "    <h2>Welcome to my <a href=\"#\" src=\"xxxx\">page</a></h2>\n",
    "    <h2>Welcome to my <a href=\"#\" src=\"xxxx\">page</a></h2>\n",
    "    <h2>Welcome to my <a href=\"#\" src=\"xxxx\">page</a></h2>\n",
    "    <h2>Welcome to my <a href=\"#\" src=\"xxxx\">page</a></h2>\n",
    "    <p>This is the first paragraph.</p>\n",
    "    <!-- this is the end -->\n",
    "  </body>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getxpath(html):\n",
    "    return Selector(text=html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s1 = getxpath(sample1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'My page'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取标题(两种方法都可以，第一种为绝对地址，第二种为相对地址)\n",
    "# s1.xpath('//title/text()').extract_first()\n",
    "s1.xpath('/html/head/title/text()').extract_first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xxxx'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.xpath('//h2/a/@src').extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['#', '#', '#', '#']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.xpath('//@href').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n  ',\n",
       " '\\n    ',\n",
       " 'My page',\n",
       " '\\n  ',\n",
       " '\\n  ',\n",
       " '\\n    ',\n",
       " 'Welcome to my ',\n",
       " 'page',\n",
       " '\\n    ',\n",
       " 'Welcome to my ',\n",
       " 'page',\n",
       " '\\n    ',\n",
       " 'Welcome to my ',\n",
       " 'page',\n",
       " '\\n    ',\n",
       " 'Welcome to my ',\n",
       " 'page',\n",
       " '\\n    ',\n",
       " 'This is the first paragraph.',\n",
       " '\\n    ',\n",
       " '\\n  ',\n",
       " '\\n']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.xpath('//text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<!-- this is the end -->']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.xpath('//comment()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample2 = \"\"\"\n",
    "<html>\n",
    "  <body>\n",
    "    <ul>\n",
    "      <li>Quote 1</li>\n",
    "      <li>Quote 2 with <a href=\"...\">link</a></li>\n",
    "      <li>Quote 3 with <a href=\"...\">another link</a></li>\n",
    "      <li><h2>Quote 4 title</h2> ...</li>\n",
    "    </ul>\n",
    "  </body>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s2 = getxpath(sample2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Quote 1', 'Quote 2 with ', 'Quote 3 with ', ' ...']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#所有的li\n",
    "s2.xpath('//li/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Quote 2 with ']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取第1个li\n",
    "s2.xpath('//li[position() = 1]/text()').extract()\n",
    "s2.xpath('//li[1]/text()').extract()\n",
    "# #获取第2个li\n",
    "s2.xpath('//li[position() = 2]/text()')\n",
    "s2.xpath('//li[2]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' ...']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#所有基数位的li\n",
    "s2.xpath('//li[position() mod2 = 1]/text()').extract()\n",
    "# #所有偶数位的li\n",
    "s2.xpath('//li[position() mod2 = 0]/text()').extract()\n",
    "# #取li最后一个的\n",
    "s2.xpath('//li[last()]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Quote 2 with ', 'Quote 3 with ', ' ...']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# li下面有a的\n",
    "s2.xpath('//li[a]/text()').extract()\n",
    "#li 下面有a活着h2的\n",
    "s2.xpath('//li[a or h2]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['link', 'another link', 'Quote 4 title']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取a 和 h2\n",
    "s2.xpath('//a/text()|//h2/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample3 = \"\"\"<html>\n",
    "  <body>\n",
    "    <ul>\n",
    "      <li id=\"begin\"><a href=\"https://scrapy.org\">Scrapy</a>begin</li>\n",
    "      <li><a href=\"https://scrapinghub.com\" xx=\"python\">Scrapinghub</a></li>\n",
    "      <li><a href=\"https://blog.scrapinghub.com\">Scrapinghub Blog</a></li>\n",
    "      <li id=\"end\"><a href=\"http://quotes.toscrape.com\">Quotes To Scrape</a>end</li>\n",
    "      <li data-xxxx=\"end\" abc=\"abc\"><a href=\"http://quotes.toscrape.com\">Quotes To Scrape</a>end</li>\n",
    "    </ul>\n",
    "  </body>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s3 = getxpath(sample3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Scrapy',\n",
       " 'Scrapinghub',\n",
       " 'Scrapinghub Blog',\n",
       " 'Quotes To Scrape',\n",
       " 'Quotes To Scrape']"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取 a标签下 href以https开始的\n",
    "s3.xpath('//a[starts-with(@href, \"http\")]/text()').re('')\n",
    "#获取 href=https://scrapy.org\n",
    "# s3.xpath('//li/a[@href=\"https://scrapy.org\"]/text()').extract()\n",
    "# # #获取 id=begin\n",
    "# s3.xpath('//li[@id=\"begin\"]/text()').extract()\n",
    "# # #获取text=Scrapinghub\n",
    "# s3.xpath('//li/a[text()=\"Scrapinghub\"]/@href').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Scrapinghub']"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s3.xpath('//a[@xx=\"python\"]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取某个标签下 某个参数=xx\n",
    "s3.xpath('//li[@data-xxxx=\"end\"]/text()')\n",
    "s3.xpath('//li[@abc=\"abc\"]/text()')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample4 = u\"\"\"\n",
    "<html>\n",
    "  <head>\n",
    "    <title>My page</title>\n",
    "  </head>\n",
    "  <body>\n",
    "    <h2>Welcome to my <a href=\"https://www.baidu.com\" src=\"x\">page</a></h2>\n",
    "    <p>This is the first paragraph.</p>\n",
    "    <p class=\"test\">\n",
    "            编程语言<a href=\"#\">python</a>\n",
    "        <img src=\"#\" alt=\"test\"/>javascript\n",
    "        <a href=\"#\"><strong>C#</strong>JAVA</a>\n",
    "    </p>\n",
    "    <p class=\"content-a\">a</p>\n",
    "    <p id=\"xxx\" href=\"python\">ssssss</p>\n",
    "    <p class=\"content-c\">c</p>\n",
    "    <p class=\"content-d\">d</p>\n",
    "    <p class=\"econtent-e\">e</p>\n",
    "    <a class=\"econtent-e\">f</a>\n",
    "    <a href=\"https://www.baidu.com\" src=\"x\">page</a>\n",
    "    <a href=\"https://www.demon.com\" src=\"x\">demon</a>\n",
    "    <a href=\"https://www.baidu.com\" src=\"x\">page</a>\n",
    "    <a href=\"https://www.baidu.com\" src=\"x\">page</a>\n",
    "    <!-- this is the end -->\n",
    "  </body>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s4 = getxpath(sample4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ssssss']"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4.css('p#xxx').xpath('text()').re('\\w+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n    编程语言', '\\n    ', 'javascript\\n    ', '\\n    ']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4.xpath('//p[@class=\"test\"]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'编程语言python\\n    javascript\\n    C#JAVA'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取p下面的所有文字\n",
    "s4.xpath('string(//p[@class=\"test\"])').extract_first().strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['demon']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取所有class中有content的\n",
    "s4.xpath('//p[starts-with(@class,\"content\")]/text()').extract()\n",
    "s4.xpath(('//a[contains(@href,\"demo\") and starts-with(@href, \"https\")]/text()')).extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a']"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4.css('.content-a ::text').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a']"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s4.xpath('//p[@class=\"content-a\"]/text()').extract()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参考 ：[AN INTRODUCTION TO XPATH: HOW TO GET STARTED](https://blog.scrapinghub.com/2016/10/27/an-introduction-to-xpath-with-examples/)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from parsel import Selector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z = requests.get('https://www.qiushibaike.com/hot/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "z.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sel = Selector(text=z.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sel.xpath('//div[@class=\"content\"]/span/text() & ').extract_first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sel.css"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}