spider_01.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 爬虫简介\n",
    "# 数据是机器学习的燃料，爬虫就是获取数据的挖掘机。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 第一节 直接使用封装好的接口获取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>close</th>\n",
       "      <th>low</th>\n",
       "      <th>volume</th>\n",
       "      <th>price_change</th>\n",
       "      <th>p_change</th>\n",
       "      <th>ma5</th>\n",
       "      <th>ma10</th>\n",
       "      <th>ma20</th>\n",
       "      <th>v_ma5</th>\n",
       "      <th>v_ma10</th>\n",
       "      <th>v_ma20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2018-01-05</th>\n",
       "      <td>2.92</td>\n",
       "      <td>2.94</td>\n",
       "      <td>2.93</td>\n",
       "      <td>2.92</td>\n",
       "      <td>3220088.25</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.34</td>\n",
       "      <td>2.906</td>\n",
       "      <td>2.887</td>\n",
       "      <td>2.875</td>\n",
       "      <td>3906197.70</td>\n",
       "      <td>4670167.18</td>\n",
       "      <td>4674675.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-04</th>\n",
       "      <td>2.91</td>\n",
       "      <td>2.93</td>\n",
       "      <td>2.92</td>\n",
       "      <td>2.91</td>\n",
       "      <td>3084130.75</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.34</td>\n",
       "      <td>2.892</td>\n",
       "      <td>2.883</td>\n",
       "      <td>2.871</td>\n",
       "      <td>4165531.45</td>\n",
       "      <td>4922446.00</td>\n",
       "      <td>4812177.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-03</th>\n",
       "      <td>2.91</td>\n",
       "      <td>2.95</td>\n",
       "      <td>2.91</td>\n",
       "      <td>2.91</td>\n",
       "      <td>5708743.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.874</td>\n",
       "      <td>2.879</td>\n",
       "      <td>2.868</td>\n",
       "      <td>5265180.70</td>\n",
       "      <td>5021250.08</td>\n",
       "      <td>4890998.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-01-02</th>\n",
       "      <td>2.86</td>\n",
       "      <td>2.91</td>\n",
       "      <td>2.91</td>\n",
       "      <td>2.86</td>\n",
       "      <td>4776970.00</td>\n",
       "      <td>0.05</td>\n",
       "      <td>1.75</td>\n",
       "      <td>2.870</td>\n",
       "      <td>2.875</td>\n",
       "      <td>2.868</td>\n",
       "      <td>5108968.20</td>\n",
       "      <td>4969266.33</td>\n",
       "      <td>5051746.58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            open  high  close   low      volume  price_change  p_change  \\\n",
       "date                                                                      \n",
       "2018-01-05  2.92  2.94   2.93  2.92  3220088.25          0.01      0.34   \n",
       "2018-01-04  2.91  2.93   2.92  2.91  3084130.75          0.01      0.34   \n",
       "2018-01-03  2.91  2.95   2.91  2.91  5708743.00          0.00      0.00   \n",
       "2018-01-02  2.86  2.91   2.91  2.86  4776970.00          0.05      1.75   \n",
       "\n",
       "              ma5   ma10   ma20       v_ma5      v_ma10      v_ma20  \n",
       "date                                                                 \n",
       "2018-01-05  2.906  2.887  2.875  3906197.70  4670167.18  4674675.88  \n",
       "2018-01-04  2.892  2.883  2.871  4165531.45  4922446.00  4812177.54  \n",
       "2018-01-03  2.874  2.879  2.868  5265180.70  5021250.08  4890998.78  \n",
       "2018-01-02  2.870  2.875  2.868  5108968.20  4969266.33  5051746.58  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tushare获取股票数  \n",
    "import pandas as pd\n",
    "import tushare as ts\n",
    "\n",
    "data = ts.get_hist_data('510050',start='2018-01-01',end='2018-01-05')\n",
    "#data.index =  \n",
    "#data = data[::-1]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 第二节 requests库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-5-049b72533498>, line 1)",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-5-049b72533498>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    http://www.w3school.com.cn/# 官网：http://cn.python-requests.org/zh_CN/latest/\u001b[0m\n\u001b[0m        ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "http://www.w3school.com.cn/# 官网：http://cn.python-requests.org/zh_CN/latest/\n",
    "# 一句话简介：Requests 唯一的一个非转基因的 Python HTTP 库，人类可以安全享用。\n",
    "# 不服的话请自行挑战urllib库。\n",
    "# 安装：pip install requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 常用HTTP请求的功能\n",
    "# Get: 请求指定的资源\n",
    "# Head: 检查资源是否存在\n",
    "# Put: 更新资源 \n",
    "# Delete: 删除资源\n",
    "# Post: 上传资源和表单"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# HTTP请求结构：\n",
    "# https://www.xxx.com/yyy?v1=a&v2=b\n",
    "# 协议名称是https\n",
    "# url是www.xxx.com/yyy\n",
    "# 有2个参数v1和v2，对应值是a和b。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# http://httpbin.org/是个很好的测试和学习对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'>\n",
      "{\"one\": 1, \"two\": 2}\n",
      "<class 'dict'>\n",
      "{'one': 1, 'two': 2}\n"
     ]
    }
   ],
   "source": [
    "# 关于json格式字符串的处理\n",
    "import json\n",
    "\n",
    "data = {'one': 1, 'two': 2}\n",
    "s = json.dumps(data) # 字典 -> json格式字符串\n",
    "print(type(s))\n",
    "print(s)\n",
    "jdata = json.loads(s) # json格式字符串 -> 字典\n",
    "print(type(jdata))\n",
    "print(jdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get请求\n",
    "import requests\n",
    "r = requests.get(\"http://httpbin.org/get\")\n",
    "print(r.status_code) # 返回HTTP状态码\n",
    "print(r.encoding) # 响应内容的编码\n",
    "print(r.headers) # 响应的头部信息\n",
    "print(r.text)\n",
    "jdata = r.json()\n",
    "print(type(jdata))\n",
    "print(jdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://httpbin.org/get?key2=value2&key1=value1\n",
      "{'key2': 'value2', 'key1': 'value1'}\n"
     ]
    }
   ],
   "source": [
    "# 带参数的get请求\n",
    "payload = {'key1': 'value1', 'key2': 'value2'}\n",
    "r = requests.get(\"http://httpbin.org/get\", params=payload)\n",
    "print(r.url)\n",
    "print(r.json()['args'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'key2': 'value2', 'key1': 'value1'}\n"
     ]
    }
   ],
   "source": [
    "# Post请求\n",
    "payload = {'key1': 'value1', 'key2': 'value2'}\n",
    "r = requests.post(\"http://httpbin.org/post\", data=payload)\n",
    "print(r.json()['form'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Pragma': 'no-cache', 'Expires': 'Sun, 1 Jan 2006 01:00:00 GMT', 'X-DAE-App': 'sns', 'X-DOUBAN-NEWBID': '7bF3-MPLFmA', 'Keep-Alive': 'timeout=30', 'Strict-Transport-Security': 'max-age=15552000;', 'X-Xss-Protection': '1; mode=block', 'Vary': 'Accept-Encoding', 'Connection': 'keep-alive', 'Content-Encoding': 'gzip', 'Cache-Control': 'must-revalidate, no-cache, private', 'Server': 'dae', 'Content-Type': 'text/html; charset=utf-8', 'X-DAE-Node': 'daisy4b', 'X-Douban-Mobileapp': '0', 'Date': 'Wed, 31 Jan 2018 02:33:30 GMT', 'Transfer-Encoding': 'chunked', 'Set-Cookie': 'll=\"108288\"; path=/; domain=.douban.com; expires=Thu, 31-Jan-2019 02:33:29 GMT, bid=7bF3-MPLFmA; Expires=Thu, 31-Jan-19 02:33:30 GMT; Domain=.douban.com; Path=/'}\n",
      "bid 7bF3-MPLFmA\n",
      "ll \"108288\"\n"
     ]
    }
   ],
   "source": [
    "# Cookie处理\n",
    "r = requests.get('http://www.douban.com')\n",
    "print(r.headers)\n",
    "for k in r.cookies.keys():\n",
    "    print(k, r.cookies[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 二进制内容处理\n",
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "r = requests.get('http://image1.ljcdn.com/shfdfs-image/m_fit,w_960,q_90/20180127/81244784-d394-47f4-8a2c-77826b8f9e0f.jpg')\n",
    "image = Image.open(BytesIO(r.content))\n",
    "image.save('lianjia.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "万 科Ａ: 36.500, 37.380, 37.900, 35.800, \n"
     ]
    }
   ],
   "source": [
    "# 使用requests库从新浪获取股票数据\n",
    "def get_stock_info(code):\n",
    "    api = 'http://hq.sinajs.cn/list='\n",
    "    r = requests.get(api + code)\n",
    "    data_str = r.text.split('=')[1].replace('\"', '')[:-2]\n",
    "    infos = data_str.split(',')\n",
    "    s = infos[0] + ': '\n",
    "    s += infos[1] + ', ' # 开\n",
    "    s += infos[3] + ', ' # 收\n",
    "    s += infos[4] + ', ' # 高\n",
    "    s += infos[5] + ', ' # 低\n",
    "    print(s)\n",
    "    \n",
    "get_stock_info('sz000002')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 第3节 css定位器与bs4网页解析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# HTML基础知识\n",
    "# 对于HTML/CSS/JavaScript零基础同学：http://www.w3school.com.cn/\n",
    "# <标签 属性_1=值_1 属性_2=值_2>内容...</标签>\n",
    "# id和class作为2个特殊标签被特别对待"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# css选择器（可以使用浏览器的开发者工具检查或生成选择器）\n",
    "# 选择标签：直接输入标签名字，如head/ul，用 > 表示层级关系，比如ul > li（当中的空格必须保留）。\n",
    "# 选择id：#id，比如百度搜索的“百度一下”按钮可以用 #su 定位。\n",
    "# 选择class: .classname，比如百度搜索的搜索框可以用 .s_ipt 定位。\n",
    "# id和class的区别：理论上id是唯一的。而一个标签则可以同时属于多个class，可以用 .class_1.class_2 表示选择该标签的2个class进行匹配。\n",
    "# 元素属性选择：[属性=值]，多个属性的话直接连接即可，[属性_1=值_1][属性_1=值_2]\n",
    "# 例子，选择以下元素<div id=su class=\"name1 name2\" size=100 show=popup>Some Text</div>\n",
    "# - div#su.name1.name2[size=100][show=popup]\n",
    "# - 因为id一般唯一，所以 #su 也足够表达。div.name1 如果能唯一识别也可以用的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bs4网页解析\n",
    "# 安装：pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ISO-8859-1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'百度一下'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 结合requests找到百度提交按钮\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "resp = requests.get('http://www.baidu.com')\n",
    "print(resp.encoding)\n",
    "# encode: 字符串 -> 二进制字节\n",
    "# decode: 二进制字节 -> 字符串\n",
    "text = resp.text.encode('iso-8859-1').decode('utf-8') # 编码问题校正\n",
    "bs = BeautifulSoup(text, 'html5lib') # 模式使用lxml，但是html属于不严谨的xml。\n",
    "su = bs.select('#su')[0] # BeautifulSoup的select是找到所有满足条件的元素并返回数组\n",
    "su.get('value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "北京市\n",
      "\t通县, 101100\n",
      "\t东城区, 100010\n",
      "\t西城区, 100032\n",
      "\t崇文区, 100061\n",
      "\t宣武区, 100054\n",
      "\t朝阳区, 100020\n",
      "\t丰台区, 100071\n",
      "\t石景山区, 100043\n",
      "\t海淀区, 100080\n",
      "\t门头沟区, 102300\n",
      "\t房山区, 102400\n",
      "\t通州区, 101100\n",
      "\t顺义区, 101300\n",
      "\t昌平区, 102200\n",
      "\t大兴区, 102600\n",
      "\t怀柔区, 101400\n",
      "\t平谷区, 101200\n",
      "\t密云区, 101500\n",
      "\t延庆区, 102100\n",
      "\t北京市, 100000\n"
     ]
    }
   ],
   "source": [
    "# 抓取全国各地邮政编码与长途区号（自己加一点代码）\n",
    "BASE_URL = 'http://www.ip138.com'\n",
    "UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'\n",
    "HEADERS = {'user-agent':UA}\n",
    "\n",
    "def handle_zip_code(province, url):\n",
    "    print(province)\n",
    "    resp = requests.get(url, headers=HEADERS)\n",
    "    text = resp.text.encode('iso-8859-1').decode('gbk')\n",
    "    bs = BeautifulSoup(text, 'html5lib')\n",
    "    rows = bs.select('table.t12 > tbody > tr')\n",
    "    for i in range(1, len(rows)):\n",
    "        items = rows[i].select('td')\n",
    "        t_1 = items[0].text.strip()\n",
    "        if t_1:\n",
    "            print('\\t%s, %s' % (t_1, items[1].text.strip()))\n",
    "        if len(items) > 4:\n",
    "            t_2 = items[3].text.strip()\n",
    "            if t_2:\n",
    "                print('\\t%s, %s' % (t_2, items[4].text.strip()))\n",
    "    \n",
    "resp = requests.get(BASE_URL + '/post', headers=HEADERS)\n",
    "text = resp.text.encode('iso-8859-1').decode('gbk')\n",
    "bs = BeautifulSoup(text, 'html5lib')\n",
    "for item in bs.select('div#newAlexa > table.t4 > tbody > tr > td > a'):\n",
    "    handle_zip_code(item.text, BASE_URL + item.get('href'))\n",
    "    break # 不然显示内容太多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tr>\n",
      "   <td>row 1, cell 1</td>\n",
      "   <td>row 1, cell 2</td>\n",
      "</tr>, <tr>\n",
      "   <td>row 2, cell 1</td>\n",
      "   <td>row 2, cell 2</td>\n",
      "</tr>]\n"
     ]
    }
   ],
   "source": [
    "# 定位器的小坑\n",
    "text = ''\n",
    "with open('sample.html', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "bs = BeautifulSoup(text, 'html5lib')\n",
    "rows = bs.select('table > tbody > tr')\n",
    "print(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Selenium安装配置\n",
    "# 1. 安装python的selenium包：pip install selenium\n",
    "# 2. Selenium驱动（Chrome）下载：https://sites.google.com/a/chromium.org/chromedriver/downloads\n",
    "# 3. 驱动下载后放在任意位置，但需要把路径添加到PATH环境变量。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m                                             \u001b[0mclose_fds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'Windows'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m                                             stdout=self.log_file, stderr=self.log_file)\n\u001b[0m\u001b[1;32m     75\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds)\u001b[0m\n\u001b[1;32m    946\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m    948\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m   1550\u001b[0m                                 \u001b[0merr_msg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m': '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_executable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1551\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1552\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'chromedriver'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-800893f503c0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# options.add_argument('--disable-gpu')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# options.add_argument('--ignore-certificate-errors')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mdriver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mChrome\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchrome_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://www.baidu.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0melement\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'#su'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/chrome/webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, executable_path, port, chrome_options, service_args, desired_capabilities, service_log_path)\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mservice_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mservice_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m             log_path=service_log_path)\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m                 raise WebDriverException(\n\u001b[1;32m     80\u001b[0m                     \"'%s' executable needs to be in PATH. %s\" % (\n\u001b[0;32m---> 81\u001b[0;31m                         os.path.basename(self.path), self.start_error_message)\n\u001b[0m\u001b[1;32m     82\u001b[0m                 )\n\u001b[1;32m     83\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEACCES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWebDriverException\u001b[0m: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# 使用示例\n",
    "from selenium import webdriver\n",
    "options = webdriver.ChromeOptions() # 创建driver的参数，可选。\n",
    "# options.add_argument('--headless')\n",
    "# options.add_argument('--disable-gpu')\n",
    "# options.add_argument('--ignore-certificate-errors')\n",
    "driver = webdriver.Chrome(chrome_options=options)\n",
    "driver.get('http://www.baidu.com')\n",
    "element = driver.find_element_by_css_selector('#su')\n",
    "print(element.get_attribute('value'))\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m                                             \u001b[0mclose_fds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'Windows'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m                                             stdout=self.log_file, stderr=self.log_file)\n\u001b[0m\u001b[1;32m     75\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds)\u001b[0m\n\u001b[1;32m    946\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m    948\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m   1550\u001b[0m                                 \u001b[0merr_msg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m': '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_executable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1551\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1552\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'chromedriver'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-ed510bb1b344>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mdriver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mChrome\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaximize_window\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://www.baidu.com'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/chrome/webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, executable_path, port, chrome_options, service_args, desired_capabilities, service_log_path)\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mservice_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mservice_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m             log_path=service_log_path)\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m                 raise WebDriverException(\n\u001b[1;32m     80\u001b[0m                     \"'%s' executable needs to be in PATH. %s\" % (\n\u001b[0;32m---> 81\u001b[0;31m                         os.path.basename(self.path), self.start_error_message)\n\u001b[0m\u001b[1;32m     82\u001b[0m                 )\n\u001b[1;32m     83\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEACCES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWebDriverException\u001b[0m: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# 模拟百度搜索\n",
    "import time\n",
    "\n",
    "driver = webdriver.Chrome()\n",
    "driver.maximize_window()\n",
    "driver.get('http://www.baidu.com')\n",
    "time.sleep(1)\n",
    "search_input = driver.find_element_by_css_selector('input#kw')\n",
    "search_input.send_keys('python')\n",
    "time.sleep(1)\n",
    "submit = driver.find_element_by_css_selector('input#su')\n",
    "submit.click() # 如果click无效，可以尝试driver.execute_script('arguments[0].click();', element)\n",
    "time.sleep(1)\n",
    "print(driver.current_url)\n",
    "results = driver.find_elements_by_css_selector('h3 > a')\n",
    "for result in results:\n",
    "    print(result.text, result.get_attribute('href'))\n",
    "driver.quit()\n",
    "\n",
    "# 思考题：百度格式的url如何转换成标准url？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     73\u001b[0m                                             \u001b[0mclose_fds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplatform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'Windows'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m                                             stdout=self.log_file, stderr=self.log_file)\n\u001b[0m\u001b[1;32m     75\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds)\u001b[0m\n\u001b[1;32m    946\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 947\u001b[0;31m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m    948\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m   1550\u001b[0m                                 \u001b[0merr_msg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m': '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_executable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1551\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1552\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'chromedriver'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-85-9f6b8b2cad04>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_argument\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'--headless'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 如果截图不正确请先检查option\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_argument\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'--disable-gpu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdriver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mChrome\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchrome_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 带options的初始化\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'http://news.sina.com.cn'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mwidth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_script\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'return Math.max(document.body.scrollWidth, document.body.offsetWidth, document.documentElement.clientWidth, document.documentElement.scrollWidth, document.documentElement.offsetWidth);'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/chrome/webdriver.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, executable_path, port, chrome_options, service_args, desired_capabilities, service_log_path)\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mservice_args\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mservice_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m             log_path=service_log_path)\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/selenium/webdriver/common/service.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m                 raise WebDriverException(\n\u001b[1;32m     80\u001b[0m                     \"'%s' executable needs to be in PATH. %s\" % (\n\u001b[0;32m---> 81\u001b[0;31m                         os.path.basename(self.path), self.start_error_message)\n\u001b[0m\u001b[1;32m     82\u001b[0m                 )\n\u001b[1;32m     83\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEACCES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWebDriverException\u001b[0m: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# 利用selenium制作截图工具\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('--headless') # 如果截图不正确请先检查option\n",
    "options.add_argument('--disable-gpu')\n",
    "driver = webdriver.Chrome(chrome_options=options) # 带options的初始化\n",
    "driver.get('http://news.sina.com.cn')\n",
    "width = driver.execute_script('return Math.max(document.body.scrollWidth, document.body.offsetWidth, document.documentElement.clientWidth, document.documentElement.scrollWidth, document.documentElement.offsetWidth);')\n",
    "height = driver.execute_script('return Math.max(document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight);')\n",
    "print(width, height)\n",
    "driver.set_window_size(width, height) # 告诉driver截图大小区域\n",
    "driver.save_screenshot('sina_news.jpg')\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-27-67fac9b889cc>, line 6)",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-27-67fac9b889cc>\"\u001b[0;36m, line \u001b[0;32m6\u001b[0m\n\u001b[0;31m    element = driver.find_element_by_css_selector(css):\u001b[0m\n\u001b[0m                                                      ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# 编程注意事项！\n",
    "# find_element(s)_by_xxx找不到？\n",
    "# 实际编程建议如下封装\n",
    "def find_element_by_css_selector(driver, css): # driver可以是driver，也可以是某个element\n",
    "    try:\n",
    "        element = driver.find_element_by_css_selector(css):\n",
    "    except:\n",
    "        element = None\n",
    "    return element\n",
    "\n",
    "def find_elements_by_css_selector(driver, css): # driver可以是driver，也可以是某个element\n",
    "    try:\n",
    "        element = driver.find_elements_by_css_selector(css):\n",
    "    except:\n",
    "        element = []\n",
    "    return element\n",
    "# 创建driver建议如此封装\n",
    "def create_chrome_driver(options):\n",
    "    chrome_options = webdriver.ChromeOptions()\n",
    "    for option in options:\n",
    "        chrome_options.add_argument(option)\n",
    "    driver = webdriver.Chrome(chrome_options=chrome_options)\n",
    "    return driver\n",
    "# 如何等待异步元素出现\n",
    "i = 0\n",
    "while True:\n",
    "    element = find_element_by_css_selector(driver, css)\n",
    "    if element:\n",
    "        break\n",
    "    else:\n",
    "        time.sleep(1)\n",
    "        if i >= 10:\n",
    "            break\n",
    "        else:\n",
    "            i += 1\n",
    "# 元素正确找到，但是无法获取内容文本。比如类似以下html：\n",
    "# <div id=content>\n",
    "#     <i>text \n",
    "#     some text<br>\n",
    "# </div>\n",
    "# element.text返回None，怎么办？\n",
    "# element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 作业\n",
    "# 1. 从http://qq.ip138.com/train/入口进入，找到与上海虹桥相关的列车时刻表。\n",
    "#    从入口界面进入后，找到上海，再找到虹桥，然后拿到相关的列车时刻表。\n",
    "# 2. 利用selenium实现豆瓣用户名/密码登陆，\n",
    "#    如果有验证码，提示用户在命令行交互界面输入，然后再由selenium提交验证码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}