您现在的位置：程式師世界 >> 編程語言 > >> 更多編程語言 >> Python

Python爬蟲（五）

編輯：Python

Python爬蟲（五）

一、Seleniums簡述
二、selenium元素定位
三、selenium獲取元素信息
四、selenium_handless
五、request
六、古詩文網案例
七、超級鷹打碼平台的使用

一、Seleniums簡述

作用：
1.應用於web測試的工具
2.其測試直接在浏覽器中，就像真正用戶操作一樣
3.支持各種驅動浏覽器
4.支持無界面浏覽器操作
如何安裝賴？
操作谷歌浏覽器驅動下載地址：https://chromedriver.storage.googleapis.com/index.html

pip install selenium -i https://www.pypi.douban.com/simple

# selenium
# import urllib.request
# url = 'https://www.jd.com/'
# response = urllib.request.urlopen(url)
# content = response.read().decode('utf-8')
# print(content)
# (1) 導入selenium
from selenium import webdriver
# (2) 創建浏覽器操作對象
path = 'chromedriver.exe'
browser = webdriver.Chrome(path)
# (3) 訪問網站
# url = 'https://www.baidu.com'
# browser.get(url)
url = 'https://www.jd.com/'
browser.get(url)
# page_source獲取網頁源碼
content = browser.page_source
print(content)

二、selenium元素定位

# selenium元素定位
from selenium import webdriver
from selenium.webdriver.common.by import By
path = 'chromedriver.exe'
browser = webdriver.Chrome(path)
url = 'https://www.baidu.com'
browser.get(url)
# 元素定位
# 根據id找到對象
# button = browser.find_element(by=By.ID, value='su')
# print(button)
# 根據標簽屬性值獲取對象名
# button = browser.find_element_by_name('wd')
# print(button)
# 根據xpath語句獲取對象
button = browser.find_element_by_xpath('//input[@id="su"]')
print(button)
# 根據標簽名獲取對象
button = browser.find_element_by_tag_name('input')
print(button)
# 使用bs4語法實現
button = browser.find_element_by_css_selector('#su')
button = browser.find_element_by_link_text("直播")
print(button)

三、selenium獲取元素信息

from selenium import webdriver
from selenium.webdriver.common.by import By
path = 'chromedriver.exe'
browser = webdriver.Chrome(path)
url = 'http://www.baidu.com'
browser.get(url)
input = browser.find_element_by_id('su')
# 獲取標簽屬性
print(input.get_attribute('class'))
# 獲取標簽名
print(input.tag_name)
# 獲取元素文本
a = browser.find_element_by_link_text('新聞')
print(a.text)

四、selenium_handless

# selenium_handless
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
# path是個人的chrome浏覽器文件路徑
path = r'C:\Program Files\Google\Chrome\Application\chrome.exe'
chrome_options.binary_location = path
browser = webdriver.Chrome(chrome_options=chrome_options)
url = 'https://www.baidu.com'
browser.get(url)
browser.save_screenshot('./file/baidu.png')

封裝的handless（極其好用）

# 封裝的handless
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def share_browser():
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
# path是個人的chrome浏覽器文件路徑
path = r'C:\Program Files\Google\Chrome\Application\chrome.exe'
chrome_options.binary_location = path
browser = webdriver.Chrome(chrome_options=chrome_options)
return browser
browser = share_browser()
url = 'https://www.baidu.com'
browser.get(url)

五、request

官方文檔：https://doc.codingdict.com/request/docs.python-requests.org/zh_CN/latest/index.html
安裝：pip install requests
requests基本使用
1.一個類型和六個屬性

# requests基本使用
import requests
url = 'http://www.baidu.com'
response = requests.get(url=url)
# 一個類型和六個屬性
# 1.response類型
print(type(response))
# 2.1設置響應的編碼格式
response.encoding = 'utf-8'
# 2.2 以字符串的形式返回網頁源碼
print(response.text)
# 2.3 返回url地址
print(response.url)
# 2.4 返回二進制的數據
print(response.content)
# 2.5 返回響應的狀態碼
print(response.status_code)
# 2.6 返回響應頭
print(response.headers)

2.get請求

import requests
url = 'https://www.baidu.com/s?'
headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44'
}
data = {

'wd':'北京'
}
# url 請求資源路徑
# params 參數
# kwargs 字典
response = requests.get(url=url,params=data,headers=headers)
content = response.text
print(content)
# (1)參數使用params傳遞
# (2)參數無需urlencode編碼
# (3)不需要請求對象的定制
# (4)請求資源路徑中的? 可以添加也可以不添加

3.post請求

import requests
url = 'https://fanyi.baidu.com/sug'
headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44'
}
data = {

'kw':'eye'
}
# url 請求地址
# data 請求參數
# kwargs 字典
response = requests.post(url=url,data=data,headers=headers)
content = response.text
import json
obj = json.loads(content)
print(obj)
# 總結
# (1)post請求不需要解碼
# (2)post請求參數是data
# (3)不需要請求對象的定制

4.request代理
失敗的案例，爬取的頁面被百度安全驗證攔截，目前沒找到解決方法。。

import requests
url = 'https://www.baidu.com/s?'
headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44'
}
data = {

"wd":"ip"
}
proxy = {

'http':'112.6.117.135:8085'
}
response = requests.get(url=url,params=data,headers=headers,proxies=proxy)
content = response.text
with open('./file/daili.html','w',encoding='utf-8')as fp:
fp.write(content)

六、古詩文網案例

# 通過登錄進入到主頁面
# 登錄時需要的參數
# __VIEWSTATE: MhTU6ngpY+d6+v03OI2VLwWkT9WEEg0WJXCgtQVoV3ub3U8WFLzuZ+6GAihlB8lY7d0Ndwv3vVQ1a191DlG8aU65pA604tMI4bfSRa51oYBFQynfi//xkA+oIOw=
# __VIEWSTATEGENERATOR: C93BE1AE
# from: http://so.gushiwen.cn/user/collect.aspx
# email: [email protected]
# pwd: action
# code:
# denglu: 登錄
# 觀察到__VIEWSTATE __VIEWSTATEGENERATOR code 是變量
# 難點:(1) __VIEWSTATE __VIEWSTATEGENERATOR
# (2) 驗證碼
import requests
# 這是登錄頁面的url地址
url = 'https://so.gushiwen.cn/user/login.aspx?from=http://so.gushiwen.cn/user/collect.aspx'
headers = {

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.124 Safari/537.36 Edg/102.0.1245.44'
}
response = requests.get(url=url,headers=headers)
content = response.text
# 解析頁面源碼
from bs4 import BeautifulSoup
soup = BeautifulSoup(content,'lxml')
# 獲取__VIEWSTATE
viewstate = soup.select('#__VIEWSTATE')[0].attrs.get('value')
print(viewstate)
# 獲取__VIEWSTATEGENERATOR
viewstategenerator = soup.select('#__VIEWSTATEGENERATOR')[0].attrs.get('value')
print(viewstategenerator)
# 獲取驗證碼圖片
code = soup.select('#imgCode')[0].attrs.get('src')
code_url = 'https://so.gushiwen.cn' + code
print(code_url)
# 獲取驗證碼的圖片之後下載到本地，觀察圖片內容 在控制台輸入驗證碼
# import urllib.request
# urllib.request.urlretrieve(url=code_url,filename='./file/code.jpg')
# requests裡面有一個方法，session()方法 通過session的返回值使請求變成一個對象
session = requests.session()
# 驗證碼url的內容
response_code = session.get(code_url)
# 注意此時要使用二進制數 圖片的下載
content_code = response_code.content
# wb是將二進制數據寫入文件
with open('./file/code.jpg','wb')as fp:
fp.write(content_code)
code_name = input('請輸入驗證碼')
# 點擊登錄
url_post = 'https://so.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fso.gushiwen.cn%2fuser%2fcollect.aspx'
data_post = {

"__VIEWSTATE": viewstate,
"__VIEWSTATEGENERATOR": viewstategenerator,
"from": "http://so.gushiwen.cn/user/collect.aspx",
"email": "[email protected]",
"pwd": "090711zgf",
"code": code_name,
"denglu": "登錄"
}
response_post = session.post(url=url,headers=headers,data=data_post)
content_post = response_post.text
with open('./file/gushiwen.html','w',encoding='utf-8') as fp:
fp.write(content_post)
# 難點 (1)隱藏域問題 (2)驗證碼

七、超級鷹打碼平台的使用

https://www.chaojiying.com/

#!/usr/bin/env python
# coding:utf-8
import requests
from hashlib import md5
class Chaojiying_Client(object):
def __init__(self, username, password, soft_id):
self.username = username
password = password.encode('utf8')
self.password = md5(password).hexdigest()
self.soft_id = soft_id
self.base_params = {

'user': self.username,
'pass2': self.password,
'softid': self.soft_id,
}
self.headers = {

'Connection': 'Keep-Alive',
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
}
def PostPic(self, im, codetype):
""" im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """
params = {

'codetype': codetype,
}
params.update(self.base_params)
files = {
'userfile': ('ccc.jpg', im)}
r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
return r.json()
def PostPic_base64(self, base64_str, codetype):
""" im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """
params = {

'codetype': codetype,
'file_base64':base64_str
}
params.update(self.base_params)
r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, headers=self.headers)
return r.json()
def ReportError(self, im_id):
""" im_id:報錯題目的圖片ID """
params = {

'id': im_id,
}
params.update(self.base_params)
r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
return r.json()
if __name__ == '__main__':
# 用戶中心>>軟件ID 生成一個替換 96001
chaojiying = Chaojiying_Client('', '', '')
# 本地圖片文件路徑 來替換 a.jpg 有時WIN系統須要//
im = open('a.jpg', 'rb').read()
# 1902 驗證碼類型 官方網站>>價格體系 3.4+版 print 後要加()
print(chaojiying.PostPic(im, 1902).get('pic_str'))
#print chaojiying.PostPic(base64_str, 1902) #此處為傳入 base64代碼