直接获取

import requests

python基于HTTP协议进行网络请求的第三方库

发送请求
1、requests.get(url, /*, headers, params, proxies) — 发送get请求

2、requests.post(url, /*, headers, params, proxies) — 发送post请求

参数:

url — 请求地址(一个网站的网址、接口的地址、图片地址等)

headers — 设置请求头(设置cookie和UserAgent)

params — 设置参数

proxies — 设置代理

发送get请求参数直接拼接到URL中

response = requests.get('http://api.tianapi.com/auto/index?key=c9d408fefd8ed4081a9079d0d6165d43&num=10')

发送post请求，参数设置在params

params = {
    'key':'c9d408fefd8ed4081a9079d0d6165d43',
    'num':10
}
response = requests.post('http://api.tianapi.com/auto/index',params = params)

response = requests.get('http://www.yingjiesheng.com/')

设置编码方式(乱码的时候才需要设置在控制台head标签中查看编码方式)

response.encoding = 'GBK'

获取响应头信息

print(response.headers)

获取响应体

# a.获取text值(用于请求网页，直接拿到网页源代码)
print(response.text)
# b.获取json解析结果(用于返回json数据的数据接口)
print(response.json())
# c.获取content值(获取二进制类型的原数据,用于图片、视频、音频的下载)
print(response.content)

添加请求头

添加User-Agent

headers = {
    # 伪装为浏览器访问
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4542.2 Safari/537.36'
}

获取网页数据

response = requests.get('https://www.51job.com/',headers = headers)

添加cookie

需要登录才能进入的网页添加

headers = {
    # 伪装为浏览器访问
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4542.2 Safari/537.36',
    # 跳过登录
    'cookie':'_zap=1667ed8e-6095-4318-89ec-330c6ef0b8ee; d_c0="AMCve_TnZRCPTh3LhiwHGSuGFq7l4XAr6UE=|1574496707"; _ga=GA1.2.1669928704.1583753947; _9755xjdesxxd_=32; YD00517437729195%3AWM_TID=4dmOpmKKlNhAEUAEBUYqyFtCZ219gaGq; __snaker__id=iRbYMmY5zvFTkQUm; _xsrf=W3e22M3aL6nZ9ZTPAfuxTjelvXUcstnM; YD00517437729195%3AWM_NI=OdDI%2FzdmzhFqywo4cAVWWYPWnNiJrU6p%2FZ6OQ%2BjdzwAOU6Hhd3ew%2Fym8NqBmEq2q%2BwGmAfs605pBNi%2FWHBKGmL9J9OsMdf%2FEaRsTp9tJRBthnh%2Fi3b5l6HTOzQca8GmWSng%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6eea6d880e98ba7afcd68f79a8bb3c44b979f9a85f573aceebe97ca6598b9ba91fb2af0fea7c3b92a8dbc8caaae72a8b3b689c57bbba9a1d5e24b968999bad35dae92a7abf15ff6b0e1bbb54593f5a685f97d83919bd7cb64a8edab98ae73afe88292c16787affe92aa7d85b38f87f23ef4ebc0a3cd63ed8988a3ca4daf94f7abbc45b193ac87ae4fa38ebdb5f333b8969ea3f764aeb8e18ab26af7b4faa9d745ba8fad82bb4daf899d8fe637e2a3; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1628255651,1628596803,1628599663,1628664693; SESSIONID=TTagVSrOcand6lnQK3Tx9P3QfIUJQzCSuwiM8mBdAez; JOID=UFgVB07VQ9GbCjEQQ9PWTZvaJpBQgxe0_VtgYgixIL_7PVp8EqM0x_wPMxRG_xEMMhp1TrjTjeBgzXIl0sn6nZM=; osd=U1sVBEjWQNGYDDITQ9DQTpjaJZZTgBe3-1hjYgu3I7z7Plx_EaM3wf8MMxdA_BIMMRx2TbjQi-NjzXEj0cr6npU=; gdxidpyhxdE=Os7BNOHW1sAPAdCRxVjImRb%5CwvHiXuqyeKWE%2BDdcs5%2Fx4LeRYQ56kPLuEZUvnGyGU0vPHAv35sZ3GWqnI2bOR8Udxg5iLXq6fNMuqkowIZ2Z2%5CZOCRh3PvZu3NluLjDKO9H0HSCg0iYDEXUrbPrKDZ4iZsPvjwbC5pHrj4Niel4djY9B%3A1628665594139; r_cap_id="2|1:0|10:1628664703|8:r_cap_id|44:YWI1NDJjZWMyN2NkNDI3MGI2YWQ3MWFkOTNlYzUwN2Q=|0ac1b35368b1b87333d0d8910f9b90e172f777bd7aba8b213955aeb522ebbf3a"; l_n_c=1; cap_id="ZDM0MjhjMmEyN2Y3NDM4ZGIxMzM2YTY4NThmMThjOTE=|1628664713|5484319ad8db598aff3bdfce4d09effda9863624"; l_cap_id="ZjAwNTAzNjA3YzlmNDUzNWIyN2Y1OWE3M2VmZjFhOWI=|1628664713|eb132e13718f49ec0e33cb23468ea33d39b1027e"; n_c=1; atoken=D6D67B4069B60CE4499DFCBCDA83299B; atoken_expired_in=7776000; client_id="OUQxOTkwRTI4MTQUM3RDQzRTM1M0U=|1628664722|80c987621bc12b757cf3578041cfe972142d55ee"; capsion_ticket="2|1:0|10:1628664723|14:capsion_ticket|44:NWJjNDFjMmIwMzBiNGYyMTg4NTA1MWYxNWFkNjFmYmU=|4393e57fcf2fc0719cfac5005f054d90f3a79973be8d3633fa2dffc23e7231a1"; captcha_session_v2="2|1:0|10:1628664888|18:captcha_session_v2|88:QWUweXlmcXNHeGVBcTJMRXlDcVZmSWdEQW1rdEhrNTdlUXhGYThobm8xalZ3SUNwUmxiM2FoM2diTTBIWnJjSw==|d3176dd52a4897549525550efa2ff752507bb7b98c8db2563812f2b91f51ca6d"; captcha_ticket_v2="2|1:0|10:1628665001|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfSkpWQ243QVFzVlJ0QnFoaElrMUdJSFouY1M0VTFndVZIS0RpTXFXa0ZZUUdLSWNzQjFxT3dPdUtUSTFtZEkxTHFneDZaNFZrUjlsTEZxU1J4aXFHZ1VrMEJPWG91YmZwbVl6MHFSQUtBMm5MbEpyOWZ5NGFtMjR3RGNodC5GYkoycnVlaV9aTEtqc09CMjVDa01xdG12blJ0aE1oZ0NjLVRMV0h1OHJqUFc3MkVtMlJKUXN2aWlRekI5MXRrQkN0OEpLajZlN2phbllBYm1PWjFONXZ5alpZVFBPQzhGZkdaX1FHR18tMjdXcW42bEtqVjRRazdRSk9mVzIxVTBtb1lVcjhSS3p4UWVlODFVSkwyYnE4QmtSRFJLb0RkQTdmeE4uWGd6U0Y2S0dtQlJkTnRnQlc5Y1RMN3NrdERBQUdfU0dUMG51RUNYamI4Tkc0ODFfQllsaHFYYk84ZlRRVy4yejBiYVJDeGpLbHc3eHlLOFNWUXFTWlhod1VpcUx4SERWQWZNN0hpcnM3dFJqYUhHSWtqLXVGVklMNzBJbDhZVU9NLkdYMGpQZDkxN1ktUldlR1hmYmdDTlk4dDhuU25NUmVoNmhsVmxnU0w1WW5hd1BvMW44RUtfbl9kWVRfT2hDVWx6cks3Und1U2hlV3dpeUFhTUxOTVhaMyJ9|8944f13d09a8e61fd716fe8586d7543635409716f0fec0375a18299b844bfc76"; z_c0="2|1:0|10:1628665057|4:z_c0|92:Mi4xRmI0UU1RQUFBQUFBd0s5NzlPZGxFQ1lBQUFCZ0FsVk40Y0lBWWdDSDNQZFhUSW5NT2RtS1JMbWJPMUdTaUxBRmxR|ffeef5b0187eb3b1d598cb47fedb36fe8dfe811d39dc8c6d330dbe9c5bc97ec7"; tst=r; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1628665060; KLBRSID=dc02df4a8178e8c4dfd0a3c8cbd8c726|1628665063|1628664691'
}

User-Agent以及cookie的获取：

cookie在相同位置

json数据解析

除了通过网页源代码获取数据外，可以查看该网页是否由json接口提供数据；

通过json数据接口获取json数据(今日头条)
response = requests.get('https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc&_signature=_02B4Z6wo00f01MS82QAAAIDARL4jQ3jJuyjEmN2AAFBBfDYzO5Ue5Tr6dyHZEvYM5aPjg9xOHE3LKbbaAQksLHvKEx9q4O10B2Py6VQFVZhhEklIQRg.uiWqNZdo-z5rHcpbpKbT0oYa8I0j61')
all_title = response.json()
# print(all_title)
for title in all_title['data']:
    print(title['Title'])

查看json接口：

同User-Agent方法操作后点击Preview查看

图片爬取

import requests
def download_image(url:str):
    # 请求网络图片数据
    response = requests.get(url)
    data = response.content
    # print(data)
    file = open(f'test/images/{url.split("/")[-1]}','wb')
    file.write(data)
if __name__ == '__main__':
    download_image('https://p5.toutiaoimg.com/img/pgc-image/9f5d102756354b6db8fa9408c57d01c8~cs_noop.png')

通过选择器提取数据

from bs4 import BeautifulSoup

准备需要解析的网页数据(实际就是用requests或selenium获取)

data = open('test/practice.html',encoding = 'utf-8').read()

创建BeautifulSoup对象(可自动纠正数据中错误HTML结构)

# BeautifulSoup(数据,解析器)
soup = BeautifulSoup(data,'lxml')

lxml需要下载

通过BeautifulSoup对象获取标签和标签内容

# 1)、获取标签
# BeautifulSoup对象.select(css选择器) --- 获取css选择器中所有当前选择器所有标签(返回列表,列表中的元素是标签对象)
# BeautifulSoup对象.select_one(css选择器) --- 获取css选择器中所有当前选择器第一个标签(返回标签对象)
result = soup.select('p')
print(result)           # [<p>我是段落1</p>, <p>我是段落2</p>, <p>我是超链接3</p>]
result = soup.select_one('p')
print(result)           # <p>我是段落1</p>
result = soup.select('#p1')
print(result)           # [<p id="p1">我是超链接3</p>]
result = soup.select_one('#p1')
print(result)       # <p id="p1">我是超链接3</p
result = soup.select('div p')
print(result)       # [<p>我是段落2</p>, <p id="p1">我是超链接3</p>]
result = soup.select('div>p')
print(result)       # [<p>我是段落2</p>]
# 2)、获取标签内容
# a.标签对象.string --- 获取标签中的文字内容(只有在标签内容是纯文字的时候有效,否则结果为None)
p2 = soup.select_one('div>p')
print(p2)       # <p>我是段落2</p>
print(p2.string)        # '我是段落2'
s1 = soup.select_one('#s1')
print(s1)       # <span id="s1">我是<b>span1</b></span>
print(s1.string)    # None
# b.标签对象.get_text() --- 获取标签内容中所有的文字信息
print(p2.get_text())        # '我是段落2'
print(s1.get_text())        # '我是span1'
# c.标签对象.contents
print(p2.contents)      # ['我是段落2']
result = s1.contents
print(result)      # ['我是', <b>span1</b>]
print(result[-1].get_text())    # 'span1'
# 3)、获取标签属性
# 标签对象.attrs['属性名']
a1 = soup.select_one('div>a')
print(a1)       # <a href="https://www.baidu.com">我是超链接2</a>
print(a1.attrs['href'])     # 'https://www.baidu.com'
img1 = soup.select_one('img')
print(img1)     # <img alt="" src="http://www.gaoimg.com/uploads/allimg/210801/1-210P1151401S1.jpg"/>
print(img1.attrs['src'])        # 'http://www.gaoimg.com/uploads/allimg/210801/1-210P1151401S1.jpg'

上述HTML：

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
</head>
<body>
<p>我是段落1</p>
    <a href="">我是超链接1</a>
    <div>
        <a href="https://www.baidu.com">我是超链接2</a>
        <p>我是段落2</p>
        <span>
            <p id="p1">我是超链接3</p>
        </span>
    </div>
    <img src="http://www.gaoimg.com/uploads/allimg/210801/1-210P1151401S1.jpg" alt="">
    <span id="s1">我是<b>span1</b></span>
</body>
</html>

豆瓣top250爬取练习

导包

from bs4 import BeautifulSoup
import requests
import csv

添加请求头

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4542.2 Safari/537.36'
}

数据爬取函数

# 获取每页数据
def get_info(url = 'https://movie.douban.com/top250'):
    #分别用来存放电影名、电影简评、电影详情路径
    movies_name = []
    movies_view = []
    movies_url = []
    
    # 访问
    response = requests.get(url, headers=headers)
    # print(response.text)
    soup = BeautifulSoup(response.text, 'lxml')
    
    # 获取每个电影的li标签
    movies = soup.select('#content div.article li')
    # print(movies)
    for item in movies:
        movies_name.append(item.select_one('.title').get_text())
        movies_url.append(item.select_one('a').attrs['href'])
        if(item.select_one('.inq') != None):   #get_text()方法对None值无效所以需要判断
            movies_view.append((item.select_one('.inq').get_text()))
        else:
            movies_view.append('暂无')#没有简评的电影用暂无替换
    movies1 = zip(movies_name, movies_view, movies_url)#三个列表中相同下标的数据来自同一电影，将他们整理在一起
    return movies1

保存数据函数

# 将读到的数据写入CSV文件
def write_file(movies1):
    file = open('files/douban/detail.csv', 'a', encoding='utf-8')
    writer = csv.writer(file)
    writer.writerow(['电影名称', '电影描述', '电影详情地址'])  #每页都写一个头
    for it in movies1:
        writer.writerow(it)

主函数

url_list = [25,50,75,100,125,150,175,200,225]
index = -1
url = 'https://movie.douban.com/top250'
while(index < 9):
    if(index == -1 ):
        movies = get_info()
        write_file(movies)
    else:
        movies = get_info(f'https://movie.douban.com/top250?start={url_list[index]}&filter=')
        write_file(movies)
    index += 1
else:
    print('数据写入完成！')

在使用get_text()方法时一定要注意是否存在空值。否则数据不全或无法对号入座。当然程序也会报错！

requests的使用方法