文章目录1、构建请求头2、提取数据3、保存数据爬取豆瓣电影Top250数据包括电影的电影名、导演、演员等基本信息以及海报图片、剧情简介和评论数量。运行截图如下1、构建请求头总共有10页每页25条电影数据page_start为每页的起始位置如第一页为0第二页为25因此想要爬取全部页数的数据只用从0遍历到250以25为步长即可即range(0, 250, 25)。请求头可以使用https://curlconverter.com/快速构建使用方法可访问https://blog.csdn.net/Pangaoyang_/article/details/140873357?spm1001.2014.3001.5502cookies { ll: 118282, bid: qpeBkdWNQ30, __utma: 30149280.1285408772.1722931171.1722931171.1722931171.1, __utmc: 30149280, __utmz: 30149280.1722931171.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/, __utmt: 1, __utmb: 30149280.1.10.1722931171, __utma: 223695111.549597820.1722931184.1722931184.1722931184.1, __utmb: 223695111.0.10.1722931184, __utmc: 223695111, __utmz: 223695111.1722931184.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/, _pk_ref.100001.4cf6: %5B%22%22%2C%22%22%2C1722931184%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D, _pk_id.100001.4cf6: 39e7e842a6abee49.1722931184., _pk_ses.100001.4cf6: 1, ap_v: 0,6.0, __yadk_uid: 5tRoftzrzq0L8EylRtLcRgAgQ8c6kVkb, } headers { accept: text/html,application/xhtmlxml,application/xml;q0.9,image/avif,image/webp,image/apng,*/*;q0.8,application/signed-exchange;vb3;q0.7, accept-language: zh-CN,zh;q0.9,en;q0.8,en-GB;q0.7,en-US;q0.6, # cookie: ll118282; bidqpeBkdWNQ30; __utma30149280.1285408772.1722931171.1722931171.1722931171.1; __utmc30149280; __utmz30149280.1722931171.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/; __utmt1; __utmb30149280.1.10.1722931171; __utma223695111.549597820.1722931184.1722931184.1722931184.1; __utmb223695111.0.10.1722931184; __utmc223695111; __utmz223695111.1722931184.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/; _pk_ref.100001.4cf6%5B%22%22%2C%22%22%2C1722931184%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D; _pk_id.100001.4cf639e7e842a6abee49.1722931184.; _pk_ses.100001.4cf61; ap_v0,6.0; __yadk_uid5tRoftzrzq0L8EylRtLcRgAgQ8c6kVkb, priority: u0, i, referer: https://movie.douban.com/top250, sec-ch-ua: Not)A;Brand;v99, Microsoft Edge;v127, Chromium;v127, sec-ch-ua-mobile: ?0, sec-ch-ua-platform: Windows, sec-fetch-dest: document, sec-fetch-mode: navigate, sec-fetch-site: same-origin, sec-fetch-user: ?1, upgrade-insecure-requests: 1, user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0, } params { start: f{page_start}, # 每页起始位置 filter: , }2、提取数据用XPath提取网页数据在提取的时候发现演员数据放在了JS中因此这部分需要用正则表达式提取。response requests.get(https://movie.douban.com/top250, paramsparams, cookiescookies, headersheaders) html etree.HTML(response.text) hrefs html.xpath(//*[idcontent]/div/div[1]/ol/li/div/div[2]/div[1]/a/href) # 详情链接 # 遍历每页的所有电影 for href in hrefs: response requests.get(href, cookiescookies, headersheaders) html2 etree.HTML(response.text) title html2.xpath(//*[idcontent]/h1/span[1]/text())[0] # 电影名 year html2.xpath(//*[idcontent]/h1/span[2]/text())[0][1:-1] # 年份 director html2.xpath(//*[idinfo]/span[1]/span[2]/a/text())[0] # 导演 try: writer html2.xpath(//*[idinfo]/span[2]/span[2]/a/text())[0] # 编剧 except: # 没有编剧 writer plot html2.xpath(//*[idinfo]/span[5]/text())[0] # 剧情 score html2.xpath(//*[idinterest_sectl]/div[1]/div[2]/strong/text())[0] # 评分 synopsis html2.xpath(//*[idlink-report-intra]/span/text()) # 剧情介绍 comment_number html2.xpath(//*[idcomments-section]/div[1]/h2/span/a/text())[0].split( )[1] # 评论数 image_url html2.xpath(//*[idmainpic]/a/img/src)[0] # 图片地址 # 包含地区、语言、又名 temps html2.xpath(//*[idinfo]/text()) # 地区 new_temp [] for temp in temps: temp temp.replace( , ) # 清除空格 if ( not in temp) and (temp ! /) and (temp ! ): # 筛选出换行符、空字符串和/ new_temp.append(temp) area new_temp[0] # 地区 language new_temp[1] # 语言 alias new_temp[-2] # 别名 # 正则提取演员名 data re.findall(script typeapplication/ldjson(.*?)/script, response.text, re.DOTALL)[0] # 正则匹配 data data.replace( , ) # 替换换行符 data data.replace( , ) # 一个特殊字符串 data json.loads(data) # 转换为字典 actors [] for act in data[actor]: # 遍历演员列表 actors.append(act[name])3、保存数据数据存储此处使用pandas模块直接保存为了.xlsx文件可以很方便地使用Excel查看也可以根据需要选择保存到数据库中。data [title, year, director, writer, actors, area, plot, language, alias, score, synopsis, comment_number, image_url] columns [电影名, 年份, 导演, 编剧, 主演, 地区, 剧情, 语言, 又名, 评分, 剧情介绍, 评论数, 海报地址] result pd.DataFrame(all_data, columnscolumns) result.to_excel(豆瓣电影数据.xlsx, indexTrue)保存结果如下完整代码如下import requests from lxml import etree import re import json import pandas as pd cookies { ll: 118282, bid: qpeBkdWNQ30, __utma: 30149280.1285408772.1722931171.1722931171.1722931171.1, __utmc: 30149280, __utmz: 30149280.1722931171.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/, __utmt: 1, __utmb: 30149280.1.10.1722931171, __utma: 223695111.549597820.1722931184.1722931184.1722931184.1, __utmb: 223695111.0.10.1722931184, __utmc: 223695111, __utmz: 223695111.1722931184.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/, _pk_ref.100001.4cf6: %5B%22%22%2C%22%22%2C1722931184%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D, _pk_id.100001.4cf6: 39e7e842a6abee49.1722931184., _pk_ses.100001.4cf6: 1, ap_v: 0,6.0, __yadk_uid: 5tRoftzrzq0L8EylRtLcRgAgQ8c6kVkb, } headers { accept: text/html,application/xhtmlxml,application/xml;q0.9,image/avif,image/webp,image/apng,*/*;q0.8,application/signed-exchange;vb3;q0.7, accept-language: zh-CN,zh;q0.9,en;q0.8,en-GB;q0.7,en-US;q0.6, # cookie: ll118282; bidqpeBkdWNQ30; __utma30149280.1285408772.1722931171.1722931171.1722931171.1; __utmc30149280; __utmz30149280.1722931171.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/; __utmt1; __utmb30149280.1.10.1722931171; __utma223695111.549597820.1722931184.1722931184.1722931184.1; __utmb223695111.0.10.1722931184; __utmc223695111; __utmz223695111.1722931184.1.1.utmcsrcn.bing.com|utmccn(referral)|utmcmdreferral|utmcct/; _pk_ref.100001.4cf6%5B%22%22%2C%22%22%2C1722931184%2C%22https%3A%2F%2Fcn.bing.com%2F%22%5D; _pk_id.100001.4cf639e7e842a6abee49.1722931184.; _pk_ses.100001.4cf61; ap_v0,6.0; __yadk_uid5tRoftzrzq0L8EylRtLcRgAgQ8c6kVkb, priority: u0, i, referer: https://movie.douban.com/top250, sec-ch-ua: Not)A;Brand;v99, Microsoft Edge;v127, Chromium;v127, sec-ch-ua-mobile: ?0, sec-ch-ua-platform: Windows, sec-fetch-dest: document, sec-fetch-mode: navigate, sec-fetch-site: same-origin, sec-fetch-user: ?1, upgrade-insecure-requests: 1, user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0, } all_data [] def acquire_movie(page_start): params { start: f{page_start}, # 每页起始位置 filter: , } response requests.get(https://movie.douban.com/top250, paramsparams, cookiescookies, headersheaders) html etree.HTML(response.text) hrefs html.xpath(//*[idcontent]/div/div[1]/ol/li/div/div[2]/div[1]/a/href) # 详情链接 # 遍历每页的所有电影 for href in hrefs: response requests.get(href, cookiescookies, headersheaders) html2 etree.HTML(response.text) title html2.xpath(//*[idcontent]/h1/span[1]/text())[0] # 电影名 year html2.xpath(//*[idcontent]/h1/span[2]/text())[0][1:-1] # 年份 director html2.xpath(//*[idinfo]/span[1]/span[2]/a/text())[0] # 导演 try: writer html2.xpath(//*[idinfo]/span[2]/span[2]/a/text())[0] # 编剧 except: # 没有编剧 writer plot html2.xpath(//*[idinfo]/span[5]/text())[0] # 剧情 score html2.xpath(//*[idinterest_sectl]/div[1]/div[2]/strong/text())[0] # 评分 synopsis html2.xpath(//*[idlink-report-intra]/span/text()) # 剧情介绍 comment_number html2.xpath(//*[idcomments-section]/div[1]/h2/span/a/text())[0].split( )[1] # 评论数 image_url html2.xpath(//*[idmainpic]/a/img/src)[0] # 图片地址 # 包含地区、语言、又名 temps html2.xpath(//*[idinfo]/text()) # 地区 new_temp [] for temp in temps: temp temp.replace( , ) # 清除空格 if ( not in temp) and (temp ! /) and (temp ! ): # 筛选出换行符、空字符串和/ new_temp.append(temp) area new_temp[0] # 地区 language new_temp[1] # 语言 alias new_temp[-2] # 别名 # 正则提取演员名 data re.findall(script typeapplication/ldjson(.*?)/script, response.text, re.DOTALL)[0] # 正则匹配 data data.replace( , ) # 替换换行符 data data.replace( , ) # 一个特殊字符串 data json.loads(data) # 转换为字典 actors [] for act in data[actor]: # 遍历演员列表 actors.append(act[name]) data [title, year, director, writer, actors, area, plot, language, alias, score, synopsis, comment_number, image_url] all_data.append(data) print(data) if __name__ __main__: for i in range(0, 250, 25): print(f--------第{i//25 1}页---------) acquire_movie(i) columns [电影名, 年份, 导演, 编剧, 主演, 地区, 剧情, 语言, 又名, 评分, 剧情介绍, 评论数, 海报地址] result pd.DataFrame(all_data, columnscolumns) result.to_excel(豆瓣电影数据.xlsx, indexTrue)