尧图网页设计 
利用selenium库配合谷歌驱动实现自动化pandas库保存网页数据实现方式如下rom bs4 import BeautifulSoup import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time import re import pandas as pd from openpyxl.styles import Border,Side,Font,Alignment,PatternFill import os def write_file(v_df,out_put_flie,md,sheent_name): with pd.ExcelWriter(out_put_flie, engineopenpyxl, modemd) as writer: v_df.to_excel(writer, sheet_namesheent_name, indexFalse) # 输出总体文件 ws writer.sheets[sheent_name] # # 加边框 thin Side(stylethin, color000000) border Border(topthin, rightthin, bottomthin, leftthin) # 给所有表格加边框 for row in ws.iter_rows(min_row1, max_rowws.max_row, min_col1, max_colws.max_column): for cell in row: cell.border border # 自适应列宽处理 for col in ws.columns: col_center col[0].column_letter max_len max(len(str(cell.value)) for cell in col) ws.column_dimensions[col_center].width max_len 6 def acess_new_website(out_put_flie,yp_bh): if os.path.exists(out_put_flie): mda #os.remove(out_put_flie) else: mdw login_url 目标登录页面 USERNAME USERNAME PASSWORD PASSWORD chrome_optionsOptions() chrome_options.binary_locationrC:\chrome.exe #指定谷歌浏览器程序路径 chrome_options.add_argument(--enable-javascript) chrome_options.add_argument(--no-sandbox) chrome_options.add_argument(disable-dev-shm-usage) chrome_options.add_experimental_option(excludeSwitches, [enable-automation]) chrome_options.add_experimental_option(useAutomationExtension, False) #关闭密码保存弹窗 chrome_options.add_experimental_option(prefs,{ credentials_enable_service:False, profile.password_manager_enabled:False }) driver webdriver.Chrome(executable_pathrE:\chromedriver.exe,optionschrome_options) #executable_path 指定谷歌浏览器驱动程序 driver.maximize_window() wait WebDriverWait(driver, 15) try: driver.get(login_url) time.sleep(2) driver.find_element(By.NAME, username).send_keys(USERNAME) driver.find_element(By.NAME, password).send_keys(PASSWORD) time.sleep(3) driver.find_element(By.NAME, password).submit() time.sleep(4) menu1 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()一级菜单]))) #一级菜单按钮 menu1.click() time.sleep(5) menu2 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()二级菜单]))) #二级菜单按钮 menu2.click() time.sleep(5) wait.until(EC.presence_of_element_located((By.XPATH, //div[.//label[contains(text(),筛选字段)]]))) #筛选字段 ypbh_str //div[label筛选字段]//input[placeholder请输入] input_ypbh wait.until(EC.element_to_be_clickable((By.XPATH, ypbh_str))) input_ypbh.clear() input_ypbh.send_keys(yp_bh) time.sleep(2) cx_str //label[.//button/span[normalize-space()查询]]/button #查询按钮 search_btn wait.until(EC.element_to_be_clickable((By.XPATH, cx_str))) search_btn.click() time.sleep(6) # 点击详情专用 # 先定位并勾选单行单选框 click_radio_js var r document.querySelector(.el-table__body-wrapper tbody tr:first-child .el-radio input); if(r){ r.focus(); r.dispatchEvent(new MouseEvent(click,{bubbles: true})); console.log(单选框点击成功); } else { console.log(没找到单选框需要手动点); } driver.execute_script(click_radio_js) time.sleep(3) # 点击查看按钮 ck_str //label[.//button/span[normalize-space()查看]]/button search_btn wait.until(EC.element_to_be_clickable((By.XPATH, ck_str))) search_btn.click() time.sleep(4) menu2 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()一级菜单]))) #按钮链接一级菜单 menu2.click() time.sleep(8) # 打印JS值显示字段值 driver.execute_script( var arr document.querySelectorAll(.el-input__inner,.el-textarea__inner); for(var i0;iarr.length;i){ var d arr[i]; //处理下拉框读取展示文字 var selectWrap d.closest(.el-select); if(selectWrap){ //优先取input_value var showVal d.value || selectWrap.querySelector(.el-input__inner).value || ; d.setAttribute(value,showVal); }else{ d.setAttribute(value,d.value); } } ) # var showVal d.value || selectWrap.querySelector(.el-input__inner).value || ; page_cx driver.page_source soup_cx BeautifulSoup(page_cx, html.parser) #print(soup_cx) # # 查找小项标题字段 cx_all_html soup_cx.select( div[data-v-nametsscCollapseItem]) valid_block [cx for cx in cx_all_html ] # 剔除基本信息页签 df_pivot_new pd.DataFrame() df2_pivot_new pd.DataFrame() total_pivot_df pd.DataFrame() input_col_dict{} for idx,item in enumerate(valid_block): input_col_list [] title_col item.find(span, class_el-tooltip item header-title) # 获取板块标题 #find(span,class_el-tooltip item header-title) #获取标题 #print(title_col) single_table item.select(.myTable) #展示型字段 for num,table in enumerate(single_table): filed_names [] table_header table.select(.el-table__header-wrapper .el-table__header .cell) # 获取字段 for cell in table_header: raw cell.get_text(stripTrue) if raw not in [用户名称,用户编号,]: filed_names.append(raw) #filed_names [cell.get_text(stripTrue) for cell in table_header if cell.get_text(stripTrue) not in [用户名称,用户编号,]] # 表字段列表 #查找字段值 filed_names_len len(filed_names) cell_list table.select(.el-table__body-wrapper .el-table__body .cell) if cell_list: # 有数据则正常取数 data_col [cell.get_text(stripTrue) for cell in cell_list] # 获取表数据 else: #无数据时则按照空列表填充 data_col [] *filed_names_len # 一维列表转二维列表 if filed_names_len 0: data_col_new [data_col[i:i filed_names_len] for i in range(0, len(data_col), filed_names_len)] else: data_col_new [] if len(filed_names) 0 and len(data_col_new) 0: df2 pd.DataFrame(data_col_new, columnsfiled_names) else: df2 pd.DataFrame([], columnsfiled_names) df2_pivot df2.T df2_pivot.reset_index(inplaceTrue) # 无值时新增列 if df2_pivot.shape[1] 0: df2_pivot[0] df2_pivot.rename(columns{index: 新字段, 0: 新字段值}, inplaceTrue) if df2_pivot.shape[1] 3: df2_pivot.rename(columns{1: 新字段值2}, inplaceTrue) df2_pivot.insert(0, 新增字段1, 字段1值) df2_pivot.insert(1, 新增字段2, f字段2值-{title_col.get_text(stripTrue)}) #df2_pivot.insert(0, 序号, f{idx}-{num}) suffix_list [f{idx}-{num}-{i} for i in range(len(df2_pivot))] df2_pivot.insert(0, 序号, suffix_list) #df2_pivot[序号]suffix_list df2_pivot_new pd.concat([df2_pivot_new, df2_pivot], ignore_indexTrue) # 查找输入型字段 xq_lable_data item.select(.el-form-item[prop]) filed_names_input [item.select_one(.el-form-item__label).get_text(stripTrue) for item in xq_lable_data] for item in xq_lable_data: lab item.select_one(.el-form-item__label) # 字段中文名称 inp item.select_one(.el-input__inner) or item.select_one(.el-textarea__inner) if inp: var inp.get(value, ).strip() # 字段值 input_col_dict[lab.get_text(stripTrue)] var print(f生成 {title_col.get_text(stripTrue)} 标题完毕) #只保留本次字段和值 new_col_dict {k:input_col_dict[k] for k in input_col_dict if k in filed_names_input} input_col_list.append(new_col_dict) if len(input_col_list)0: df pd.DataFrame(input_col_list) df_pivot df.T df_pivot.reset_index(inplaceTrue) df_pivot.rename(columns{index: 新字段, 0: 新字段值}, inplaceTrue) df_pivot.insert(0, 新增字段1, 字段1值) df_pivot.insert(1, f新增字段2, f字段2值-{title_col.get_text(stripTrue)}) df_pivot[新字段值2] df_pivot.insert(0, 序号, str(idx)) df_pivot_new pd.concat([df_pivot_new, df_pivot], ignore_indexTrue) total_pivot_df pd.concat([df_pivot_new, df2_pivot_new], ignore_indexTrue) total_pivot_df[临时序号] total_pivot_df[序号].str.split(-).apply(lambda x:tuple(int(i) for i in x)) df2_pivot_new2 total_pivot_df.sort_values(by临时序号) df2_pivot_new2.drop(columns[临时序号],inplaceTrue) write_file(df2_pivot_new2, out_put_flie, w, Sheet1) finally: driver.quit() #pass #入口函数 yp_zl分类 # yp_bh 输入查询字段值# out_put_flie rE:\临时文件\los_output\客户- yp_zl .xlsx acess_new_website(out_put_flie,yp_bh) python自动化提取网页数据（Google浏览器版）
利用selenium库配合谷歌驱动实现自动化pandas库保存网页数据实现方式如下rom bs4 import BeautifulSoup import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time import re import pandas as pd from openpyxl.styles import Border,Side,Font,Alignment,PatternFill import os def write_file(v_df,out_put_flie,md,sheent_name): with pd.ExcelWriter(out_put_flie, engineopenpyxl, modemd) as writer: v_df.to_excel(writer, sheet_namesheent_name, indexFalse) # 输出总体文件 ws writer.sheets[sheent_name] # # 加边框 thin Side(stylethin, color000000) border Border(topthin, rightthin, bottomthin, leftthin) # 给所有表格加边框 for row in ws.iter_rows(min_row1, max_rowws.max_row, min_col1, max_colws.max_column): for cell in row: cell.border border # 自适应列宽处理 for col in ws.columns: col_center col[0].column_letter max_len max(len(str(cell.value)) for cell in col) ws.column_dimensions[col_center].width max_len 6 def acess_new_website(out_put_flie,yp_bh): if os.path.exists(out_put_flie): mda #os.remove(out_put_flie) else: mdw login_url 目标登录页面 USERNAME USERNAME PASSWORD PASSWORD chrome_optionsOptions() chrome_options.binary_locationrC:\chrome.exe #指定谷歌浏览器程序路径 chrome_options.add_argument(--enable-javascript) chrome_options.add_argument(--no-sandbox) chrome_options.add_argument(disable-dev-shm-usage) chrome_options.add_experimental_option(excludeSwitches, [enable-automation]) chrome_options.add_experimental_option(useAutomationExtension, False) #关闭密码保存弹窗 chrome_options.add_experimental_option(prefs,{ credentials_enable_service:False, profile.password_manager_enabled:False }) driver webdriver.Chrome(executable_pathrE:\chromedriver.exe,optionschrome_options) #executable_path 指定谷歌浏览器驱动程序 driver.maximize_window() wait WebDriverWait(driver, 15) try: driver.get(login_url) time.sleep(2) driver.find_element(By.NAME, username).send_keys(USERNAME) driver.find_element(By.NAME, password).send_keys(PASSWORD) time.sleep(3) driver.find_element(By.NAME, password).submit() time.sleep(4) menu1 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()一级菜单]))) #一级菜单按钮 menu1.click() time.sleep(5) menu2 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()二级菜单]))) #二级菜单按钮 menu2.click() time.sleep(5) wait.until(EC.presence_of_element_located((By.XPATH, //div[.//label[contains(text(),筛选字段)]]))) #筛选字段 ypbh_str //div[label筛选字段]//input[placeholder请输入] input_ypbh wait.until(EC.element_to_be_clickable((By.XPATH, ypbh_str))) input_ypbh.clear() input_ypbh.send_keys(yp_bh) time.sleep(2) cx_str //label[.//button/span[normalize-space()查询]]/button #查询按钮 search_btn wait.until(EC.element_to_be_clickable((By.XPATH, cx_str))) search_btn.click() time.sleep(6) # 点击详情专用 # 先定位并勾选单行单选框 click_radio_js var r document.querySelector(.el-table__body-wrapper tbody tr:first-child .el-radio input); if(r){ r.focus(); r.dispatchEvent(new MouseEvent(click,{bubbles: true})); console.log(单选框点击成功); } else { console.log(没找到单选框需要手动点); } driver.execute_script(click_radio_js) time.sleep(3) # 点击查看按钮 ck_str //label[.//button/span[normalize-space()查看]]/button search_btn wait.until(EC.element_to_be_clickable((By.XPATH, ck_str))) search_btn.click() time.sleep(4) menu2 wait.until(EC.element_to_be_clickable((By.XPATH, //*[text()一级菜单]))) #按钮链接一级菜单 menu2.click() time.sleep(8) # 打印JS值显示字段值 driver.execute_script( var arr document.querySelectorAll(.el-input__inner,.el-textarea__inner); for(var i0;iarr.length;i){ var d arr[i]; //处理下拉框读取展示文字 var selectWrap d.closest(.el-select); if(selectWrap){ //优先取input_value var showVal d.value || selectWrap.querySelector(.el-input__inner).value || ; d.setAttribute(value,showVal); }else{ d.setAttribute(value,d.value); } } ) # var showVal d.value || selectWrap.querySelector(.el-input__inner).value || ; page_cx driver.page_source soup_cx BeautifulSoup(page_cx, html.parser) #print(soup_cx) # # 查找小项标题字段 cx_all_html soup_cx.select( div[data-v-nametsscCollapseItem]) valid_block [cx for cx in cx_all_html ] # 剔除基本信息页签 df_pivot_new pd.DataFrame() df2_pivot_new pd.DataFrame() total_pivot_df pd.DataFrame() input_col_dict{} for idx,item in enumerate(valid_block): input_col_list [] title_col item.find(span, class_el-tooltip item header-title) # 获取板块标题 #find(span,class_el-tooltip item header-title) #获取标题 #print(title_col) single_table item.select(.myTable) #展示型字段 for num,table in enumerate(single_table): filed_names [] table_header table.select(.el-table__header-wrapper .el-table__header .cell) # 获取字段 for cell in table_header: raw cell.get_text(stripTrue) if raw not in [用户名称,用户编号,]: filed_names.append(raw) #filed_names [cell.get_text(stripTrue) for cell in table_header if cell.get_text(stripTrue) not in [用户名称,用户编号,]] # 表字段列表 #查找字段值 filed_names_len len(filed_names) cell_list table.select(.el-table__body-wrapper .el-table__body .cell) if cell_list: # 有数据则正常取数 data_col [cell.get_text(stripTrue) for cell in cell_list] # 获取表数据 else: #无数据时则按照空列表填充 data_col [] *filed_names_len # 一维列表转二维列表 if filed_names_len 0: data_col_new [data_col[i:i filed_names_len] for i in range(0, len(data_col), filed_names_len)] else: data_col_new [] if len(filed_names) 0 and len(data_col_new) 0: df2 pd.DataFrame(data_col_new, columnsfiled_names) else: df2 pd.DataFrame([], columnsfiled_names) df2_pivot df2.T df2_pivot.reset_index(inplaceTrue) # 无值时新增列 if df2_pivot.shape[1] 0: df2_pivot[0] df2_pivot.rename(columns{index: 新字段, 0: 新字段值}, inplaceTrue) if df2_pivot.shape[1] 3: df2_pivot.rename(columns{1: 新字段值2}, inplaceTrue) df2_pivot.insert(0, 新增字段1, 字段1值) df2_pivot.insert(1, 新增字段2, f字段2值-{title_col.get_text(stripTrue)}) #df2_pivot.insert(0, 序号, f{idx}-{num}) suffix_list [f{idx}-{num}-{i} for i in range(len(df2_pivot))] df2_pivot.insert(0, 序号, suffix_list) #df2_pivot[序号]suffix_list df2_pivot_new pd.concat([df2_pivot_new, df2_pivot], ignore_indexTrue) # 查找输入型字段 xq_lable_data item.select(.el-form-item[prop]) filed_names_input [item.select_one(.el-form-item__label).get_text(stripTrue) for item in xq_lable_data] for item in xq_lable_data: lab item.select_one(.el-form-item__label) # 字段中文名称 inp item.select_one(.el-input__inner) or item.select_one(.el-textarea__inner) if inp: var inp.get(value, ).strip() # 字段值 input_col_dict[lab.get_text(stripTrue)] var print(f生成 {title_col.get_text(stripTrue)} 标题完毕) #只保留本次字段和值 new_col_dict {k:input_col_dict[k] for k in input_col_dict if k in filed_names_input} input_col_list.append(new_col_dict) if len(input_col_list)0: df pd.DataFrame(input_col_list) df_pivot df.T df_pivot.reset_index(inplaceTrue) df_pivot.rename(columns{index: 新字段, 0: 新字段值}, inplaceTrue) df_pivot.insert(0, 新增字段1, 字段1值) df_pivot.insert(1, f新增字段2, f字段2值-{title_col.get_text(stripTrue)}) df_pivot[新字段值2] df_pivot.insert(0, 序号, str(idx)) df_pivot_new pd.concat([df_pivot_new, df_pivot], ignore_indexTrue) total_pivot_df pd.concat([df_pivot_new, df2_pivot_new], ignore_indexTrue) total_pivot_df[临时序号] total_pivot_df[序号].str.split(-).apply(lambda x:tuple(int(i) for i in x)) df2_pivot_new2 total_pivot_df.sort_values(by临时序号) df2_pivot_new2.drop(columns[临时序号],inplaceTrue) write_file(df2_pivot_new2, out_put_flie, w, Sheet1) finally: driver.quit() #pass #入口函数 yp_zl分类 # yp_bh 输入查询字段值# out_put_flie rE:\临时文件\los_output\客户- yp_zl .xlsx acess_new_website(out_put_flie,yp_bh)