selenium自动翻页爬取数据信息-Toy模板网

这篇具有很好参考价值的文章主要介绍了selenium自动翻页爬取数据信息。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

爬取数据

import time
from datetime import datetime
import csv
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup as bs
from bs4 import BeautifulSoup
import re
# 驱动路径
driver_path = r'E:\Computer\notebook\TB_Reptile\chromedriver.exe'
browser = None

# 搜索关键词
search_key_words = "固态硬盘"
info_array = []

#爬取网站的页数
number = int(input('请输入爬取页数:'))

def init_driver():
    # 浏览器选项
    options = webdriver.ChromeOptions()
    # 不自动关闭浏览器
    options.add_experimental_option('detach', True)
    # 最大化
    options.add_argument('--start-maximized')

    global browser
    browser = webdriver.Chrome(service=Service(executable_path=driver_path), options=options)


def login():
    # 进入登陆页面
    browser.get('https://login.taobao.com/member/login.jhtml?spm=a21bo.jianhua.0.0.5af911d9mqkcEe&f=top&redirectURL=http%3A%2F%2Fwww.taobao.com%2F')

    print("请在10秒内扫码登陆")
    time.sleep(15)

    now = datetime.now()
    print("login success:", now.strftime("%Y-%m-%d %H:%M:%S"))

def search_info():
    global array
    array = []
    browser.find_element_by_xpath('//*[@id="q"]').send_keys(search_key_words)
    browser.find_element_by_xpath('//*[@id="J_TSearchForm"]/div[1]/button').click()
    divs = browser.find_elements_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div') 
    
    global total_info
    global test
    total_info = []
    count = 1
    while True:
        start = datetime.now()
        print('------------第{}页数据信息开始爬取-------时间：{}-------------'.format(count,start.strftime("%Y-%m-%d %H:%M:%S")))
        
        browser.find_element_by_xpath('//*[@id="J_relative"]/div[1]/div/div[2]/ul/li[3]/a/span').click()
        divs = browser.find_elements_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div')
        html = browser.page_source
        
        test = re.findall(' g_page_config =(.*?);',html)[0]
        title = re.findall('"raw_title":"(.*?)",',test) # 商品标题
        price = re.findall('"view_price":"(.*?)",',test) # 价格
        paynum = re.findall('"view_sales":"(\d+)',test) # 付款人数
        location = re.findall('"item_loc":"(.*?)",',test) # 地点
        name = re.findall('"nick":"(.*?)",',test) # 店铺名
        comment = re.findall('"comment_count":"(.*?)","',test) # 评论数
        delivery = []
        description = []
        service = []
        d1 = re.findall('"delivery":(.*?),',test)
        d2 = re.findall('"description":(.*?),',test)
        s = re.findall('"service":(.*?),',test)
        for j in range(len(d1)):
            delivery.append(d1[j][1:]) # 物流
            description.append(d2[j][1:]) # 描述
            service.append(s[j][1:]) # 服务
        for i in range(len(title)):
            total_info.append([title[i],price[i],paynum[i],location[i],name[i],comment[i],delivery[i],description[i],service[i]])
        endtime = datetime.now()
        print('------------第{}页数据信息爬取完毕-------时间：{}-------------'.format(count,endtime.strftime("%Y-%m-%d %H:%M:%S")))
        
        if count >= number:
            break
        count += 1
        time.sleep(5)

def browser_quit():
    browser.quit()

init_driver()
login()
search_info()
browser_quit()
print("商品总信息数：",len(total_info))

运行结果：

selenium自动翻页爬取数据信息

数据保存

with open ('淘宝.csv',mode='a+',newline='',encoding='gbk') as f:
        f = csv.writer(f)
        f.writerow(['商品标题','价格','付款数','发货地','店铺','评论数','描述评分','物流评分','服务评分'])
    
for i in range(len(total_info)):
    with open ('淘宝.csv',mode='a+',newline='',encoding='gbk') as f:
        f = csv.writer(f)
        f.writerow(total_info[i])
import pandas as pd
data = pd.read_csv('淘宝.csv',encoding='gbk')
data

运行结果：

selenium自动翻页爬取数据信息