【python】当当网热卖榜页面爬虫练习
君哥
阅读:1697
2023-12-22 17:18:06
评论:0
import requests
from lxml import etree
import csv
import time
import random
import os
os.makedirs('dangdang', exist_ok=True)
writer =csv.writer(open('dangdang/newhotsales.csv', 'w', newline='', encoding='utf-8-sig'))
writer.writerow(['图书名称', '上架时间', '出版社', '价格'])
allUrl = ["http://bang.dangdang.com/books/newhotsales/01.00.00.00.00.00-recent7-0-0-1-{}".format(str(i)) for i in range(1, 10)]
count=0
for url in allUrl:
count = count + 1
print(url)
response = requests.get(url)
response.encoding = 'GB2312'
html = etree.html(response.text)
print(html)
allli = html.xpath( "/html/body/div[@class='bang_wrapper']/div[@class='bang_content']/div[@class='bang_list_box']/ul[@class='bang_list clearfix bang_list_mode']/li")
print(allli)
print('第{}页开始采集'.format(count))
for li in allli:
print(li)
book_name = li.xpath('./div[3]/a[1]/text()')[0]
book_time = li.xpath('./div[5]/a/@title')
book_pub = li.xpath('./div[6]/a[1]/text()')
book_price = li.xpath('./div[7]/p[1]/span[1]/text()')
if book_time:
book_time = book_time[0].replace('/','')
else:
book_time = '无'
if book_pub:
book_pub = book_pub[0]
else:
book_pub = '暂无'
if book_price:
book_price = book_price[0].strip('¥')
else:
book_price = 0
rowInfo = (
book_name,
book_time,
book_pub,
book_price
)
print(rowInfo)
writer.writerow(rowInfo)
print('第{}页采集完成!'.format(count))
time.sleep(random.randint(3, 10))
发表评论