跟着AI学Python 第16课：爬虫进阶！

# === 第16课：爬虫进阶 ===

import requests
from bs4 import BeautifulSoup
import json
import time

# 1. 翻页抓取：分析URL规律
print("--- 第1段：翻页规律 ---")
# books.toscrape.com 第1页：
# https://books.toscrape.com/catalogue/page-1.html
# 第2页：
# https://books.toscrape.com/catalogue/page-2.html
# 规律：只有页码数字不同

for page in range(1, 4):               # 抓前3页
    url = f"https://books.toscrape.com/catalogue/page-{page}.html"
    print(f"第{page}页URL: {url}")

print()

# 2. 完整翻页抓取
print("--- 第2段：翻页抓取 ---")
all_books = []

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}

for page in range(1, 4):
    url = f"https://books.toscrape.com/catalogue/page-{page}.html"
    try:
        response = requests.get(url, headers=headers, timeout=10)
        if response.status_code != 200:
            print(f"第{page}页请求失败: {response.status_code}")
            continue

        soup = BeautifulSoup(response.text, "lxml")
        books = soup.find_all("article", class_="product_pod")

        for book in books:
            title = book.h3.a["title"]
            price_text = book.find("p", class_="price_color").text
            price = price_text.replace("£", "").replace("£", "")   # 清洗价格
            rating = book.p["class"][1]
            all_books.append({
                "title": title,
                "price": price,
                "rating": rating
            })

        print(f"第{page}页: 抓取了 {len(books)} 本书")
        time.sleep(1)                   # 礼貌等待1秒，不要给服务器太大压力

    except Exception as e:
        print(f"第{page}页出错: {e}")

print(f"\n总共抓取: {len(all_books)} 本书")

print()

# 3. 数据清洗：评分转数字
print("--- 第3段：数据清洗 ---")
rating_map = {
    "One": 1, "Two": 2, "Three": 3,
    "Four": 4, "Five": 5
}

for book in all_books:
    word = book["rating"]
    book["rating_num"] = rating_map.get(word, 0)    # 把英文评分转成数字

# 显示前5本
for b in all_books[:5]:
    print(f"{b['title']} | £{b['price']} | {b['rating']}({b['rating_num']}星)")

print()

# 4. 数据筛选：找高评分低价书
print("--- 第4段：数据筛选 ---")
good_books = [b for b in all_books if b["rating_num"] >= 4 and float(b["price"]) < 30]

print(f"4星以上且价格<£30的书有 {len(good_books)} 本：")
for b in good_books:
    print(f"  {b['title']} | £{b['price']} | {b['rating_num']}星")

print()

# 5. 保存完整数据
print("--- 第5段：保存数据 ---")
with open(r"D:\python\books_all.json", "w", encoding="utf-8") as f:
    json.dump(all_books, f, ensure_ascii=False, indent=2)
print(f"已保存 {len(all_books)} 本书到 books_all.json")

# 保存筛选结果
with open(r"D:\python\books_good.json", "w", encoding="utf-8") as f:
    json.dump(good_books, f, ensure_ascii=False, indent=2)
print(f"已保存 {len(good_books)} 本精选书到 books_good.json")

print()

# 6. 实战：统计每页平均价格
print("--- 第6段：统计 ---")
prices = [float(b["price"]) for b in all_books]
print(f"平均价格: £{sum(prices)/len(prices):.2f}")
print(f"最高价格: £{max(prices):.2f}")
print(f"最低价格: £{min(prices):.2f}")

# 按评分统计数量
from collections import Counter
rating_count = Counter(b["rating_num"] for b in all_books)
for rating in sorted(rating_count):
    print(f"  {rating}星: {rating_count[rating]} 本")