跟着AI学Python 第15课：爬虫基础

# === 第15课：爬虫基础 ===

import requests
from bs4 import BeautifulSoup

# 1. 获取网页内容
print("--- 第1段：获取网页 ---")
url = "https://books.toscrape.com/"          # 专门给爬虫练习的网站
response = requests.get(url, timeout=10)
print(f"状态码: {response.status_code}")
print(f"网页内容长度: {len(response.text)} 字符")
print(f"前200字: {response.text[:200]}")

print()

# 2. 解析网页：提取书名和价格
print("--- 第2段：提取数据 ---")
soup = BeautifulSoup(response.text, "lxml")   # 解析HTML

# 找到所有书籍
books = soup.find_all("article", class_="product_pod")
print(f"找到 {len(books)} 本书")
print()

for i, book in enumerate(books[:5], 1):       # 只显示前5本
    title = book.h3.a["title"]                 # 书名在h3下的a标签的title属性里
    price = book.find("p", class_="price_color").text  # 价格
    print(f"{i}. {title} - {price}")

print()

# 3. 提取评分
print("--- 第3段：提取评分 ---")
for i, book in enumerate(books[:5], 1):
    title = book.h3.a["title"]
    rating = book.p["class"][1]                # 评分在p标签的class里
    print(f"{i}. {title} - 评分: {rating}")

print()

# 4. 数据整理：存成列表和字典
print("--- 第4段：整理数据 ---")
book_list = []
for book in books:
    info = {
        "title": book.h3.a["title"],
        "price": book.find("p", class_="price_color").text,
        "rating": book.p["class"][1]
    }
    book_list.append(info)

for b in book_list[:5]:
    print(f"{b['title']} | {b['price']} | {b['rating']}")

print()

# 5. 保存数据到文件
print("--- 第5段：保存数据 ---")
import json

with open(r"D:\python\books.json", "w", encoding="utf-8") as f:
    json.dump(book_list, f, ensure_ascii=False, indent=2)
print(f"已保存 {len(book_list)} 本书到 books.json")

# 保存到CSV（Excel能直接打开）
import csv

with open(r"D:\python\books.csv", "w", encoding="utf-8-sig", newline="") as f:
    writer = csv.DictWriter(f, fieldnames=["title", "price", "rating"])
    writer.writeheader()
    writer.writerows(book_list)
print(f"已保存 {len(book_list)} 本书到 books.csv")

获取网页 → 解析HTML → 提取数据 → 存储
requests    BeautifulSoup   find/find_all   json/csv

未经允许不得转载：百花谷博客 » 跟着AI学Python 第15课：爬虫基础