跟着AI学Python 第17课：爬虫请求优化 — Session + 超时重试

# 普通请求：像每次换一个浏览器访问
resp1 = requests.get("https://books.toscrape.com")
print(f"普通请求：状态码 {resp1.status_code}")

# Session请求：像同一个浏览器在访问，自动管理Cookie
session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
})

resp2 = session.get("https://books.toscrape.com")
print(f"Session请求：状态码 {resp2.status_code}")
print(f"Session获得Cookie：{dict(session.cookies)}")

# 第2次请求自动带上Cookie，网站觉得你是"老访客"
resp3 = session.get("https://books.toscrape.com/catalogue/page-2.html")
print(f"第2次Session请求：状态码 {resp3.status_code}")

print("场景1：访问需要登录的供应商网站")
print("  - 普通请求：每次都是'陌生人'，登录状态丢失")
print("  - Session：保持登录状态，登录一次，后面自动带Cookie")
print()
print("场景2：网站根据Cookie给你推荐内容")
print("  - 普通请求：每次看到默认页面")
print("  - Session：网站知道你来过，给你个性化内容")

# 用Session抓一页数据试试
from bs4 import BeautifulSoup

resp = session.get("https://books.toscrape.com", timeout=10)
soup = BeautifulSoup(resp.text, "lxml")
books = soup.find_all("article", class_="product_pod")[:5]

print("用Session抓取前5本书：")
for i, book in enumerate(books, 1):
    name = book.h3.a["title"]
    price = book.find("p", class_="price_color").text
    print(f"  {i}. {name} - {price}")

# 没有超时：如果网站没响应，程序会一直等下去（卡死）
# 设置超时：超过指定秒数就放弃，程序继续运行

try:
    # timeout=15表示最多等15秒
    resp = session.get("https://books.toscrape.com", timeout=15)
    print(f"正常响应：状态码 {resp.status_code}")
except requests.exceptions.Timeout:
    print("请求超时！15秒内没收到响应")
except Exception as e:
    print(f"请求出错：{e}")

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edg/120.0.0.0",
]

def safe_get(url, max_retries=3, timeout=10):
    """安全的请求函数：超时自动重试"""
    for attempt in range(1, max_retries + 1):
        try:
            ua = random.choice(USER_AGENTS)
            headers = {"User-Agent": ua}
            resp = session.get(url, headers=headers, timeout=timeout)
            resp.raise_for_status()           # 状态码非200就抛异常
            return resp
        except requests.exceptions.Timeout:
            print(f"  第{attempt}次超时，重试中……")
        except requests.exceptions.ConnectionError:
            print(f"  第{attempt}次连接失败，重试中……")
        except requests.exceptions.HTTPError as e:
            print(f"  HTTP错误：{e}")
            return None
        except Exception as e:
            print(f"  未知错误：{e}")
            return None

        if attempt < max_retries:
            wait = random.uniform(2, 5)     # 随机等2-5秒再重试
            time.sleep(wait)

    print(f"  重试{max_retries}次均失败")
    return None

# 测试
resp = safe_get("https://books.toscrape.com")
if resp:
    print(f"请求成功！页面大小：{len(resp.text)}字符")

请求 → 成功？→ 返回结果
         ↓失败
      随机等2-5秒 → 重试
         ↓再次失败
      再等再试（最多3次）
         ↓全失败
      返回None，程序继续运行不崩溃

all_books = []

for page in range(1, 4):
    url = f"https://books.toscrape.com/catalogue/page-{page}.html"
    print(f"正在采集第{page}页……")

    resp = safe_get(url)
    if not resp:
        continue

    soup = BeautifulSoup(resp.text, "lxml")
    books = soup.find_all("article", class_="product_pod")

    for book in books:
        name = book.h3.a["title"]
        price = book.find("p", class_="price_color").text
        all_books.append({"name": name, "price": price})

    time.sleep(1)

print(f"\n共抓取 {len(all_books)} 本书")
print("前5本：")
for i, b in enumerate(all_books[:5], 1):
    print(f"  {i}. {b['name']} - {b['price']}")

# 保存到JSON
filepath = os.path.join(DIR, "books_session.json")
with open(filepath, "w", encoding="utf-8") as f:
    json.dump(all_books, f, ensure_ascii=False, indent=2)
print(f"\n已保存到：{filepath}")