Python 網頁爬蟲 教學指引

import requests
from bs4 import BeautifulSoup

# 1. 發送請求
url = "https://zh.wikipedia.org/zh-tw/藍染"
headers = {"User-Agent": "Mozilla/5.0"}
response = requests.get(url, headers=headers)

# 2. 解析 HTML
soup = BeautifulSoup(response.text, "html.parser")

# 3. 爬取所有章節標題
for heading in soup.select(".mw-heading h2, .mw-heading h3"):
    print("📌", heading.text.strip())

# 4. 爬取第一張圖片
img = soup.select_one(".mw-body-content img")
if img:
    print("🖼️ 圖片:", "https:" + img["src"])

# 5. 爬取表格與畫廊 (圖片網格) 資料
print("\n📊 表格與畫廊資料：")

# 處理傳統表格 (.wikitable)
for i, table in enumerate(soup.select("table.wikitable"), 1):
    print(f"\n--- 第 {i} 個表格 ---")
    for row in table.find_all("tr"):
        row_data = [col.text.strip() for col in row.find_all(["th", "td"])]
        if any(row_data): print(" | ".join(row_data))

# 處理畫廊 (排版上像表格的圖片網格)
for i, gallery in enumerate(soup.select("ul.gallery"), 1):
    print(f"\n--- 第 {i} 個圖片網格 (Gallery) ---")
    for box in gallery.select(".gallerybox"):
        caption = box.select_one(".gallerytext")
        img = box.select_one("img")
        
        caption_text = caption.text.strip() if caption else "無說明"
        img_url = "https:" + img["src"] if img and img.get("src") else "無圖片"
        print(f"🔹 {caption_text} (圖片: {img_url})")

import requests
from bs4 import BeautifulSoup

url = "https://www.ptt.cc/bbs/movie/index.html"
cookies = {"over18": "1"}

for page in range(3):
    resp = requests.get(url, cookies=cookies)
    soup = BeautifulSoup(resp.text, "html.parser")

    for entry in soup.select(".r-ent"):
        title_el = entry.select_one(".title a")
        if not title_el:
            continue
        push = entry.select_one(".nrec span")
        author = entry.select_one(".meta .author")
        date = entry.select_one(".meta .date")
        print(f"[{push.text if push else ' '}] {title_el.text.strip()} - {author.text if author else 'N/A'}")

    # 取得上一頁的連結
    prev_link = soup.select(".btn-group-paging a")[1]["href"]
    url = "https://www.ptt.cc" + prev_link

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    # 啟動瀏覽器（headless=False 可看到操作過程）
    browser = p.chromium.launch(headless=False)
    page = browser.new_page()

    # 前往目標網頁
    page.goto("https://example.com")

    # 等待特定元素載入
    page.wait_for_selector(".content")

    # 擷取資料
    items = page.query_selector_all(".item")
    for item in items:
        print(item.inner_text())

    # 截圖存證
    page.screenshot(path="result.png")
    browser.close()

from playwright.sync_api import sync_playwright
import csv

with sync_playwright() as p:
    browser = p.chromium.launch(headless=False)
    page = browser.new_page()
    page.goto("https://www.kingstone.com.tw/bestseller/best/book/")

    # 等待書籍列表載入
    page.wait_for_selector(".modProList", timeout=10000)
    page.wait_for_timeout(2000)  # 額外等待確保完全載入

    # 擷取書籍資訊
    books = page.query_selector_all(".modProList")
    results = []
    for i, book in enumerate(books, 1):
        title = book.query_selector(".modProName")
        author = book.query_selector(".modProAuthor")
        price = book.query_selector(".priceset")
        results.append({
            "排名": i,
            "書名": title.inner_text() if title else "N/A",
            "作者": author.inner_text() if author else "N/A",
            "價格": price.inner_text() if price else "N/A",
        })
        print(f"#{i} {results[-1]['書名']}")

    # 截圖 & 關閉
    page.screenshot(path="kingstone_bestseller.png", full_page=True)
    browser.close()

import json
from playwright.sync_api import sync_playwright

def scrape_rethink_details(urls):
    results = []
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)
        page = browser.new_page()
        
        for url in urls:
            print(f"🌍 正在爬取：{url}")
            page.goto(url, wait_until="domcontentloaded", timeout=30000)
            page.wait_for_selector("h1", timeout=15000)
            page.wait_for_timeout(1500) # 等待動態元件渲染
                
            # 1. 垃圾名稱
            title_el = page.query_selector("h1")
            title = title_el.inner_text().strip() if title_el else "未知名稱"
            
            # 2. 價值與說明
            value_els = page.query_selector_all("span, p, div")
            value = ""
            for el in value_els:
                text = el.inner_text().strip()
                if "高回收價值" in text: value = "高回收價值"
                elif "低回收價值" in text: value = "低回收價值"
                if value: break

            desc_els = page.query_selector_all("p.chakra-text")
            description = max([el.inner_text().strip() for el in desc_els], key=len, default="")
                    
            item_data = {
                "name": title,
                "value": value,
                "description": description,
                "url": url
            }
            results.append(item_data)
            
        browser.close()
    return results

# 執行擷取並存成 JSON
urls = ["https://recycle.rethinktw.org/trash/1/", "https://recycle.rethinktw.org/trash/115/"]
data = scrape_rethink_details(urls)
with open("rethink_data.json", "w", encoding="utf-8") as f:
    json.dump(data, f, ensure_ascii=False, indent=2)

import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.extraction_strategy import LLMExtractionStrategy

async def main():
    config = CrawlerRunConfig(
        extraction_strategy=LLMExtractionStrategy(
            provider="openai/gpt-4o-mini",
            instruction="擷取所有商品名稱和價格，以 JSON 格式回傳",
        )
    )
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(
            url="https://example.com/products",
            config=config,
        )
        print(result.extracted_content)

asyncio.run(main())

import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig

async def main():
    config = CrawlerRunConfig(
        word_count_threshold=10,  # 過濾太短的內容
    )
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(
            url="http://127.0.0.1:5500/demo.html", # ⚠️ 請先使用 Live Server 啟動準備好的本地教學靶機網頁
            config=config,
        )
        # 輸出乾淨的 Markdown
        print(result.markdown)

        # 也可以看網頁上所有的連結
        for link in result.links["internal"][:10]:
            print("🔗", link["text"], link["href"])

asyncio.run(main())

import asyncio, json, os
from pydantic import BaseModel, Field
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig
from crawl4ai.extraction_strategy import LLMExtractionStrategy

class Product(BaseModel):
    name: str = Field(description="商品的完整名稱")
    price: str = Field(description="商品的價格數值")
    link: str = Field(description="商品的超連結網址")

# 設定 API Key（或用環境變數）
os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"

async def main():
    llm_config = LLMConfig(provider="gemini/gemini-2.5-flash")
    strategy = LLMExtractionStrategy(
        llm_config=llm_config,
        schema=Product.model_json_schema(),
        instruction="從這個網頁中擷取所有重點商品資訊，並嚴格遵循提供的 schema 回傳 JSON 陣列",
    )
    config = CrawlerRunConfig(
        extraction_strategy=strategy,
        css_selector=".listArea",  # 💡 若解析真實大型電商，可利用此過濾器刪減廣告避免中斷；本地靶機則非必要
    )

    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(
            url="http://127.0.0.1:5500/demo.html", # ⚠️ 針對教學示範本機網頁進行解析
            config=config,
        )
        data = json.loads(result.extracted_content)
        for item in data:
            print(f"📦 {item.get('name', 'N/A')} — 💰 {item.get('price', 'N/A')}")

asyncio.run(main())