实验9：在秘塔搜索上做一次 GEO 小型实证

queries = [
    # 5 条直接查询（与你标题高度相关）
    "LoRA 微调 Qwen3 显存占用",
    "LoRA r=32 和 r=8 显存差多少",
    ...
    # 5 条同义变体（不同措辞表达相同意图）
    "Qwen3 LoRA 训练需要多少显存",
    ...
    # 5 条长尾查询（更具体的子问题）
    "LoRA alpha 32 显存会怎么变化",
    ...
    # 5 条横向对比查询（与你内容竞争但不完全重合）
    "LoRA vs QLoRA 显存对比",
    ...
]
assert len(queries) == 20

import json
from datetime import datetime
from pathlib import Path

# ===== 1. 加载查询集 =====
queries = json.loads(Path("queries.json").read_text(encoding="utf-8"))

# ===== 2. 记录模板 =====
def record_query(query: str, engine: str = "metaso") -> dict:
    """
    对每条 query，在秘塔搜索上手动查询后填写以下字段。
    建议每条 query 重复 3 次取最稳定的结果。
    """
    return {
        "query": query,
        "engine": engine,
        "timestamp": datetime.now().isoformat(),
        "answer_text": "",        # 粘贴秘塔完整答案文本
        "all_cited_urls": [],     # 答案中所有编号引用的 URL 列表
        "my_url": "",             # 你自己博文的 URL
        "my_cited": False,        # 是否被引用
        "my_rank": None,          # 被引用时的编号位次（1-based；未引用填 None）
        "my_word_count": 0,       # 你内容中有多少词元出现在答案里（人工估计或脚本计算）
        "competitor_cited": [],   # 竞争内容被引的 URL
        "notes": "",              # 主观观察
    }

# ===== 3. 为每条 query 手动填写 =====
baseline_records = []
for q in queries:
    for repeat in range(3):       # 每条重复 3 次
        rec = record_query(q)
        # ... 手动填入 ...
        baseline_records.append(rec)

Path("baseline.json").write_text(
    json.dumps(baseline_records, ensure_ascii=False, indent=2),
    encoding="utf-8"
)

def position_adjusted_wc(answer_text: str, my_content_tokens: list[str]) -> float:
    """
    统计 my_content_tokens 中有多少出现在 answer_text 中，
    并按位置倒数加权。
    """
    answer_tokens = answer_text.split()  # 简化：按空格切；中文可用 jieba
    pawc = 0.0
    for my_tok in my_content_tokens:
        if my_tok in answer_tokens:
            pos = answer_tokens.index(my_tok) + 1  # 1-based
            pawc += 1.0 / pos
    return pawc

import statistics

def summarize(records):
    grouped = {}
    for r in records:
        grouped.setdefault(r["query"], []).append(r)

    summary = {}
    for q, runs in grouped.items():
        cited = [r["my_cited"] for r in runs]
        ranks = [r["my_rank"] for r in runs if r["my_rank"] is not None]
        pawc = [r.get("pawc", 0) for r in runs]
        summary[q] = {
            "cite_rate": sum(cited) / len(cited),
            "mean_rank": statistics.mean(ranks) if ranks else None,
            "mean_pawc": statistics.mean(pawc),
        }
    return summary

baseline_summary = summarize(baseline_records)

def compare(baseline_summary, post_summary):
    comparison = []
    for q in baseline_summary:
        b = baseline_summary[q]
        p = post_summary.get(q, {})
        comparison.append({
            "query": q,
            "cite_rate_delta": p.get("cite_rate", 0) - b["cite_rate"],
            "rank_delta": (p.get("mean_rank") or 99) - (b["mean_rank"] or 99),
            "pawc_delta": p.get("mean_pawc", 0) - b["mean_pawc"],
        })
    return comparison

comparison = compare(baseline_summary, post_summary)

# 统计检验
from scipy import stats
cite_rate_pre = [baseline_summary[q]["cite_rate"] for q in baseline_summary]
cite_rate_post = [post_summary[q]["cite_rate"] for q in post_summary]
t_stat, p_value = stats.ttest_rel(cite_rate_pre, cite_rate_post)
print(f"paired t-test: t={t_stat:.3f}, p={p_value:.4f}")

import matplotlib.pyplot as plt

fig, axes = plt.subplots(1, 3, figsize=(15, 4))

# 被引率分布
axes[0].hist([c["cite_rate_delta"] for c in comparison], bins=10)
axes[0].set_title("Cite Rate Delta per Query")
axes[0].axvline(0, color='red', linestyle='--')

# 位次变化
axes[1].hist([c["rank_delta"] for c in comparison], bins=10)
axes[1].set_title("Rank Delta per Query")

# PAWC 变化
axes[2].hist([c["pawc_delta"] for c in comparison], bins=10)
axes[2].set_title("PAWC Delta per Query")

plt.tight_layout()
plt.savefig("geo_experiment_result.png", dpi=150)
plt.show()