实验8：为中英翻译任务搭 Judge 基线

from datasets import load_dataset
import random

# 加载 opus-100 中英子集
ds = load_dataset("Helsinki-NLP/opus-100", "en-zh", split="test")
random.seed(42)
indices = random.sample(range(len(ds)), 200)
samples = [ds[i] for i in indices]

# 展示前 3 条
for s in samples[:3]:
    print(f"ZH: {s['translation']['zh']}")
    print(f"EN (ref): {s['translation']['en']}\n")

from openai import OpenAI
# 伪代码 - 分别用 Model A 和 Model B 生成翻译
model_a = "qwen3-7b-sft-local"
model_b = "gpt-3.5-turbo"

def translate(model, zh_text):
    # 调用对应模型做中译英
    prompt = f"Translate the following Chinese to English:\n{zh_text}"
    return call_model(model, prompt)

pairs = []
for s in samples:
    zh = s["translation"]["zh"]
    ref = s["translation"]["en"]
    trans_a = translate(model_a, zh)
    trans_b = translate(model_b, zh)
    pairs.append({
        "zh": zh, "ref": ref,
        "trans_a": trans_a, "trans_b": trans_b
    })

RUBRIC = """
评分维度（各 1-5 分）：

1. Faithfulness 忠实度
   5 = 所有信息准确完整地传达
   4 = 核心信息正确，极少次要信息偏差
   3 = 核心信息正确，次要信息有遗漏或偏差
   2 = 主要信息部分错误或遗漏
   1 = 主要信息严重错误

2. Fluency 流畅度
   5 = 接近母语水平，自然流畅
   4 = 自然，极少不地道表达
   3 = 可读但存在语法或搭配问题
   2 = 可读性差，需反复理解
   1 = 母语者难以理解

3. Terminology 术语准确性
   5 = 所有术语准确且一致
   4 = 术语基本准确
   3 = 术语基本正确但不统一
   2 = 关键术语错译
   1 = 大多数术语错译

4. Style 文体匹配
   5 = 文体与原文完美匹配
   4 = 文体基本匹配
   3 = 文体大致合适但不精确
   2 = 文体有明显偏差
   1 = 文体完全错配
"""

POINTWISE_PROMPT = f"""你是一位专业的中英翻译质量评估员。

{RUBRIC}

请按以下步骤评估：
Step 1: 识别原文的关键信息点。
Step 2: 对照译文，逐条检查是否保留。
Step 3: 基于证据给出 4 个维度的分数。

**Few-shot 示例：**

原文：北京大学是中国顶尖的综合性大学。
译文：Peking University is one of China's top comprehensive universities.
评分：{{"faithfulness": 5, "fluency": 5, "terminology": 5, "style": 5, 
       "rationale": "完整准确；Peking University 为官方译名；自然流畅。"}}

原文：北京大学是中国顶尖的综合性大学。
译文：Beijing school is top in China.
评分：{{"faithfulness": 2, "fluency": 2, "terminology": 1, "style": 2,
       "rationale": "'school' 错译，漏译'综合性'，用词口语化。"}}

---

**待评：**
原文：{{zh}}
译文：{{translation}}

仅输出 JSON，格式如上。
"""

PAIRWISE_PROMPT = f"""你是一位专业的中英翻译质量评估员。

{RUBRIC}

比较以下两个译文（A 和 B），判断哪个翻译整体更好。
请先给出 4 个维度的简要分析，再给出最终判断。

原文：{{zh}}
译文 A：{{trans_a}}
译文 B：{{trans_b}}

仅输出 JSON：
{{"analysis": "...", "winner": "A" | "B" | "tie", "confidence": 0-1}}
"""

import json

def parse_json(raw):
    raw = raw.strip().removeprefix("```json").removesuffix("```").strip()
    return json.loads(raw)

# 对 5 条样本跑 Pointwise
for p in pairs[:5]:
    raw = call_judge(POINTWISE_PROMPT.format(
        zh=p["zh"], translation=p["trans_a"]
    ))
    result = parse_json(raw)
    print(f"ZH: {p['zh']}")
    print(f"Trans A: {p['trans_a']}")
    print(f"Scores: {result}")
    print("-" * 60)

from collections import Counter
from tqdm import tqdm

def pairwise_judge(judge, zh, trans_a, trans_b):
    """单次 Pairwise 调用"""
    prompt = PAIRWISE_PROMPT.format(zh=zh, trans_a=trans_a, trans_b=trans_b)
    raw = call_judge(judge, prompt)
    result = parse_json(raw)
    return result["winner"]

# === 不带位置校正的原始评测 ===
raw_results = []
for p in tqdm(pairs):
    w = pairwise_judge("gpt-4o", p["zh"], p["trans_a"], p["trans_b"])
    raw_results.append(w)

print("原始结果：", Counter(raw_results))
# 可能看到：Counter({'A': 120, 'B': 70, 'tie': 10}) -> A 明显多，疑似位置偏差

def robust_pairwise(judge, zh, trans_a, trans_b):
    """A/B 交换双跑"""
    v1 = pairwise_judge(judge, zh, trans_a, trans_b)  # 原顺序
    v2 = pairwise_judge(judge, zh, trans_b, trans_a)  # 交换
    
    # 映射到"实际胜者"
    # v1: A 表示 trans_a 在第一位获胜
    # v2: A 表示 trans_b 在第一位获胜（因为交换了）
    
    if v1 == "A" and v2 == "B":
        return "trans_a"  # 两次都选 trans_a
    elif v1 == "B" and v2 == "A":
        return "trans_b"  # 两次都选 trans_b
    elif v1 == v2 and v1 in ("A", "B"):
        return "position_bias"  # 两次都选同一位置
    else:
        return "tie"

# 跑校正版
robust_results = []
for p in tqdm(pairs):
    r = robust_pairwise("gpt-4o", p["zh"], p["trans_a"], p["trans_b"])
    robust_results.append(r)

print("校正后：", Counter(robust_results))
# 预期：Counter({'trans_a': 85, 'trans_b': 80, 'tie': 20, 'position_bias': 15})

bias_rate = sum(1 for r in robust_results if r == "position_bias") / len(robust_results)
print(f"位置偏差率: {bias_rate:.2%}")
# 如果 > 15%，说明 Judge 存在显著位置偏差

# 示例：人工 gold 格式
human_gold = [
    {"pair_id": 0, "annotator_1": "A", "annotator_2": "A",
     "scores_a": [5,4,5,5], "scores_b": [3,3,3,4]},
    # ...
]

# 以多数投票聚合
def majority_vote(labels):
    c = Counter(labels).most_common()
    return c[0][0] if c[0][1] > 1 else "tie"

gold = [{"pair_id": g["pair_id"], 
         "gold": majority_vote([g["annotator_1"], g["annotator_2"]])}
        for g in human_gold]

from scipy.stats import spearmanr, kendalltau
from sklearn.metrics import cohen_kappa_score

# 1. Pairwise 一致率 + Cohen's κ
gold_labels = [g["gold"] for g in gold]
judge_labels = [robust_results[g["pair_id"]] for g in gold]

# 归一化（position_bias 视为 tie）
judge_labels = ["tie" if l == "position_bias" else l for l in judge_labels]
judge_labels = [{"trans_a": "A", "trans_b": "B", "tie": "tie"}[l] for l in judge_labels]

agree = sum(1 for a, b in zip(gold_labels, judge_labels) if a == b) / len(gold)
kappa = cohen_kappa_score(gold_labels, judge_labels)

print(f"Pairwise 一致率: {agree:.2%}")
print(f"Cohen's κ: {kappa:.3f}")

# 2. Pointwise Spearman & Kendall（若同时跑了 Pointwise）
#    假设 judge_scores_a 和 gold_scores_a 是 50 条的 overall 分
rho, _ = spearmanr(judge_scores_a, gold_scores_a)
tau, _ = kendalltau(judge_scores_a, gold_scores_a)
print(f"Spearman ρ: {rho:.3f}")
print(f"Kendall τ:  {tau:.3f}")