实验 5：构建一个中文领域 SFT 数据集

领域: 中文学术写作润色
覆盖任务类型:
  - T1: 句子级润色（去冗余、改被动、调语序）
  - T2: 段落衔接（补充过渡、修改承接关系）
  - T3: 术语一致性（专有名词首次全称 + 缩写）
  - T4: 学术风格改写（口语化 → 学术化）
  - T5: 参考文献格式转换（GB/T 7714 ↔ APA）
输出长度: assistant 200-800 字
禁忌: 不改变原意、不删除关键信息、保留作者署名引用

{
  "user_msg": "请润色以下句子，使其更符合学术写作规范：\n\n本研究使用了我们自己开发的一个新方法，这个方法很新颖，可以解决以前的一些问题。",
  "assistant_msg": "修改建议：\n\n**原句**：本研究使用了我们自己开发的一个新方法，这个方法很新颖，可以解决以前的一些问题。\n\n**润色**：本研究提出了一种新颖的方法，可有效解决现有工作中的若干关键问题。\n\n**修改理由**：\n1. 删除"我们自己开发"——在学术语境下隐含于"本研究提出"；\n2. "很新颖"改为"新颖的"——避免口语化副词；\n3. "以前的一些问题"替换为"现有工作中的若干关键问题"——更准确、更学术。"
}

import json, time
from openai import OpenAI

client = OpenAI()  # 或 DeepSeek 兼容端点

GEN_PROMPT = """你是一位帮助构建 SFT 训练数据的专家。现在需要为【中文学术写作润色】领域生成一条高质量训练样本。

任务类型：{task_type}
任务说明：{task_desc}

要求：
1. user 消息：构造一个真实研究生可能提出的润色请求（50-300 字）；
2. assistant 消息：给出结构化的润色方案，包含"原句/润色/修改理由"三段；
3. 风格严肃、学术、准确；不要以"作为 AI"开头。

仅返回 JSON：{{"user_msg": "...", "assistant_msg": "..."}}

参考示例：
{few_shot}
"""

TASK_TYPES = {
    "T1": "句子级润色（去冗余、改被动、调语序）",
    "T2": "段落衔接",
    "T3": "术语一致性",
    "T4": "口语化 → 学术化",
    "T5": "参考文献格式转换",
}

def generate_one(task_type, seeds_for_type):
    few_shot = "\n\n".join([json.dumps(s, ensure_ascii=False) for s in seeds_for_type[:2]])
    prompt = GEN_PROMPT.format(
        task_type=task_type,
        task_desc=TASK_TYPES[task_type],
        few_shot=few_shot,
    )
    resp = client.chat.completions.create(
        model="gpt-4o-2024-08-06",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.9,
        response_format={"type": "json_object"},
    )
    return json.loads(resp.choices[0].message.content)

# 为每个任务类型生成 200-300 条
candidates = []
for tt, seeds in seeds_by_type.items():
    for _ in range(250):
        try:
            candidates.append({"task_type": tt, **generate_one(tt, seeds)})
        except Exception as e:
            print(f"skip: {e}")
        time.sleep(0.3)  # 避免速率限制

with open("candidates.jsonl", "w", encoding="utf-8") as f:
    for c in candidates:
        f.write(json.dumps(c, ensure_ascii=False) + "\n")

print(f"生成候选: {len(candidates)}")

JUDGE_PROMPT = """你是一位严格的 SFT 数据质检员。请从以下维度对(指令, 回复)对评分（1-5，5 最好）：
- relevance: 回复是否正确响应了润色请求？
- accuracy: 润色建议是否正确（不破坏原意、符合学术规范）？
- structure: 是否按"原句/润色/修改理由"结构输出？
- fluency: 语言是否自然、准确？
- originality: 是否与示例相似度过高（1=极相似，5=很独特）？

只返回 JSON：{{"relevance": x, "accuracy": x, "structure": x, "fluency": x, "originality": x, "overall": x, "keep": true|false}}

指令：{user}
回复：{resp}
"""

def judge(sample):
    prompt = JUDGE_PROMPT.format(user=sample["user_msg"], resp=sample["assistant_msg"])
    r = client.chat.completions.create(
        model="gpt-4o-2024-08-06",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.0,
        response_format={"type": "json_object"},
    )
    return json.loads(r.choices[0].message.content)

filtered = []
for s in candidates:
    try:
        scores = judge(s)
        s["_scores"] = scores
        if scores.get("overall", 0) >= 4 and scores.get("keep", False):
            filtered.append(s)
    except Exception as e:
        continue

print(f"Judge 通过: {len(filtered)} / {len(candidates)}")

# 语义去重 + 长度过滤
from sentence_transformers import SentenceTransformer
import numpy as np

emb_model = SentenceTransformer("BAAI/bge-m3")
embs = emb_model.encode([s["user_msg"] for s in filtered], normalize_embeddings=True)
keep = np.ones(len(filtered), dtype=bool)
for i in range(len(filtered)):
    if not keep[i]: continue
    sim = embs[i] @ embs[i+1:].T
    keep[i+1:][sim > 0.92] = False

final_candidates = [
    s for s, k in zip(filtered, keep)
    if k
    and 30 <= len(s["user_msg"]) <= 1000
    and 100 <= len(s["assistant_msg"]) <= 2500
]
print(f"最终候选: {len(final_candidates)}")

def to_sharegpt(s):
    return {
        "id": f"academic_polish_{hash(s['user_msg']) % 10**8:08d}",
        "task_type": s["task_type"],
        "conversations": [
            {"from": "human", "value": s["user_msg"]},
            {"from": "gpt",   "value": s["assistant_msg"]},
        ],
        "quality_score": s["_scores"]["overall"],
    }

sharegpt_data = [to_sharegpt(s) for s in final_candidates]

# 保存为 JSONL
with open("academic_polish_sft.jsonl", "w", encoding="utf-8") as f:
    for item in sharegpt_data:
        f.write(json.dumps(item, ensure_ascii=False) + "\n")

print(f"最终数据集: {len(sharegpt_data)} 条")

def to_openai_messages(s):
    return {
        "messages": [
            {"role": "system", "content": "你是一位中文学术写作润色助手。"},
            {"role": "user",   "content": s["user_msg"]},
            {"role": "assistant", "content": s["assistant_msg"]},
        ]
    }

from collections import Counter
dist = Counter(s["task_type"] for s in final_candidates)
print("任务类型分布：")
for k, v in dist.most_common():
    print(f"  {k}: {v} ({v/len(final_candidates)*100:.1f}%)")

def distinct_n(texts, n=3):
    all_ngrams = set()
    total = 0
    for t in texts:
        toks = list(t)  # 中文按字
        grams = [tuple(toks[i:i+n]) for i in range(len(toks)-n+1)]
        all_ngrams.update(grams)
        total += len(grams)
    return len(all_ngrams) / max(total, 1)

assts = [s["assistant_msg"] for s in final_candidates]
print(f"Distinct-3: {distinct_n(assts, 3):.4f}")  # 期望 > 0.4

from sklearn.cluster import KMeans
embs = emb_model.encode(assts, normalize_embeddings=True)
kmeans = KMeans(n_clusters=20, random_state=42, n_init=10).fit(embs)
cluster_sizes = Counter(kmeans.labels_)
max_ratio = max(cluster_sizes.values()) / len(assts)
print(f"最大簇占比: {max_ratio:.2%}（< 15% 为佳）")

import numpy as np
overalls = [s["_scores"]["overall"] for s in final_candidates]
print(f"overall 均值={np.mean(overalls):.2f} p25={np.percentile(overalls, 25):.1f} p50={np.percentile(overalls, 50):.1f}")