上机实验：Qwen2.5-1.5B 的 LoRA 微调

# 建议用 conda 或 venv 新建环境
pip install "torch>=2.2" --index-url https://download.pytorch.org/whl/cu121

# 路线 A：LLaMA-Factory
pip install llamafactory

# 路线 B：TRL
pip install "transformers>=4.45" "peft>=0.13" "trl>=0.12" \
            "datasets>=2.19" "accelerate>=0.34" "bitsandbytes" \
            "sentencepiece" "protobuf"

import torch, transformers, peft, trl
print("torch:", torch.__version__)
print("transformers:", transformers.__version__)
print("peft:", peft.__version__)
print("trl:", trl.__version__)
print("CUDA available:", torch.cuda.is_available())
if torch.cuda.is_available():
    print("GPU:", torch.cuda.get_device_name(0))

from datasets import load_dataset

# 以 Alpaca-GPT4 中文版为例
dataset = load_dataset("llamafactory/alpaca_gpt4_zh", split="train")
print(f"Total: {len(dataset)}")

# 抽样 1000 条
small_ds = dataset.shuffle(seed=42).select(range(1000))

# 转换为 messages 格式（SFT 通用）
def to_messages(example):
    user = example["instruction"]
    if example.get("input", "").strip():
        user += "\n" + example["input"]
    return {
        "messages": [
            {"role": "user", "content": user},
            {"role": "assistant", "content": example["output"]},
        ]
    }

small_ds = small_ds.map(to_messages, remove_columns=small_ds.column_names)

# 过滤过短 / 过长
def ok(ex):
    u, a = ex["messages"][0]["content"], ex["messages"][1]["content"]
    if not u.strip() or not a.strip():
        return False
    if len(a) < 20 or len(u) + len(a) > 4000:
        return False
    return True

small_ds = small_ds.filter(ok)
print(f"After filter: {len(small_ds)}")

# 划分 95/5
split = small_ds.train_test_split(test_size=0.05, seed=42)
train_ds, eval_ds = split["train"], split["test"]

# 存到本地 jsonl
train_ds.to_json("data/train.jsonl", force_ascii=False)
eval_ds.to_json("data/eval.jsonl", force_ascii=False)
print(f"Train: {len(train_ds)}, Eval: {len(eval_ds)}")

trainable params: 18,464,768 || all params: 1,561,174,016 || trainable%: 1.1827

[100/570]  loss=1.248  eval_loss=1.203  lr=1.85e-4
[200/570]  loss=1.105  eval_loss=1.094  lr=1.55e-4
[300/570]  loss=1.021  eval_loss=1.056  lr=1.12e-4
[400/570]  loss=0.962  eval_loss=1.034  lr=0.65e-4
[500/570]  loss=0.928  eval_loss=1.028  lr=0.19e-4

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

base_name = "Qwen/Qwen2.5-1.5B"
adapter_dir = "./qwen2.5-1.5b-sft/final"

tokenizer = AutoTokenizer.from_pretrained(adapter_dir, trust_remote_code=True)

def load(name, adapter=None):
    m = AutoModelForCausalLM.from_pretrained(
        name, torch_dtype=torch.bfloat16, device_map="auto",
        trust_remote_code=True,
    )
    if adapter:
        m = PeftModel.from_pretrained(m, adapter)
    m.eval()
    return m

def chat(model, prompt, max_new=256):
    messages = [{"role": "user", "content": prompt}]
    text = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    inputs = tokenizer(text, return_tensors="pt").to(model.device)
    with torch.no_grad():
        out = model.generate(
            **inputs,
            max_new_tokens=max_new,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id,
        )
    return tokenizer.decode(
        out[0][inputs.input_ids.shape[1]:],
        skip_special_tokens=True,
    )

# ===== 测试 prompts =====
prompts = [
    "用三句话介绍量子计算。",
    "写一个 Python 函数计算斐波那契数列。",
    "请以 JSON 格式列出中国四大发明。",
    "小明有 15 个苹果，给了小红 3 个，又买了 7 个，现在有多少个？",
    "用 Markdown 表格对比 Python 和 Java 的三个优缺点。",
]

# ===== 基座 =====
base = load(base_name)
print("===== BASE =====")
for p in prompts:
    print(f"\n[Q] {p}")
    print(f"[A] {chat(base, p)}")
del base
torch.cuda.empty_cache()

# ===== SFT =====
sft = load(base_name, adapter=adapter_dir)
print("\n===== SFT =====")
for p in prompts:
    print(f"\n[Q] {p}")
    print(f"[A] {chat(sft, p)}")

# 伪代码
judge_prompt = """你是一位严格的评委。下面给出同一个问题和两份回答 A/B。
请从【指令遵循、格式、事实性、流畅度】四个维度 1-5 打分，并选出胜者。

问题：{question}
A：{ans_a}
B：{ans_b}
"""