Fine-tuning 实战：LoRA / QLoRA 原理与工程实践

# 数据格式（OpenAI / HuggingFace 标准）
training_data = [
    {
        "messages": [
            {"role": "system", "content": "你是客服助手，按 JSON 格式回复"},
            {"role": "user", "content": "我要退款"},
            {"role": "assistant", "content": '{"action": "refund_request", "needs_clarification": true, "questions": ["请提供订单号"]}'}
        ]
    },
    # ... 至少 1000-5000 条
]

# 数据质量检查
def quality_check(data):
    issues = []
    for i, sample in enumerate(data):
        # 1. 格式校验
        if not validate_format(sample):
            issues.append(f"Sample {i}: invalid format")
        # 2. 长度检查
        if len(json.dumps(sample)) > 8000:
            issues.append(f"Sample {i}: too long")
        # 3. 多样性检查
        if i > 0 and similarity(sample, data[i-1]) > 0.95:
            issues.append(f"Sample {i}: too similar to previous")
    return issues

from peft import LoraConfig

lora_config = LoraConfig(
    # 核心超参
    r=16,                    # 秩：8/16/32/64，越大越能拟合复杂任务
    lora_alpha=32,           # 通常 2*r
    lora_dropout=0.05,       # 防止过拟合
    bias="none",             # 不训练 bias
    
    # 目标模块（按模型架构）
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",     # attention
        "gate_proj", "up_proj", "down_proj"          # MLP
    ],
    
    # 任务类型
    task_type="CAUSAL_LM"
)

from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",           # NF4 量化
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,      # Double Quantization
    bnb_4bit_quant_storage="nf4",        # 优化器状态分页
)

# 训练参数
from transformers import TrainingArguments

training_args = TrainingArguments(
    output_dir="./output",
    
    # 学习率
    learning_rate=2e-4,                  # QLoRA 标准
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    
    # 训练轮数
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,       # effective batch size = 16
    
    # 优化
    optim="paged_adamw_8bit",            # 8-bit 优化器
    
    # 监控
    logging_steps=10,
    save_steps=500,
    eval_steps=100,
    
    # 防止过拟合
    load_best_model_at_end=True,
    metric_for_best_model="eval_loss",
)

from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

# 加载模型（4-bit 量化）
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B-Instruct",
    quantization_config=bnb_config,
    device_map="auto",
)

# 准备 k-bit 训练
model = prepare_model_for_kbit_training(model)

# LoRA 配置
lora_config = LoraConfig(r=16, lora_alpha=32, target_modules="all-linear", lora_dropout=0.05, bias="none", task_type="CAUSAL_LM")

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: 4,194,304 || all params: 8,030,261,248 || trainable%: 0.052%

# 训练
trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    peft_config=lora_config,
    max_seq_length=2048,
)

trainer.train()
trainer.save_model("./final-adapter")

from peft import PeftModel

base = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
peft_model = PeftModel.from_pretrained(base, "./final-adapter")
merged = peft_model.merge_and_unload()
merged.save_pretrained("./merged-model")

# vLLM 支持多 LoRA
from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest

llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", enable_lora=True)

# 不同任务加载不同 adapter
outputs = llm.generate(
    prompts,
    SamplingParams(temperature=0.7),
    lora_request=LoRARequest("customer_service", 1, "./cs-adapter")
)
outputs2 = llm.generate(
    prompts,
    SamplingParams(temperature=0.7),
    lora_request=LoRARequest("code_review", 2, "./code-adapter")
)

fine_tune_metrics = {
    # 1. 任务指标
    "exact_match_rate": "输出与标准答案完全一致的比例",
    "format_compliance_rate": "符合格式要求（JSON / 结构）的比例",
    
    # 2. 质量指标（LLM-as-Judge）
    "style_adherence": "风格一致性",
    "task_accuracy": "任务准确率",
    
    # 3. 退化指标（重要！）
    "general_capability_regression": "通用能力是否下降",
    "幻觉率": "微调后是否引入幻觉"
}

# 1. 训练时就监控 eval loss
if eval_loss > train_loss * 1.5:
    print("⚠️ 可能过拟合，考虑早停")

# 2. 用通用能力测试集测
general_test = load_dataset("mmlu")  # 或其他通用测试
regression_score = evaluate(model, general_test)
if regression_score < baseline - 0.05:
    print("⚠️ 通用能力下降超过 5%")

# 用大模型（teacher）的输出训练小模型（student）
teacher = "gpt-4o"
student = "Llama-3.1-8B"

# 1. 大模型生成大量样本
training_data = []
for question in questions_pool:
    response = call_gpt4(question)  # teacher
    training_data.append({"input": question, "output": response})

# 2. 小模型 fine-tune
train(student, training_data)  # student