Need to fine-tune an LLM?
├── Consumer GPU (24GB VRAM)? → QLoRA (4-bit quantization + LoRA)
├── Multi-GPU cluster? → Full fine-tuning or Spectrum with DeepSpeed ZeRO3
├── Single high-end GPU (48GB+)? → Spectrum (selective layer training)
└── Limited compute budget? → Start with QLoRA, benchmark against base model

# Install dependencies
pip install torch transformers datasets accelerate bitsandbytes trl peft

# Train with YAML config
python run_sft.py --config config.yaml

# Or use TRL CLI directly
trl sft --model_name_or_path meta-llama/Llama-3.1-8B \
  --dataset_name your-dataset --output_dir ./output \
  --per_device_train_batch_size 4 --learning_rate 2e-4 \
  --packing --max_length 1024 --gradient_checkpointing

from transformers import AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
import torch

# 4-bit quantization config
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    quantization_config=bnb_config,
    device_map="auto",
)

# LoRA config
lora_config = LoraConfig(
    r=16,                    # Rank (8-64 typical)
    lora_alpha=16,           # Scaling (usually equal to r)
    target_modules="all-linear",  # Or specific: ["q_proj", "v_proj"]
    modules_to_save=["lm_head", "embed_tokens"],  # Train output layers for new tokens
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)

model = get_peft_model(model, lora_config)

# Generate Spectrum config for your model
python spectrum.py --model-name meta-llama/Llama-3.1-8B --top-percent 30

# Use in training
python run_sft.py --config config.yaml --spectrum_config_path spectrum_config.yaml

# With DeepSpeed ZeRO3
accelerate launch --config_file deepspeed_zero3.yaml \
  --num_processes 8 run_sft.py --config config.yaml

# accelerate config for FSDP + QLoRA
fsdp: "full_shard auto_wrap offload"
fsdp_config:
  backward_prefetch: "backward_pre"
  use_orig_params: "false"

from datasets import load_dataset

def to_prompt_completion(sample):
    return {
        "prompt": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": sample["question"]},
        ],
        "completion": [
            {"role": "assistant", "content": sample["answer"]}
        ]
    }

dataset = load_dataset("your-dataset", split="train")
dataset = dataset.map(to_prompt_completion, remove_columns=dataset.features)

def to_messages(sample):
    return {
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": sample["question"]},
            {"role": "assistant", "content": sample["answer"]}
        ]
    }

# CORRECT - Prompt-completion format (completion_only_loss works)
def to_prompt_completion(example):
    return {
        "prompt": [
            {"role": "system", "content": "You are helpful."},
            {"role": "user", "content": example["input"]},
        ],
        "completion": [
            {"role": "assistant", "content": example["output"]},
        ]
    }

# WRONG - Messages format (completion_only_loss will NOT work)
def to_messages(example):
    return {
        "messages": [
            {"role": "system", "content": "..."},
            {"role": "user", "content": "..."},
            {"role": "assistant", "content": "..."},  # No clear boundary!
        ]
    }

# In SFTConfig (TRL 0.25.0+)
completion_only_loss: true
chat_template_path: "HuggingFaceTB/SmolLM2-360M-Instruct"  # For base models
packing: false  # Required when using completion_only_loss

sample_batch = trainer.data_collator([trainer.train_dataset[0]])
labels = sample_batch["labels"][0]

masked = (labels == -100).sum().item()
total = len(labels)
print(f"Masked (prompt): {masked/total*100:.1f}%")  # Should be 30-60%
print(f"Trained (response): {(total-masked)/total*100:.1f}%")

# In config.yaml
attn_implementation: flash_attention_2  # Flash Attention
use_liger: true                          # Liger Kernels (fused ops)
gradient_checkpointing: true             # Reduce VRAM
gradient_checkpointing_kwargs:
  use_reentrant: false                   # Required for TRL compatibility
packing: true                            # Efficient batching (disable if using completion_only_loss)
torch_dtype: bfloat16                    # Mixed precision

from peft import AutoPeftModelForCausalLM
import torch

model = AutoPeftModelForCausalLM.from_pretrained(
    "path/to/adapter",
    low_cpu_mem_usage=True,
    torch_dtype=torch.float16,
)
model = model.merge_and_unload()
model.save_pretrained("merged-model", safe_serialization=True)

# Using lm-eval harness
lm_eval --model local-chat-completions \
  --tasks gsm8k_cot \
  --model_args model=your-model,base_url=http://localhost:8080/v1/chat/completions \
  --apply_chat_template