{
 "measured": "2026-11-04",
 "by": "Mixpeek",
 "question": "Does fine-tuning an embedding model with LoRA improve retrieval, and how close does it get to full fine-tuning?",
 "setup": {
  "base_model": "BAAI/bge-small-en-v1.5",
  "dataset": "SciFact (BEIR, mteb/scifact)",
  "train": "919 (query, relevant abstract) pairs from the train qrels",
  "test": "300 held-out test queries over the full 5,183-abstract corpus; train and test queries are disjoint",
  "loss": "MultipleNegativesRankingLoss (in-batch negatives), batch 32, 3 epochs, warmup 10%, seed 0, no_duplicates sampler",
  "lora": "r=16, alpha=16, dropout 0.1, task_type FEATURE_EXTRACTION, via SentenceTransformer.add_adapter (sentence-transformers 5.1.1, peft 0.17.1)",
  "query_prefix": "Represent this sentence for searching relevant passages: ",
  "metric": "nDCG@10 and recall@10 by cosine over normalized embeddings",
  "stats": "paired bootstrap over the 300 queries, 10,000 resamples, delta nDCG@10 against the base model",
  "hardware": "Apple silicon Mac (MPS) shared with other jobs; training times are indicative only. The base arm was first scored on CPU and rescored on MPS with the same result (0.7057)."
 },
 "arms": {
  "base": {
   "ndcg@10": 0.7057,
   "recall@10": 0.8296,
   "top1": 0.59,
   "label": "Base model, no training"
  },
  "lora_qv": {
   "ndcg@10": 0.7135,
   "recall@10": 0.8402,
   "top1": 0.6,
   "delta_ndcg": 0.0078,
   "ci95": [
    -0.0023,
    0.0181
   ],
   "better": 27,
   "worse": 15,
   "same": 258,
   "label": "LoRA on query and value layers, lr 1e-4",
   "trainable_params": 294912,
   "trainable_pct": 0.88,
   "weights_on_disk_mb": 1.19,
   "train_s": 637.5,
   "lr": 0.0001
  },
  "lora_all": {
   "ndcg@10": 0.7189,
   "recall@10": 0.8529,
   "top1": 0.6,
   "delta_ndcg": 0.0132,
   "ci95": [
    -0.0013,
    0.028
   ],
   "better": 43,
   "worse": 28,
   "same": 229,
   "label": "LoRA on all linear layers, lr 1e-4",
   "trainable_params": 1339392,
   "trainable_pct": 4.01,
   "weights_on_disk_mb": 5.38,
   "train_s": 2027.3,
   "lr": 0.0001
  },
  "full": {
   "ndcg@10": 0.7507,
   "recall@10": 0.8852,
   "top1": 0.62,
   "delta_ndcg": 0.045,
   "ci95": [
    0.0232,
    0.0671
   ],
   "better": 65,
   "worse": 31,
   "same": 204,
   "label": "Full fine-tuning, lr 2e-5",
   "trainable_params": 33360000,
   "trainable_pct": 100.0,
   "weights_on_disk_mb": 133.46,
   "train_s": 780.5,
   "lr": 2e-05
  },
  "lora_all_hi": {
   "ndcg@10": 0.7444,
   "recall@10": 0.8936,
   "top1": 0.5933,
   "delta_ndcg": 0.0387,
   "ci95": [
    0.0148,
    0.063
   ],
   "better": 62,
   "worse": 38,
   "same": 200,
   "label": "LoRA on all linear layers, lr 1e-3",
   "trainable_params": 1339392,
   "trainable_pct": 4.01,
   "weights_on_disk_mb": 5.38,
   "train_s": 1771.9,
   "lr": 0.001
  }
 }
}