{
  "id": "colqwen2-v1-0-merged",
  "identity": {
    "family": "colpali",
    "display_name": "ColQwen2 v1.0 (merged)",
    "hub_id": "vidore/colqwen2-v1.0-merged",
    "api_name": null,
    "revision": "8076c402380ee60b54663fbfdea2f6a9feb8ee03",
    "variants": [],
    "license": "Apache-2.0",
    "commercial_use": true,
    "commercial_use_note": "Apache-2.0 on the repo; the base Qwen2-VL-2B is Apache-2.0.",
    "release_date": "2024-11",
    "paper_url": "https://arxiv.org/abs/2407.01449",
    "repo_url": "https://github.com/illuin-tech/colpali"
  },
  "task": {
    "category": "embedding",
    "modalities_in": [
      "image",
      "document",
      "text"
    ],
    "modalities_out": [
      "multi_vector"
    ],
    "best_at": "Text search over document page images (PDF pages, slides, infographics, tables) by late interaction, with no OCR step.",
    "worst_at": "Video frames and product photos: committed eval nDCG@10 0.133 on MSR-VTT keyframes and 0.170 on product images, below single-vector SigLIP. About 750 x 128-d vectors per page make storage and cold reads expensive."
  },
  "io": {
    "input": {
      "image": {
        "max_visual_tokens": 768,
        "patch": "28 px merged patches",
        "max_pixels": 602112
      },
      "text": "query text, one vector per token"
    },
    "output": {
      "vector": {
        "dims": 128,
        "distance": "cosine",
        "type": "multi_dense"
      }
    }
  },
  "mixpeek_wiring": {
    "extractors": [
      {
        "extractor": "colqwen_extractor@v1",
        "selected_by": "fixed"
      }
    ],
    "feature_uris": [
      "mixpeek://colqwen_extractor@v1/colqwen2_v1"
    ],
    "vector_indexes": [
      "colqwen_extractor_v1_tokens"
    ],
    "retriever_stages": [
      "maxsim_rerank",
      "feature_search"
    ],
    "composes_with": [
      "PDF pages or frames -> ColQwen2 patch matrices -> maxsim_rerank over a SigLIP or E5 first stage"
    ]
  },
  "run": {
    "status": "served",
    "status_reason": "Ray Serve app in the inference registry; colqwen_extractor batch calls it (ColQwen2EmbeddingBatchService, a ServeProxyProcessor). The embedding registry marks it PREVIEW.",
    "serve_apps": [
      "vidore__colqwen2"
    ],
    "runtime": "transformers Qwen2VLModel + Qwen2VLImageProcessor + AutoTokenizer, 1_Dense projection to 128 (ColQwen2Encoder, engine code); no colpali-engine dependency",
    "weights_format": "safetensors",
    "trust_remote_code": false,
    "size_on_disk_mb": 4419,
    "params_millions": 2209,
    "preprocessing": "RGB, resized by the Qwen2-VL processor to at most 768 visual tokens; bf16 (or fp16) on CUDA, fp32 on CPU.",
    "postprocessing": "Project every image patch or query token to 128 dims and L2-normalize; one matrix per page, scored by MaxSim.",
    "serving_shape": "both",
    "batch_size": 8,
    "serve_batch": null,
    "scale_to_zero": {
      "min_replicas": 0,
      "max_replicas": 1,
      "idle_timeout_s": 300,
      "memory_gb": 12,
      "num_cpus": 4,
      "num_gpus": 0,
      "cold_start_s": {
        "warm_node_s": null,
        "new_node_s": null,
        "source": null
      },
      "cpu_fallback": true
    }
  },
  "cost": {
    "unmeasured": true,
    "measured": null,
    "note": "No committed per-item throughput, latency, memory and cost measurement of the Serve app."
  },
  "evidence": {
    "unmeasured": false,
    "items": [
      {
        "dataset": "ViDoRe v1 + v2 document pages, 1,500 English queries (93,992-item mixed corpus)",
        "metric": "nDCG@10, exact f32 MaxSim (vidore/colqwen2-v1.0 via colpali-engine, bf16 on L4)",
        "k": 10,
        "score": 0.696,
        "baseline": "SigLIP base alone, top 10",
        "baseline_score": 0.135,
        "held_out": false,
        "date": "2026-10-15",
        "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md"
      },
      {
        "dataset": "MSR-VTT 1K-A captions over scene-cut keyframes (1,000 queries)",
        "metric": "nDCG@10, exact f32 MaxSim",
        "k": 10,
        "score": 0.133,
        "baseline": "SigLIP base alone, top 10",
        "baseline_score": 0.14,
        "held_out": false,
        "date": "2026-10-15",
        "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md"
      },
      {
        "dataset": "Marqo/KAGL product images, 1,000 product-title queries",
        "metric": "nDCG@10, exact f32 MaxSim",
        "k": 10,
        "score": 0.17,
        "baseline": "SigLIP base alone, top 10",
        "baseline_score": 0.293,
        "held_out": false,
        "date": "2026-10-15",
        "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md"
      }
    ]
  },
  "refs": [
    "hf://vidore/colqwen2-v1.0-merged@8076c402380ee60b54663fbfdea2f6a9feb8ee03"
  ],
  "formats": [
    {
      "ref": "hf://vidore/colqwen2-v1.0-merged@8076c402380ee60b54663fbfdea2f6a9feb8ee03",
      "format": "safetensors",
      "file": "2 files, e.g. 1_Dense/model.safetensors",
      "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)",
      "supported_today": true
    }
  ],
  "use_cases": [
    "document-search",
    "ocr"
  ],
  "recommended": {
    "value": true,
    "for": [
      "document-search"
    ],
    "reason": "Served late-interaction page retriever with a committed ViDoRe evaluation."
  },
  "demos": [],
  "seo": {
    "title": "ColQwen2 v1.0 (merged): embedding model",
    "description": "ColQwen2 v1.0 (merged) is a image, document and text embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.",
    "keywords": [
      "ColQwen2 v1.0 (merged)",
      "colpali",
      "embedding",
      "vidore/colqwen2-v1.0-merged",
      "document-search",
      "ocr"
    ],
    "faqs": [
      {
        "q": "What is ColQwen2 v1.0 (merged) best at?",
        "a": "Text search over document page images (PDF pages, slides, infographics, tables) by late interaction, with no OCR step."
      },
      {
        "q": "What are the limits of ColQwen2 v1.0 (merged)?",
        "a": "Video frames and product photos: committed eval nDCG@10 0.133 on MSR-VTT keyframes and 0.170 on product images, below single-vector SigLIP. About 750 x 128-d vectors per page make storage and cold reads expensive."
      },
      {
        "q": "Can ColQwen2 v1.0 (merged) be used commercially?",
        "a": "Yes. Apache-2.0 on the repo; the base Qwen2-VL-2B is Apache-2.0."
      },
      {
        "q": "How does ColQwen2 v1.0 (merged) run on Mixpeek?",
        "a": "Ray Serve app in the inference registry; colqwen_extractor batch calls it (ColQwen2EmbeddingBatchService, a ServeProxyProcessor). The embedding registry marks it PREVIEW."
      }
    ]
  },
  "last_verified": "2026-11-05",
  "sources": [
    "https://huggingface.co/vidore/colqwen2-v1.0-merged",
    "https://huggingface.co/api/models/vidore/colqwen2-v1.0-merged",
    "https://arxiv.org/abs/2407.01449",
    "server/engine/inference/vidore/colqwen2/encoder.py",
    "server/engine/inference/vidore/colqwen2/services.py",
    "server/engine/extractors/colqwen_extractor/v1/definition.py",
    "server/engine/extractors/colqwen_extractor/v1/processors.py"
  ],
  "registry": {
    "vidore__colqwen2": {
      "deployment": "vidore__colqwen2",
      "service_id": "vidore/colqwen2",
      "routes": "engine/inference/vidore/colqwen2/routes.py",
      "route_prefix": "/vidore/colqwen2/v1",
      "enabled": true,
      "description": "ColQwen2 late-interaction visual document encoder (vidore/colqwen2-v1.0). Returns one 128-dimensional vector per image patch (about 768 for a page) or per query token, for MaxSim scoring of text queries against page, slide and video frame images.",
      "capabilities": {
        "category": "embedding",
        "modality_input": [
          "text",
          "image"
        ],
        "modality_output": "vector",
        "supports_batch": true,
        "reranking_mode": null
      },
      "vector_config": {
        "dimensions": 128,
        "distance": "cosine",
        "vector_type": "multi_dense"
      },
      "serve": {
        "min_replicas": 0,
        "max_replicas": 1,
        "idle_timeout_s": 300,
        "target_ongoing_requests": 4,
        "num_cpus": 4,
        "num_gpus": 0,
        "memory_gb": 12,
        "pinnable": true,
        "placement": "worker",
        "enabled": true
      },
      "interactive_twin": false,
      "interactive_serve": null,
      "schemas": {
        "input": "shared.inference.vidore.colqwen2.models.InferenceRequest",
        "output": "shared.inference.vidore.colqwen2.models.InferenceResponse",
        "config": "shared.inference.vidore.colqwen2.models.InferenceConfigs"
      }
    }
  },
  "templates": [],
  "usage": {
    "namespaces": null,
    "collections": null,
    "uncounted": true,
    "templates": 0,
    "counted_at": null,
    "source": null
  },
  "facets": {
    "status": "served",
    "category": "embedding",
    "family": "colpali",
    "license": "Apache-2.0",
    "commercial_use": true,
    "modalities_in": [
      "image",
      "document",
      "text"
    ],
    "modalities_out": [
      "multi_vector"
    ],
    "use_cases": [
      "document-search",
      "ocr"
    ],
    "recommended": true,
    "release_date": "2024-11",
    "params_millions": 2209,
    "size_on_disk_mb": 4419,
    "vector_dims": 128,
    "vector_distance": "cosine",
    "weights_format": "safetensors",
    "formats": [
      "safetensors"
    ],
    "loadable_today": true,
    "trust_remote_code": false,
    "cpu_fallback": true,
    "num_gpus": 0,
    "memory_gb": 12,
    "min_replicas": 0,
    "serving_shape": "both",
    "served": true,
    "ref_schemes": [
      "hf"
    ],
    "has_evidence": true,
    "cost_measured": false,
    "template_count": 0,
    "usage_namespaces": null,
    "usage_collections": null,
    "usage_uncounted": true,
    "has_demo": false,
    "rank_score": 5
  },
  "rank_score": 5
}