{
  "id": "multimodalembedding",
  "identity": {
    "family": "vertex-multimodal-embedding",
    "display_name": "Vertex AI multimodalembedding@001",
    "hub_id": null,
    "api_name": "multimodalembedding@001",
    "revision": "001",
    "variants": [],
    "license": "Google Cloud Vertex AI terms of service",
    "commercial_use": true,
    "commercial_use_note": "Paid Google Cloud API under Google's terms.",
    "release_date": null,
    "paper_url": null,
    "repo_url": null
  },
  "task": {
    "category": "embedding",
    "modalities_in": [
      "image",
      "video",
      "text"
    ],
    "modalities_out": [
      "vector"
    ],
    "best_at": "Category-level text-to-video and text-to-image search in one 1408-d space (committed mAP 0.855 on a 10-video text eval).",
    "worst_at": "Telling instances apart: the TubeScience re-ID test linked across categories (recall 0.630 at 0.059 false), where DINOv2/v3 separated them."
  },
  "io": {
    "input": {
      "image": "image bytes or GCS URI",
      "video": "video segment (one vector per segment)",
      "text": "text query"
    },
    "output": {
      "vector": {
        "dims": 1408,
        "distance": "cosine"
      }
    }
  },
  "mixpeek_wiring": {
    "extractors": [
      {
        "extractor": "multimodal_extractor@v1",
        "selected_by": "fixed"
      }
    ],
    "feature_uris": [
      "mixpeek://multimodal_extractor@v1/vertex_multimodal_embedding"
    ],
    "vector_indexes": [
      "multimodal_extractor_v1_multimodal_embedding"
    ],
    "retriever_stages": [
      "feature_search"
    ],
    "composes_with": [
      "multimodal_extractor v1: video segment vector beside the E5 transcription vector; v2 replaced it with Gemini Embedding 2"
    ]
  },
  "run": {
    "status": "api",
    "status_reason": "Vertex AI API. multimodal_extractor v1 calls it from VertexMultimodalEmbeddingBatch; query embeds go through universal__api (model multimodalembedding).",
    "serve_apps": [
      "universal__api"
    ],
    "runtime": "Vertex AI MultiModalEmbeddingModel (vertexai SDK)",
    "weights_format": "api",
    "trust_remote_code": false,
    "size_on_disk_mb": null,
    "params_millions": null,
    "preprocessing": "Modality detected per row; video cut into segments by ffmpeg before the call.",
    "postprocessing": "Vector as returned (1408 dims).",
    "serving_shape": "both",
    "batch_size": null,
    "serve_batch": null,
    "scale_to_zero": {
      "min_replicas": 0,
      "max_replicas": 6,
      "idle_timeout_s": 600,
      "memory_gb": null,
      "num_cpus": 0.1,
      "num_gpus": 0,
      "cold_start_s": {
        "warm_node_s": null,
        "new_node_s": null,
        "source": null
      },
      "cpu_fallback": true
    }
  },
  "cost": {
    "unmeasured": true,
    "measured": null,
    "note": "No committed full measurement. customers/primis/BENCHMARK_RESULTS.md notes ~7 s first call and 1.3 to 2.1 s warm queries end to end, not the model alone."
  },
  "evidence": {
    "unmeasured": false,
    "items": [
      {
        "dataset": "TubeScience instance-retrieval text eval, 10 videos",
        "metric": "mAP, text-to-instance search",
        "k": null,
        "score": 0.855,
        "baseline": "SigLIP2 base on SAM2 crops",
        "baseline_score": 0.29,
        "held_out": false,
        "date": "2026-09-09",
        "source": "customers/tubescience/archived/instance-retrieval/SOTA_EVAL_RESULTS.md"
      },
      {
        "dataset": "TubeScience cross-video instance re-ID (same dog in 3 clips + cat + person)",
        "metric": "best recall with at most 0.10 cross-category false links (false rate 0.059)",
        "k": null,
        "score": 0.63,
        "baseline": null,
        "baseline_score": null,
        "held_out": false,
        "date": "2026-09-09",
        "source": "customers/tubescience/archived/instance-retrieval/SOTA_EVAL_RESULTS.md"
      }
    ]
  },
  "refs": [
    "api://google/multimodalembedding@001"
  ],
  "formats": [
    {
      "ref": "api://google/multimodalembedding@001",
      "format": "api",
      "file": null,
      "loader": "provider SDK, called by model name",
      "supported_today": true
    }
  ],
  "use_cases": [
    "visual-search",
    "text-search",
    "video-understanding"
  ],
  "recommended": {
    "value": false,
    "for": [],
    "reason": null
  },
  "demos": [],
  "seo": {
    "title": "Vertex AI multimodalembedding@001: embedding model",
    "description": "Vertex AI multimodalembedding@001 is a embedding model. Called on Mixpeek through the provider API.",
    "keywords": [
      "Vertex AI multimodalembedding@001",
      "vertex-multimodal-embedding",
      "embedding",
      "multimodalembedding@001",
      "visual-search",
      "text-search",
      "video-understanding"
    ],
    "faqs": [
      {
        "q": "What is Vertex AI multimodalembedding@001 best at?",
        "a": "Category-level text-to-video and text-to-image search in one 1408-d space (committed mAP 0.855 on a 10-video text eval)."
      },
      {
        "q": "What are the limits of Vertex AI multimodalembedding@001?",
        "a": "Telling instances apart: the TubeScience re-ID test linked across categories (recall 0.630 at 0.059 false), where DINOv2/v3 separated them."
      },
      {
        "q": "Can Vertex AI multimodalembedding@001 be used commercially?",
        "a": "Yes. Paid Google Cloud API under Google's terms."
      },
      {
        "q": "How does Vertex AI multimodalembedding@001 run on Mixpeek?",
        "a": "Vertex AI API. multimodal_extractor v1 calls it from VertexMultimodalEmbeddingBatch; query embeds go through universal__api (model multimodalembedding)."
      },
      {
        "q": "What does Vertex AI multimodalembedding@001 output?",
        "a": "A 1408-dimensional dense vector, compared by cosine similarity."
      }
    ]
  },
  "last_verified": "2026-11-05",
  "sources": [
    "https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings",
    "server/engine/inference/google/vertex/services.py",
    "server/engine/extractors/multimodal_extractor/v1/definition.py",
    "server/engine/extractors/multimodal_extractor/v1/pipeline.py",
    "server/engine/inference/llm/routes.py",
    "server/shared/models/embeddings/registry.py"
  ],
  "registry": {
    "universal__api": {
      "deployment": "universal__api",
      "service_id": "universal/api",
      "routes": "engine/inference/llm/routes.py",
      "route_prefix": "/universal/api",
      "enabled": true,
      "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.",
      "capabilities": {
        "category": "generation",
        "modality_input": [
          "text"
        ],
        "modality_output": "text",
        "supports_batch": true,
        "reranking_mode": null
      },
      "vector_config": null,
      "serve": {
        "min_replicas": 0,
        "max_replicas": 6,
        "idle_timeout_s": 600,
        "target_ongoing_requests": 25,
        "num_cpus": 0.1,
        "num_gpus": 0,
        "memory_gb": null,
        "pinnable": false,
        "placement": "worker",
        "enabled": true
      },
      "interactive_twin": false,
      "interactive_serve": null,
      "schemas": {
        "input": null,
        "output": null,
        "config": null
      }
    }
  },
  "templates": [
    "ai-data-curation",
    "contextual-video-match",
    "creative-dna",
    "footage-intelligence",
    "ux-session-analysis"
  ],
  "usage": {
    "namespaces": null,
    "collections": null,
    "uncounted": true,
    "templates": 5,
    "counted_at": null,
    "source": null
  },
  "facets": {
    "status": "api",
    "category": "embedding",
    "family": "vertex-multimodal-embedding",
    "license": "Google Cloud Vertex AI terms of service",
    "commercial_use": true,
    "modalities_in": [
      "image",
      "video",
      "text"
    ],
    "modalities_out": [
      "vector"
    ],
    "use_cases": [
      "visual-search",
      "text-search",
      "video-understanding"
    ],
    "recommended": false,
    "release_date": null,
    "params_millions": null,
    "size_on_disk_mb": null,
    "vector_dims": 1408,
    "vector_distance": "cosine",
    "weights_format": "api",
    "formats": [
      "api"
    ],
    "loadable_today": true,
    "trust_remote_code": false,
    "cpu_fallback": true,
    "num_gpus": 0,
    "memory_gb": null,
    "min_replicas": 0,
    "serving_shape": "both",
    "served": true,
    "ref_schemes": [
      "api"
    ],
    "has_evidence": true,
    "cost_measured": false,
    "template_count": 5,
    "usage_namespaces": null,
    "usage_collections": null,
    "usage_uncounted": true,
    "has_demo": false,
    "rank_score": 10.5
  },
  "rank_score": 10.5
}