{ "schema": "shared/inference/catalog/schema.py:ModelRecord", "generated_by": "python -m shared.inference.catalog.build", "rank_formula": "4*log1p(namespaces) + 2*log1p(collections) + 1.5*templates\n + STATUS_WEIGHT[status] + 1 if our own evidence exists\n + 1 if recommended", "count": 91, "by_status": { "api": 13, "not_runnable": 30, "plugin": 33, "served": 15 }, "registry_deployments": [ "BAAI__bge_reranker_v2_m3", "colbert_ir__colbertv2", "facebook__dinov2_base", "google__gemini_embedding_2", "google__siglip_base_patch16_224", "insightface__arcface", "intfloat__multilingual_e5_large_instruct", "jinaai__jina_embeddings_v2_base_code", "laion__CLIP_ViT_L_14_laion2B_s32B_b82K", "laion__clap_htsat_tiny", "mixpeek__api_inference", "mixpeek__clustering", "mixpeek__content_classifier_v1", "mixpeek__decision", "mixpeek__playwright", "mixpeek__rtsp_activity_gate", "openai__whisper_large_v3_turbo", "sentence_transformers__all_MiniLM_L6_v2", "taxonomy_join", "typesafe__jev", "universal__api", "vidore__colqwen2" ], "non_model_deployments": { "mixpeek__api_inference": "Host app for every provider-API target (gemini embedding, universal/api, typesafe/jev, mixpeek/decision); the models are recorded under their own names.", "mixpeek__decision": "Alias that forwards to settings.DECISION_MODEL_DEFAULT (jev unless configured); the model is recorded as typesafe-jev.", "mixpeek__clustering": "Clustering algorithms (HDBSCAN, KMeans and peers) over stored vectors; no learned model.", "mixpeek__playwright": "Headless browser rendering for web_scraper; no model.", "mixpeek__rtsp_activity_gate": "Frame-difference gate on 32x18 grayscale rasters; ffmpeg only, no model.", "taxonomy_join": "Taxonomy enrichment orchestration over retrievers; no model." }, "uncovered_deployments": [], "usage_counted_at": null, "models": [ { "id": "multilingual-e5-large-instruct", "identity": { "family": "e5", "display_name": "Multilingual E5 Large Instruct", "hub_id": "intfloat/multilingual-e5-large-instruct", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights and code; no use restrictions.", "release_date": "2024-02", "paper_url": "https://arxiv.org/abs/2402.05672", "repo_url": "https://github.com/microsoft/unilm/tree/master/e5" }, "task": { "category": "embedding", "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "best_at": "Default text vector for search over transcripts, OCR text, descriptions, documents and web pages in 100+ languages; one space for query and passage text.", "worst_at": "Exact keyword, id and number matches (pair it with BM25); text past 512 tokens is truncated; Mixpeek sends E5 prefixes (query: / passage:) rather than the model card's Instruct: template, so instruction tuning is unused." }, "io": { "input": { "text": { "max_tokens": 512, "truncation": true, "languages": "100+" }, "prompts": "embedding_task retrieval_document -> 'passage: ', retrieval_query / semantic_similarity / classification / clustering -> 'query: ', none -> no prefix" }, "output": { "vector": { "dims": 1024, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "text_extractor@v1", "selected_by": "embedding_model unset or multilingual_e5_large_instruct_v1 (registry TEXT default)" }, { "extractor": "multimodal_extractor@v1", "selected_by": "fixed (transcription embedding)" }, { "extractor": "multimodal_extractor@v2", "selected_by": "fixed (transcription; OCR and description embeddings when those outputs run)" }, { "extractor": "document_graph_extractor@v1", "selected_by": "fixed" }, { "extractor": "web_scraper@v1", "selected_by": "generate_text_embeddings=true (default true)" } ], "feature_uris": [ "mixpeek://text_extractor@v1/multilingual_e5_large_instruct_v1", "mixpeek://multimodal_extractor@v1/multilingual_e5_large_instruct_v1", "mixpeek://multimodal_extractor@v2/multilingual_e5_large_instruct_v1", "mixpeek://multimodal_extractor@v2/multilingual_e5_large_instruct_ocr_v1", "mixpeek://multimodal_extractor@v2/multilingual_e5_large_instruct_description_v1", "mixpeek://document_graph_extractor@v1/intfloat__multilingual_e5_large_instruct", "mixpeek://web_scraper@v1/intfloat__multilingual_e5_large_instruct" ], "vector_indexes": [ "text_extractor_v1_embedding", "multimodal_extractor_v1_transcription_embedding", "multimodal_extractor_v2_transcription_embedding", "multimodal_extractor_v2_ocr_embedding", "multimodal_extractor_v2_description_embedding", "document_graph_extractor_v1_text_embedding", "intfloat__multilingual_e5_large_instruct" ], "retriever_stages": [ "feature_search", "query_expand", "mmr" ], "composes_with": [ "whisper transcript per video segment -> E5 transcription vector beside the Vertex or Gemini multimodal vector, fused by RRF/DBSF", "E5 dense + BM25 full-text on the same text_extractor feature, fused by RRF", "feature_search -> rerank (bge-reranker-v2-m3)" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry, with an interactive twin for query embeds. Batch calls the Serve app when HARDWARE_E5_USE_API is true (the default); false loads E5 in the job (E5TextEmbeddingBatch).", "serve_apps": [ "intfloat__multilingual_e5_large_instruct" ], "runtime": "transformers AutoModel + AutoTokenizer (XLMRobertaModel), average pooling", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1120.0, "params_millions": 560.0, "preprocessing": "Task prefix by embedding_task, tokenize with truncation at 512 tokens.", "postprocessing": "Mean pool last hidden state over the attention mask, then L2-normalize (1024 dims).", "serving_shape": "both", "batch_size": 64, "serve_batch": true, "scale_to_zero": { "min_replicas": { "expr": "int(os.environ.get('MVS_E5_MIN_REPLICAS', '0'))" }, "max_replicas": { "expr": "int(os.environ.get('MVS_E5_MAX_REPLICAS', '10'))" }, "idle_timeout_s": 300, "memory_gb": 2, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": 31.0, "new_node_s": 365.0, "source": "server/engine/inference/intfloat/multilingual_e5_large_instruct/routes.py interactive_serve note (2026-09-30): cold start with a node present 31 s; one that adds a spot node and pulls the image about 365 s" }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput+latency+memory+cost measurement. Committed partial facts: interactive twin query embed p50 1.69 s, p90 1.98 s over 281 calls (services.py comment); batch defaults 64 texts on GPU, 16 on CPU." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "LongMemEval subset, 60 questions (hybrid_flat on Mixpeek)", "metric": "retrieval hit rate, SYSTEM number: E5 dense + BM25 fused by RRF in one retriever, top 20", "k": 20, "score": 1.0, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-09-27", "source": "research/agent-memory/RESULTS.md" }, { "dataset": "LongMemEval subset, 60 questions (hybrid_flat on Mixpeek)", "metric": "QA accuracy, SYSTEM number: E5 + BM25 RRF retrieval, Gemini v2 reader and Gemini judge", "k": null, "score": 0.683, "baseline": "local MiniLM dense + rank_bm25 RRF, same reader", "baseline_score": 0.783, "held_out": false, "date": "2026-09-27", "source": "research/agent-memory/RESULTS.md" } ] }, "refs": [ "hf://intfloat/multilingual-e5-large-instruct@274baa43b0e13e37fafa6428dbc7938e62e5c439" ], "formats": [ { "ref": "hf://intfloat/multilingual-e5-large-instruct@274baa43b0e13e37fafa6428dbc7938e62e5c439", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://intfloat/multilingual-e5-large-instruct@274baa43b0e13e37fafa6428dbc7938e62e5c439", "format": "onnx", "file": "onnx/model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true } ], "use_cases": [ "text-search", "multilingual-search", "transcripts" ], "recommended": { "value": true, "for": [ "text-search", "multilingual-search" ], "reason": "Mixpeek's default text embedding: served at zero replicas, used by text, multimodal and document extractors, with committed retrieval evidence." }, "demos": [], "seo": { "title": "Multilingual E5 Large Instruct: embedding model", "description": "Multilingual E5 Large Instruct is a text embedding model (MIT). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "Multilingual E5 Large Instruct", "e5", "embedding", "intfloat/multilingual-e5-large-instruct", "text-search", "multilingual-search", "transcripts" ], "faqs": [ { "q": "What is Multilingual E5 Large Instruct best at?", "a": "Default text vector for search over transcripts, OCR text, descriptions, documents and web pages in 100+ languages; one space for query and passage text." }, { "q": "What are the limits of Multilingual E5 Large Instruct?", "a": "Exact keyword, id and number matches (pair it with BM25); text past 512 tokens is truncated; Mixpeek sends E5 prefixes (query: / passage:) rather than the model card's Instruct: template, so instruction tuning is unused." }, { "q": "Can Multilingual E5 Large Instruct be used commercially?", "a": "Yes. MIT weights and code; no use restrictions." }, { "q": "How does Multilingual E5 Large Instruct run on Mixpeek?", "a": "Ray Serve app in the inference registry, with an interactive twin for query embeds. Batch calls the Serve app when HARDWARE_E5_USE_API is true (the default); false loads E5 in the job (E5TextEmbeddingBatch)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/intfloat/multilingual-e5-large-instruct", "https://huggingface.co/api/models/intfloat/multilingual-e5-large-instruct", "https://arxiv.org/abs/2402.05672", "server/engine/inference/intfloat/multilingual_e5_large_instruct/services.py", "server/engine/inference/intfloat/multilingual_e5_large_instruct/routes.py", "server/engine/extractors/text_extractor/v1/definition.py", "server/shared/models/embeddings/registry.py", "server/engine/extractors/multimodal_extractor/v2/definition.py", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/document_graph_extractor/v1/definition.py" ], "registry": { "intfloat__multilingual_e5_large_instruct": { "deployment": "intfloat__multilingual_e5_large_instruct", "service_id": "intfloat/multilingual-e5-large-instruct", "routes": "engine/inference/intfloat/multilingual_e5_large_instruct/routes.py", "route_prefix": "/multilingual_e5_large_instruct/v1", "enabled": true, "description": "Multilingual E5 Large Instruct embedding model. Generates 1024-dimensional dense embeddings for text. Supports 100+ languages with instruction-tuned representations.", "capabilities": { "category": "embedding", "modality_input": [ "text" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 1024, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": { "expr": "int(os.environ.get('MVS_E5_MIN_REPLICAS', '0'))" }, "max_replicas": { "expr": "int(os.environ.get('MVS_E5_MAX_REPLICAS', '10'))" }, "idle_timeout_s": 300, "target_ongoing_requests": 1, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": true, "interactive_serve": { "min_replicas": { "expr": "int(os.environ.get('MVS_E5_INTERACTIVE_MIN_REPLICAS', '0'))" }, "max_replicas": { "expr": "int(os.environ.get('MVS_E5_INTERACTIVE_MAX_REPLICAS', '2'))" }, "idle_timeout_s": 300, "target_ongoing_requests": 1, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 2, "pinnable": false, "placement": "head", "enabled": true }, "schemas": { "input": "shared.inference.intfloat.multilingual_e5_large_instruct.models.InferenceRequest", "output": "shared.inference.intfloat.multilingual_e5_large_instruct.models.InferenceResponse", "config": "shared.inference.intfloat.multilingual_e5_large_instruct.models.InferenceConfigs" } } }, "templates": [ "contextual-video-match", "creative-dna", "ux-session-analysis", "video-moderation" ], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 4, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "e5", "license": "MIT", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "use_cases": [ "text-search", "multilingual-search", "transcripts" ], "recommended": true, "release_date": "2024-02", "params_millions": 560.0, "size_on_disk_mb": 1120.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": 2.0, "min_replicas": null, "serving_shape": "both", "served": true, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 4, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 11.0 }, "rank_score": 11.0 }, { "id": "multimodalembedding", "identity": { "family": "vertex-multimodal-embedding", "display_name": "Vertex AI multimodalembedding@001", "hub_id": null, "api_name": "multimodalembedding@001", "revision": "001", "variants": [], "license": "Google Cloud Vertex AI terms of service", "commercial_use": true, "commercial_use_note": "Paid Google Cloud API under Google's terms.", "release_date": null, "paper_url": null, "repo_url": null }, "task": { "category": "embedding", "modalities_in": [ "image", "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Category-level text-to-video and text-to-image search in one 1408-d space (committed mAP 0.855 on a 10-video text eval).", "worst_at": "Telling instances apart: the TubeScience re-ID test linked across categories (recall 0.630 at 0.059 false), where DINOv2/v3 separated them." }, "io": { "input": { "image": "image bytes or GCS URI", "video": "video segment (one vector per segment)", "text": "text query" }, "output": { "vector": { "dims": 1408, "distance": "cosine" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v1", "selected_by": "fixed" } ], "feature_uris": [ "mixpeek://multimodal_extractor@v1/vertex_multimodal_embedding" ], "vector_indexes": [ "multimodal_extractor_v1_multimodal_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "multimodal_extractor v1: video segment vector beside the E5 transcription vector; v2 replaced it with Gemini Embedding 2" ] }, "run": { "status": "api", "status_reason": "Vertex AI API. multimodal_extractor v1 calls it from VertexMultimodalEmbeddingBatch; query embeds go through universal__api (model multimodalembedding).", "serve_apps": [ "universal__api" ], "runtime": "Vertex AI MultiModalEmbeddingModel (vertexai SDK)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Modality detected per row; video cut into segments by ffmpeg before the call.", "postprocessing": "Vector as returned (1408 dims).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed full measurement. customers/primis/BENCHMARK_RESULTS.md notes ~7 s first call and 1.3 to 2.1 s warm queries end to end, not the model alone." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "TubeScience instance-retrieval text eval, 10 videos", "metric": "mAP, text-to-instance search", "k": null, "score": 0.855, "baseline": "SigLIP2 base on SAM2 crops", "baseline_score": 0.29, "held_out": false, "date": "2026-09-09", "source": "customers/tubescience/archived/instance-retrieval/SOTA_EVAL_RESULTS.md" }, { "dataset": "TubeScience cross-video instance re-ID (same dog in 3 clips + cat + person)", "metric": "best recall with at most 0.10 cross-category false links (false rate 0.059)", "k": null, "score": 0.63, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-09-09", "source": "customers/tubescience/archived/instance-retrieval/SOTA_EVAL_RESULTS.md" } ] }, "refs": [ "api://google/multimodalembedding@001" ], "formats": [ { "ref": "api://google/multimodalembedding@001", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "visual-search", "text-search", "video-understanding" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Vertex AI multimodalembedding@001: embedding model", "description": "Vertex AI multimodalembedding@001 is a embedding model. Called on Mixpeek through the provider API.", "keywords": [ "Vertex AI multimodalembedding@001", "vertex-multimodal-embedding", "embedding", "multimodalembedding@001", "visual-search", "text-search", "video-understanding" ], "faqs": [ { "q": "What is Vertex AI multimodalembedding@001 best at?", "a": "Category-level text-to-video and text-to-image search in one 1408-d space (committed mAP 0.855 on a 10-video text eval)." }, { "q": "What are the limits of Vertex AI multimodalembedding@001?", "a": "Telling instances apart: the TubeScience re-ID test linked across categories (recall 0.630 at 0.059 false), where DINOv2/v3 separated them." }, { "q": "Can Vertex AI multimodalembedding@001 be used commercially?", "a": "Yes. Paid Google Cloud API under Google's terms." }, { "q": "How does Vertex AI multimodalembedding@001 run on Mixpeek?", "a": "Vertex AI API. multimodal_extractor v1 calls it from VertexMultimodalEmbeddingBatch; query embeds go through universal__api (model multimodalembedding)." }, { "q": "What does Vertex AI multimodalembedding@001 output?", "a": "A 1408-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings", "server/engine/inference/google/vertex/services.py", "server/engine/extractors/multimodal_extractor/v1/definition.py", "server/engine/extractors/multimodal_extractor/v1/pipeline.py", "server/engine/inference/llm/routes.py", "server/shared/models/embeddings/registry.py" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [ "ai-data-curation", "contextual-video-match", "creative-dna", "footage-intelligence", "ux-session-analysis" ], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 5, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "embedding", "family": "vertex-multimodal-embedding", "license": "Google Cloud Vertex AI terms of service", "commercial_use": true, "modalities_in": [ "image", "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "text-search", "video-understanding" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": null, "vector_dims": 1408, "vector_distance": "cosine", "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "api" ], "has_evidence": true, "cost_measured": false, "template_count": 5, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 10.5 }, "rank_score": 10.5 }, { "id": "siglip-base-patch16-224", "identity": { "family": "siglip", "display_name": "SigLIP Base patch16 224", "hub_id": "google/siglip-base-patch16-224", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2023-03", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Text-to-image and image-to-image search over photos, products and video frames in one shared space.", "worst_at": "Reading text in images, counting, small objects, and catalog facts that are not visible (NGA eval, self-portrait and etching queries)." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64, "padding": "max_length", "language": "English" }, "prompts": "plain captions; no instruction prefix" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "image_extractor@v1", "selected_by": "fixed" }, { "extractor": "web_scraper@v1", "selected_by": "generate_image_embeddings=true (default true)" } ], "feature_uris": [ "mixpeek://image_extractor@v1/google_siglip_base_v1", "mixpeek://web_scraper@v1/google__siglip_base_patch16_224" ], "vector_indexes": [ "image_extractor_v1_embedding", "google__siglip_base_patch16_224" ], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "image_extractor: one SigLIP vector per image; feature_search embeds a text or image query with the same Serve app", "web_scraper: SigLIP vectors for page images beside E5 text, Jina code and DINOv2 structure vectors", "colqwen_extractor reuses the image_extractor batch step with its embed swapped for ColQwen2" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry. image_extractor batch calls the Serve app (ImageEmbeddingBatchService, strict_serve: no local fallback unless SERVE_PROXY_ALLOW_LOCAL_FALLBACK=1); web_scraper loads SiglipVisionModel in-process.", "serve_apps": [ "google__siglip_base_patch16_224" ], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel) for realtime text and image; SiglipVisionModel + SiglipImageProcessor for batch (SiglipImageEmbeddingBatch, web_scraper DocTypeImageEmbedder)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 813.0, "params_millions": 203.0, "preprocessing": "Resize to 224x224, rescale, normalize mean 0.5 std 0.5; text padded to 64 tokens.", "postprocessing": "L2-normalize the pooled image or text embedding (pooler_output).", "serving_shape": "both", "batch_size": 64, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "memory_gb": 1, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cold-start measurement for this model alone." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "NGA art search, 25 text queries", "metric": "P@10 (regex relevance, lower bound)", "k": 10, "score": 0.748, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-11-02", "source": "canvas/apps/nga-art-search/eval/RESULTS.md" }, { "dataset": "NGA art search, 25 text queries", "metric": "nDCG@10", "k": 10, "score": 0.738, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-11-02", "source": "canvas/apps/nga-art-search/eval/RESULTS.md" }, { "dataset": "ViDoRe v1 + v2 document pages, 1,500 English queries (93,992-item mixed corpus)", "metric": "nDCG@10, SigLIP alone top 10 (dense single vector)", "k": 10, "score": 0.135, "baseline": "exact ColQwen2 MaxSim", "baseline_score": 0.696, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" }, { "dataset": "MSR-VTT 1K-A captions over scene-cut keyframes (1,000 queries)", "metric": "nDCG@10, SigLIP alone top 10", "k": 10, "score": 0.14, "baseline": "exact ColQwen2 MaxSim", "baseline_score": 0.133, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" }, { "dataset": "Marqo/KAGL product images, 1,000 product-title queries", "metric": "nDCG@10, SigLIP alone top 10", "k": 10, "score": 0.293, "baseline": "exact ColQwen2 MaxSim", "baseline_score": 0.17, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" } ] }, "refs": [ "hf://google/siglip-base-patch16-224@7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", "mixpeek://system/siglip_vit_b_16_224@1.0.0" ], "formats": [ { "ref": "hf://google/siglip-base-patch16-224@7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://google/siglip-base-patch16-224@7fd15f0689c79d79e38b1c2e2e2370a7bf2761ed", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": true, "for": [ "visual-search" ], "reason": "The served image embedding behind image_extractor; measured on the NGA art search eval." }, "demos": [], "seo": { "title": "SigLIP Base patch16 224: embedding model", "description": "SigLIP Base patch16 224 is a image and text embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "SigLIP Base patch16 224", "siglip", "embedding", "google/siglip-base-patch16-224", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP Base patch16 224 best at?", "a": "Text-to-image and image-to-image search over photos, products and video frames in one shared space." }, { "q": "What are the limits of SigLIP Base patch16 224?", "a": "Reading text in images, counting, small objects, and catalog facts that are not visible (NGA eval, self-portrait and etching queries)." }, { "q": "Can SigLIP Base patch16 224 be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP Base patch16 224 run on Mixpeek?", "a": "Ray Serve app in the inference registry. image_extractor batch calls the Serve app (ImageEmbeddingBatchService, strict_serve: no local fallback unless SERVE_PROXY_ALLOW_LOCAL_FALLBACK=1); web_scraper loads SiglipVisionModel in-process." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-base-patch16-224", "https://arxiv.org/abs/2303.15343", "server/engine/extractors/image_extractor/v1/definition.py", "server/engine/inference/google/siglip/routes.py", "server/engine/inference/google/siglip/services.py", "server/engine/extractors/image_extractor/v1/processors/processors.py", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/web_scraper/v1/processors/doctype_embedders.py", "server/engine/core/serve_proxy.py" ], "registry": { "google__siglip_base_patch16_224": { "deployment": "google__siglip_base_patch16_224", "service_id": "google/siglip-base-patch16-224", "routes": "engine/inference/google/siglip/routes.py", "route_prefix": "/google_siglip_base/v1", "enabled": true, "description": "Google SigLIP Base multimodal embedding model. Generates 768-dimensional embeddings for images and text in the same space. Supports cross-modal similarity search (text-to-image, image-to-image).", "capabilities": { "category": "embedding", "modality_input": [ "text", "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 768, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "target_ongoing_requests": 5, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 1, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.google.siglip.models.InferenceRequest", "output": "shared.inference.google.siglip.models.InferenceResponse", "config": "shared.inference.google.siglip.models.InferenceConfigs" } } }, "templates": [ "video-moderation" ], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 1, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": true, "release_date": "2023-03", "params_millions": 203.0, "size_on_disk_mb": 813.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf", "mixpeek" ], "has_evidence": true, "cost_measured": false, "template_count": 1, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 6.5 }, "rank_score": 6.5 }, { "id": "bge-reranker-v2-m3", "identity": { "family": "bge-reranker", "display_name": "BGE Reranker v2 M3", "hub_id": "BAAI/bge-reranker-v2-m3", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; no use restrictions.", "release_date": "2024-03", "paper_url": "https://arxiv.org/abs/2402.03216", "repo_url": "https://github.com/FlagOpen/FlagEmbedding" }, "task": { "category": "reranking", "modalities_in": [ "text" ], "modalities_out": [ "scores" ], "best_at": "Re-ordering a short candidate list of text documents against a query (cross-encoder, multilingual), most useful where the dense ordering is weak.", "worst_at": "Large candidate pools on CPU replicas: committed runs breached the 10 s stage ceiling, and it hurt queries whose dense ordering was already near perfect (0.986 -> 0.781 nDCG). Text only." }, "io": { "input": { "pairs": "[[query, document], ...]", "text": { "max_tokens": 512, "truncation": true } }, "output": { "scores": "one relevance logit per pair" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "rerank" ], "composes_with": [ "feature_search (E5 or any text feature) -> rerank inference_name=BAAI__bge_reranker_v2_m3 (the default)" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry; the rerank stage's default inference_name.", "serve_apps": [ "BAAI__bge_reranker_v2_m3" ], "runtime": "transformers AutoModelForSequenceClassification + AutoTokenizer (XLMRobertaForSequenceClassification)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 2271.0, "params_millions": 568.0, "preprocessing": "Tokenize each (query, document) pair together, truncation at 512 tokens.", "postprocessing": "Classification logit per pair as the score; sigmoid applied when the request asks for normalized scores.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 8, "idle_timeout_s": 300, "memory_gb": 2, "num_cpus": 0.5, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement. customers/livenation/manifest-loop/README.md reports about 2 s for 10 pairs in-process and every Serve call timing out at 0.25 CPU; not a full measurement." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "TubeScience capability parity D1, 13 scored queries", "metric": "nDCG (relevance by lexical overlap with the query, on the returned pool), feature_search -> rerank", "k": null, "score": 0.9078, "baseline": "feature_search dense order, same fetch", "baseline_score": 0.8606, "held_out": false, "date": "2026-09-18", "source": "customers/tubescience/parity/2026-09-18-CAPABILITY-PARITY-REPORT.md" } ] }, "refs": [ "hf://BAAI/bge-reranker-v2-m3@953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e" ], "formats": [ { "ref": "hf://BAAI/bge-reranker-v2-m3@953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "reranking", "multilingual-search" ], "recommended": { "value": true, "for": [ "reranking" ], "reason": "Served cross-encoder; raised nDCG on the TubeScience D1 eval over the unreranked baseline." }, "demos": [], "seo": { "title": "BGE Reranker v2 M3: reranking model", "description": "BGE Reranker v2 M3 is a text reranking model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "BGE Reranker v2 M3", "bge-reranker", "reranking", "BAAI/bge-reranker-v2-m3", "multilingual-search" ], "faqs": [ { "q": "What is BGE Reranker v2 M3 best at?", "a": "Re-ordering a short candidate list of text documents against a query (cross-encoder, multilingual), most useful where the dense ordering is weak." }, { "q": "What are the limits of BGE Reranker v2 M3?", "a": "Large candidate pools on CPU replicas: committed runs breached the 10 s stage ceiling, and it hurt queries whose dense ordering was already near perfect (0.986 -> 0.781 nDCG). Text only." }, { "q": "Can BGE Reranker v2 M3 be used commercially?", "a": "Yes. Apache-2.0 weights; no use restrictions." }, { "q": "How does BGE Reranker v2 M3 run on Mixpeek?", "a": "Ray Serve app in the inference registry; the rerank stage's default inference_name." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/BAAI/bge-reranker-v2-m3", "https://huggingface.co/api/models/BAAI/bge-reranker-v2-m3", "https://arxiv.org/abs/2402.03216", "server/engine/inference/baai/bge_reranker_v2_m3/services.py", "server/shared/retrievers/stages/sort/rerank/models.py", "customers/livenation/manifest-loop/README.md" ], "registry": { "BAAI__bge_reranker_v2_m3": { "deployment": "BAAI__bge_reranker_v2_m3", "service_id": "BAAI/bge-reranker-v2-m3", "routes": "engine/inference/baai/bge_reranker_v2_m3/routes.py", "route_prefix": "/inference/baai/bge_reranker_v2_m3", "enabled": true, "description": "BAAI BGE Reranker V2 M3 cross-encoder model. Scores (query, passage) pairs for relevance ranking. Multilingual support with state-of-the-art reranking performance.", "capabilities": { "category": "reranking", "modality_input": [ "text" ], "modality_output": "score", "supports_batch": true, "reranking_mode": "text-to-text" }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 8, "idle_timeout_s": 300, "target_ongoing_requests": 2, "num_cpus": 0.5, "num_gpus": 0, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.baai.bge_reranker_v2_m3.models.InferenceRequest", "output": "shared.inference.baai.bge_reranker_v2_m3.models.InferenceResponse", "config": "shared.inference.baai.bge_reranker_v2_m3.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "reranking", "family": "bge-reranker", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "scores" ], "use_cases": [ "reranking", "multilingual-search" ], "recommended": true, "release_date": "2024-03", "params_millions": 568.0, "size_on_disk_mb": 2271.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 5.0 }, "rank_score": 5.0 }, { "id": "colqwen2-v1-0-merged", "identity": { "family": "colpali", "display_name": "ColQwen2 v1.0 (merged)", "hub_id": "vidore/colqwen2-v1.0-merged", "api_name": null, "revision": "8076c402380ee60b54663fbfdea2f6a9feb8ee03", "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 on the repo; the base Qwen2-VL-2B is Apache-2.0.", "release_date": "2024-11", "paper_url": "https://arxiv.org/abs/2407.01449", "repo_url": "https://github.com/illuin-tech/colpali" }, "task": { "category": "embedding", "modalities_in": [ "image", "document", "text" ], "modalities_out": [ "multi_vector" ], "best_at": "Text search over document page images (PDF pages, slides, infographics, tables) by late interaction, with no OCR step.", "worst_at": "Video frames and product photos: committed eval nDCG@10 0.133 on MSR-VTT keyframes and 0.170 on product images, below single-vector SigLIP. About 750 x 128-d vectors per page make storage and cold reads expensive." }, "io": { "input": { "image": { "max_visual_tokens": 768, "patch": "28 px merged patches", "max_pixels": 602112 }, "text": "query text, one vector per token" }, "output": { "vector": { "dims": 128, "distance": "cosine", "type": "multi_dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "colqwen_extractor@v1", "selected_by": "fixed" } ], "feature_uris": [ "mixpeek://colqwen_extractor@v1/colqwen2_v1" ], "vector_indexes": [ "colqwen_extractor_v1_tokens" ], "retriever_stages": [ "maxsim_rerank", "feature_search" ], "composes_with": [ "PDF pages or frames -> ColQwen2 patch matrices -> maxsim_rerank over a SigLIP or E5 first stage" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry; colqwen_extractor batch calls it (ColQwen2EmbeddingBatchService, a ServeProxyProcessor). The embedding registry marks it PREVIEW.", "serve_apps": [ "vidore__colqwen2" ], "runtime": "transformers Qwen2VLModel + Qwen2VLImageProcessor + AutoTokenizer, 1_Dense projection to 128 (ColQwen2Encoder, engine code); no colpali-engine dependency", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4419.0, "params_millions": 2209.0, "preprocessing": "RGB, resized by the Qwen2-VL processor to at most 768 visual tokens; bf16 (or fp16) on CUDA, fp32 on CPU.", "postprocessing": "Project every image patch or query token to 128 dims and L2-normalize; one matrix per page, scored by MaxSim.", "serving_shape": "both", "batch_size": 8, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 300, "memory_gb": 12, "num_cpus": 4, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed per-item throughput, latency, memory and cost measurement of the Serve app." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "ViDoRe v1 + v2 document pages, 1,500 English queries (93,992-item mixed corpus)", "metric": "nDCG@10, exact f32 MaxSim (vidore/colqwen2-v1.0 via colpali-engine, bf16 on L4)", "k": 10, "score": 0.696, "baseline": "SigLIP base alone, top 10", "baseline_score": 0.135, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" }, { "dataset": "MSR-VTT 1K-A captions over scene-cut keyframes (1,000 queries)", "metric": "nDCG@10, exact f32 MaxSim", "k": 10, "score": 0.133, "baseline": "SigLIP base alone, top 10", "baseline_score": 0.14, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" }, { "dataset": "Marqo/KAGL product images, 1,000 product-title queries", "metric": "nDCG@10, exact f32 MaxSim", "k": 10, "score": 0.17, "baseline": "SigLIP base alone, top 10", "baseline_score": 0.293, "held_out": false, "date": "2026-10-15", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-MULTIMODAL-LI.md" } ] }, "refs": [ "hf://vidore/colqwen2-v1.0-merged@8076c402380ee60b54663fbfdea2f6a9feb8ee03" ], "formats": [ { "ref": "hf://vidore/colqwen2-v1.0-merged@8076c402380ee60b54663fbfdea2f6a9feb8ee03", "format": "safetensors", "file": "2 files, e.g. 1_Dense/model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "document-search", "ocr" ], "recommended": { "value": true, "for": [ "document-search" ], "reason": "Served late-interaction page retriever with a committed ViDoRe evaluation." }, "demos": [], "seo": { "title": "ColQwen2 v1.0 (merged): embedding model", "description": "ColQwen2 v1.0 (merged) is a image, document and text embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "ColQwen2 v1.0 (merged)", "colpali", "embedding", "vidore/colqwen2-v1.0-merged", "document-search", "ocr" ], "faqs": [ { "q": "What is ColQwen2 v1.0 (merged) best at?", "a": "Text search over document page images (PDF pages, slides, infographics, tables) by late interaction, with no OCR step." }, { "q": "What are the limits of ColQwen2 v1.0 (merged)?", "a": "Video frames and product photos: committed eval nDCG@10 0.133 on MSR-VTT keyframes and 0.170 on product images, below single-vector SigLIP. About 750 x 128-d vectors per page make storage and cold reads expensive." }, { "q": "Can ColQwen2 v1.0 (merged) be used commercially?", "a": "Yes. Apache-2.0 on the repo; the base Qwen2-VL-2B is Apache-2.0." }, { "q": "How does ColQwen2 v1.0 (merged) run on Mixpeek?", "a": "Ray Serve app in the inference registry; colqwen_extractor batch calls it (ColQwen2EmbeddingBatchService, a ServeProxyProcessor). The embedding registry marks it PREVIEW." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/vidore/colqwen2-v1.0-merged", "https://huggingface.co/api/models/vidore/colqwen2-v1.0-merged", "https://arxiv.org/abs/2407.01449", "server/engine/inference/vidore/colqwen2/encoder.py", "server/engine/inference/vidore/colqwen2/services.py", "server/engine/extractors/colqwen_extractor/v1/definition.py", "server/engine/extractors/colqwen_extractor/v1/processors.py" ], "registry": { "vidore__colqwen2": { "deployment": "vidore__colqwen2", "service_id": "vidore/colqwen2", "routes": "engine/inference/vidore/colqwen2/routes.py", "route_prefix": "/vidore/colqwen2/v1", "enabled": true, "description": "ColQwen2 late-interaction visual document encoder (vidore/colqwen2-v1.0). Returns one 128-dimensional vector per image patch (about 768 for a page) or per query token, for MaxSim scoring of text queries against page, slide and video frame images.", "capabilities": { "category": "embedding", "modality_input": [ "text", "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 128, "distance": "cosine", "vector_type": "multi_dense" }, "serve": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 300, "target_ongoing_requests": 4, "num_cpus": 4, "num_gpus": 0, "memory_gb": 12, "pinnable": true, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.vidore.colqwen2.models.InferenceRequest", "output": "shared.inference.vidore.colqwen2.models.InferenceResponse", "config": "shared.inference.vidore.colqwen2.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "colpali", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "document", "text" ], "modalities_out": [ "multi_vector" ], "use_cases": [ "document-search", "ocr" ], "recommended": true, "release_date": "2024-11", "params_millions": 2209.0, "size_on_disk_mb": 4419.0, "vector_dims": 128, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 12.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 5.0 }, "rank_score": 5.0 }, { "id": "gemini-embedding-2", "identity": { "family": "gemini-embedding", "display_name": "Gemini Embedding 2", "hub_id": null, "api_name": "gemini-embedding-2", "revision": null, "variants": [], "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "commercial_use_note": "Paid Google API under Google's terms; Mixpeek resells the call and pays per token.", "release_date": "2026-03", "paper_url": null, "repo_url": null }, "task": { "category": "embedding", "modalities_in": [ "text", "image", "video", "audio", "document" ], "modalities_out": [ "vector" ], "best_at": "One vector space for video segments, images, PDFs, audio and text queries; the default for object-level multimodal extractors (multimodal_extractor v2, universal_extractor).", "worst_at": "Cost and latency per call (provider API, 1.9 to 7 s query spikes after a lull per the routes comment); no task_type parameter, so Mixpeek prepends instruction text instead; 3072-d vectors are large to store." }, "io": { "input": { "text": { "max_tokens": 8192 }, "image": "inline or URI parts", "video": "segments up to 120 s (multimodal_extractor caps Gemini segments at 120 s)", "prompts": "embedding_task maps to a text prefix: RETRIEVAL_QUERY 'Represent this query for retrieving relevant documents: ', RETRIEVAL_DOCUMENT 'Represent this document for retrieval: ', SEMANTIC_SIMILARITY ..." }, "output": { "vector": { "dims": 3072, "distance": "cosine", "matryoshka": "outputDimensionality when dimensions != 3072" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v2", "selected_by": "fixed" }, { "extractor": "universal_extractor@v1", "selected_by": "fixed (GEMINI_EMBEDDING_MODEL env, default gemini-embedding-2)" } ], "feature_uris": [ "mixpeek://multimodal_extractor@v2/gemini-embedding-2", "mixpeek://universal_extractor@v1/gemini-embedding-2" ], "vector_indexes": [ "multimodal_extractor_v2_multimodal_embedding", "universal_extractor_v1_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "multimodal_extractor v2: video segment + Gemini description -> one Gemini Embedding 2 vector, beside E5 transcription/OCR/description vectors" ] }, "run": { "status": "api", "status_reason": "Provider API (Gemini developer API or Vertex). The google__gemini_embedding_2 registry app proxies query embeds; on the planes it is hosted inside mixpeek__api_inference. Extractors call the API directly from the batch step.", "serve_apps": [ "google__gemini_embedding_2" ], "runtime": "Gemini REST embedContent (gemini_rest_endpoint) for queries; google-genai client.models.embed_content in extractors", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Build content parts (text, image, video, PDF), prepend the task instruction to text.", "postprocessing": "Vector as returned (3072 dims unless dimensions is set); responses cached by model, task, dims and content.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 300, "memory_gb": null, "num_cpus": 0.5, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput+latency+cost measurement; billed per token by Google." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-embedding-2" ], "formats": [ { "ref": "api://google/gemini-embedding-2", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "visual-search", "text-search", "multilingual-search", "video-understanding", "audio-search", "document-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini Embedding 2: embedding model", "description": "Gemini Embedding 2 is a embedding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini Embedding 2", "gemini-embedding", "embedding", "gemini-embedding-2", "visual-search", "text-search", "multilingual-search", "video-understanding", "audio-search", "document-search" ], "faqs": [ { "q": "What is Gemini Embedding 2 best at?", "a": "One vector space for video segments, images, PDFs, audio and text queries; the default for object-level multimodal extractors (multimodal_extractor v2, universal_extractor)." }, { "q": "What are the limits of Gemini Embedding 2?", "a": "Cost and latency per call (provider API, 1.9 to 7 s query spikes after a lull per the routes comment); no task_type parameter, so Mixpeek prepends instruction text instead; 3072-d vectors are large to store." }, { "q": "Can Gemini Embedding 2 be used commercially?", "a": "Yes. Paid Google API under Google's terms; Mixpeek resells the call and pays per token." }, { "q": "How does Gemini Embedding 2 run on Mixpeek?", "a": "Provider API (Gemini developer API or Vertex). The google__gemini_embedding_2 registry app proxies query embeds; on the planes it is hosted inside mixpeek__api_inference. Extractors call the API directly from the batch step." }, { "q": "What does Gemini Embedding 2 output?", "a": "A 3072-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-embedding-2/", "server/engine/inference/google/gemini_embedding/services.py", "server/engine/inference/google/gemini_embedding/routes.py", "server/engine/extractors/multimodal_extractor/v2/definition.py", "server/engine/extractors/universal_extractor/v1/processors/gemini_client.py", "server/shared/models/embeddings/registry.py" ], "registry": { "google__gemini_embedding_2": { "deployment": "google__gemini_embedding_2", "service_id": "google/gemini-embedding-2", "routes": "engine/inference/google/gemini_embedding/routes.py", "route_prefix": "/google_gemini_embedding_2/v1", "enabled": true, "description": "Gemini Embedding 2 — Google's natively multimodal embedding model. Generates 3072-dimensional dense embeddings for text, images, video, audio, and PDFs. CPU-only API proxy (no GPU required).", "capabilities": { "category": "embedding", "modality_input": [ "text", "image", "video", "audio" ], "modality_output": "vector", "supports_batch": false, "reranking_mode": null }, "vector_config": { "dimensions": 3072, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 300, "target_ongoing_requests": 3, "num_cpus": 0.5, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [ "rtsp-live-monitoring", "video-moderation" ], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 2, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "embedding", "family": "gemini-embedding", "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "modalities_in": [ "text", "image", "video", "audio", "document" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "text-search", "multilingual-search", "video-understanding", "audio-search", "document-search" ], "recommended": false, "release_date": "2026-03", "params_millions": null, "size_on_disk_mb": null, "vector_dims": 3072, "vector_distance": "cosine", "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 2, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 5.0 }, "rank_score": 5.0 }, { "id": "all-minilm-l6-v2", "identity": { "family": "sentence-transformers", "display_name": "all-MiniLM-L6-v2", "hub_id": "sentence-transformers/all-MiniLM-L6-v2", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; no use restrictions.", "release_date": null, "paper_url": "https://arxiv.org/abs/1908.10084", "repo_url": "https://github.com/UKPLab/sentence-transformers" }, "task": { "category": "embedding", "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "best_at": "Small, fast English sentence vectors on CPU (22M params, 384 dims) for short text, dedup and clustering.", "worst_at": "Non-English text, long passages (truncated at 256 tokens here), and hard retrieval where E5-large wins." }, "io": { "input": { "text": { "max_tokens": 256, "truncation": true, "language": "English" }, "prompts": "none" }, "output": { "vector": { "dims": 384, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "text_extractor@v1", "selected_by": "embedding_model=all_minilm_l6_v2_v1" } ], "feature_uris": [ "mixpeek://text_extractor@v1/all_minilm_l6_v2_v1" ], "vector_indexes": [ "text_extractor_v1_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "text_extractor with embedding_model=all_minilm_l6_v2_v1 (dimensions fixed at namespace creation)" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry; text_extractor can select it per run. The registry marks it a small fast fallback, E5 is the default.", "serve_apps": [ "sentence_transformers__all_MiniLM_L6_v2" ], "runtime": "transformers AutoModel + AutoTokenizer (BertModel), mean pooling", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 91.0, "params_millions": 22.7, "preprocessing": "Tokenize with truncation at 256 tokens.", "postprocessing": "Mean pool over the attention mask, L2-normalize (384 dims).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "memory_gb": 0.5, "num_cpus": 0.25, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "LongMemEval subset, 60 questions (hybrid_flat, run outside Mixpeek)", "metric": "QA accuracy, SYSTEM number: local MiniLM dense + rank_bm25 RRF retrieval, Gemini reader; not Mixpeek's Serve app", "k": null, "score": 0.783, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-09-27", "source": "research/agent-memory/RESULTS.md" } ] }, "refs": [ "hf://sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41" ], "formats": [ { "ref": "hf://sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "format": "onnx", "file": "9 files, e.g. onnx/model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "format": "pytorch_bin", "file": "3 files, e.g. openvino/openvino_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://sentence-transformers/all-MiniLM-L6-v2@1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "format": "tensorflow", "file": "tf_model.h5", "loader": "TensorFlow; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "text-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "all-MiniLM-L6-v2: embedding model", "description": "all-MiniLM-L6-v2 is a text embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "all-MiniLM-L6-v2", "sentence-transformers", "embedding", "sentence-transformers/all-MiniLM-L6-v2", "text-search" ], "faqs": [ { "q": "What is all-MiniLM-L6-v2 best at?", "a": "Small, fast English sentence vectors on CPU (22M params, 384 dims) for short text, dedup and clustering." }, { "q": "What are the limits of all-MiniLM-L6-v2?", "a": "Non-English text, long passages (truncated at 256 tokens here), and hard retrieval where E5-large wins." }, { "q": "Can all-MiniLM-L6-v2 be used commercially?", "a": "Yes. Apache-2.0 weights; no use restrictions." }, { "q": "How does all-MiniLM-L6-v2 run on Mixpeek?", "a": "Ray Serve app in the inference registry; text_extractor can select it per run. The registry marks it a small fast fallback, E5 is the default." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2", "https://huggingface.co/api/models/sentence-transformers/all-MiniLM-L6-v2", "https://arxiv.org/abs/1908.10084", "server/engine/inference/sentence_transformers/all_minilm_l6_v2/services.py", "server/shared/models/embeddings/registry.py", "server/engine/extractors/text_extractor/v1/definition.py" ], "registry": { "sentence_transformers__all_MiniLM_L6_v2": { "deployment": "sentence_transformers__all_MiniLM_L6_v2", "service_id": "sentence-transformers/all-MiniLM-L6-v2", "routes": "engine/inference/sentence_transformers/all_minilm_l6_v2/routes.py", "route_prefix": "/sentence_transformers/all_minilm_l6_v2/v1", "enabled": true, "description": "All-MiniLM-L6-v2 sentence transformer. Generates 384-dimensional dense embeddings for text. Lightweight 22M-param CPU-only model for staging/testing.", "capabilities": { "category": "embedding", "modality_input": [ "text" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 384, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "target_ongoing_requests": 3, "num_cpus": 0.25, "num_gpus": 0, "memory_gb": 0.5, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.sentence_transformers.all_minilm_l6_v2.models.InferenceRequest", "output": "shared.inference.sentence_transformers.all_minilm_l6_v2.models.InferenceResponse", "config": "shared.inference.sentence_transformers.all_minilm_l6_v2.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "sentence-transformers", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "use_cases": [ "text-search" ], "recommended": false, "release_date": null, "params_millions": 22.7, "size_on_disk_mb": 91.0, "vector_dims": 384, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors", "tensorflow" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 0.5, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 4.0 }, "rank_score": 4.0 }, { "id": "colbertv2-0", "identity": { "family": "colbert", "display_name": "ColBERTv2", "hub_id": "colbert-ir/colbertv2.0", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT on the Hugging Face repo; no use restrictions.", "release_date": "2021-12", "paper_url": "https://arxiv.org/abs/2112.01488", "repo_url": "https://github.com/stanford-futuredata/ColBERT" }, "task": { "category": "embedding", "modalities_in": [ "text" ], "modalities_out": [ "multi_vector" ], "best_at": "Late-interaction text retrieval and reranking (MaxSim over one 128-d vector per token); strong on out-of-domain and term-heavy queries.", "worst_at": "Storage and cold reads: about 36 KB a 275-token document; English-centric BERT-base vocabulary; text only." }, "io": { "input": { "text": { "max_tokens": 512, "query_marker": "[unused0] as [Q]", "document_marker": "[unused1] as [D]", "query_padding": "[MASK] to 32 positions" } }, "output": { "vector": { "dims": 128, "distance": "cosine", "type": "multi_dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "colbert_extractor@v1", "selected_by": "fixed" } ], "feature_uris": [ "mixpeek://colbert_extractor@v1/colbertv2_v1" ], "vector_indexes": [ "colbert_extractor_v1_tokens" ], "retriever_stages": [ "maxsim_rerank", "feature_search" ], "composes_with": [ "feature_search (E5) -> maxsim_rerank on mixpeek://colbert_extractor@v1/colbertv2_v1" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry; colbert_extractor batch uses ColBERTv2Batch. The embedding registry marks it PREVIEW.", "serve_apps": [ "colbert_ir__colbertv2" ], "runtime": "transformers BertModel (add_pooling_layer=False) + AutoTokenizer, 768->128 linear head (ColBERTv2Encoder, engine code)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 438.0, "params_millions": 110.0, "preprocessing": "Documents: [CLS] [D] tokens [SEP], truncated to 512. Queries: [CLS] [Q] tokens [SEP], padded with [MASK] to 32 positions.", "postprocessing": "Project each token 768 -> 128 with the checkpoint linear head and L2-normalize; documents drop punctuation and padding tokens.", "serving_shape": "both", "batch_size": 16, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 2, "idle_timeout_s": 300, "memory_gb": 2, "num_cpus": 1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput, latency, memory and cost measurement of the Serve app." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "BEIR SciFact (5,183 documents)", "metric": "nDCG@10, exact MaxSim, engine encoder fp16 on MPS", "k": 10, "score": 0.7063, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-10-14", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-4.36-1M-RERUN.md" }, { "dataset": "BEIR SciFact padded with 994,817 PubMed abstracts (1M documents)", "metric": "nDCG@10, exact MaxSim", "k": 10, "score": 0.4414, "baseline": null, "baseline_score": null, "held_out": false, "date": "2026-10-14", "source": "research/goal-specs/mvs-scale-from-zero/RESULTS-4.36-1M-RERUN.md" } ] }, "refs": [ "hf://colbert-ir/colbertv2.0@c1e84128e85ef755c096a95bdb06b47793b13acf" ], "formats": [ { "ref": "hf://colbert-ir/colbertv2.0@c1e84128e85ef755c096a95bdb06b47793b13acf", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://colbert-ir/colbertv2.0@c1e84128e85ef755c096a95bdb06b47793b13acf", "format": "onnx", "file": "model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://colbert-ir/colbertv2.0@c1e84128e85ef755c096a95bdb06b47793b13acf", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "text-search", "reranking" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "ColBERTv2: embedding model", "description": "ColBERTv2 is a text embedding model (MIT). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "ColBERTv2", "colbert", "embedding", "colbert-ir/colbertv2.0", "text-search", "reranking" ], "faqs": [ { "q": "What is ColBERTv2 best at?", "a": "Late-interaction text retrieval and reranking (MaxSim over one 128-d vector per token); strong on out-of-domain and term-heavy queries." }, { "q": "What are the limits of ColBERTv2?", "a": "Storage and cold reads: about 36 KB a 275-token document; English-centric BERT-base vocabulary; text only." }, { "q": "Can ColBERTv2 be used commercially?", "a": "Yes. MIT on the Hugging Face repo; no use restrictions." }, { "q": "How does ColBERTv2 run on Mixpeek?", "a": "Ray Serve app in the inference registry; colbert_extractor batch uses ColBERTv2Batch. The embedding registry marks it PREVIEW." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/colbert-ir/colbertv2.0", "https://huggingface.co/api/models/colbert-ir/colbertv2.0", "https://arxiv.org/abs/2112.01488", "server/engine/inference/colbert_ir/colbertv2/encoder.py", "server/engine/inference/colbert_ir/colbertv2/services.py", "server/engine/extractors/colbert_extractor/v1/definition.py", "server/shared/retrievers/stages/sort/maxsim/models.py" ], "registry": { "colbert_ir__colbertv2": { "deployment": "colbert_ir__colbertv2", "service_id": "colbert-ir/colbertv2", "routes": "engine/inference/colbert_ir/colbertv2/routes.py", "route_prefix": "/colbert_ir/colbertv2/v1", "enabled": true, "description": "ColBERTv2 late-interaction text encoder (colbert-ir/colbertv2.0). Returns one 128-dimensional vector per token for MaxSim scoring: documents keep their non-punctuation tokens, queries are padded to 32.", "capabilities": { "category": "embedding", "modality_input": [ "text" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 128, "distance": "cosine", "vector_type": "multi_dense" }, "serve": { "min_replicas": 0, "max_replicas": 2, "idle_timeout_s": 300, "target_ongoing_requests": 2, "num_cpus": 1, "num_gpus": 0, "memory_gb": 2, "pinnable": true, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.colbert_ir.colbertv2.models.InferenceRequest", "output": "shared.inference.colbert_ir.colbertv2.models.InferenceResponse", "config": "shared.inference.colbert_ir.colbertv2.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "colbert", "license": "MIT", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "multi_vector" ], "use_cases": [ "text-search", "reranking" ], "recommended": false, "release_date": "2021-12", "params_millions": 110.0, "size_on_disk_mb": 438.0, "vector_dims": 128, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 4.0 }, "rank_score": 4.0 }, { "id": "nsfw-image-detection", "identity": { "family": "vit", "display_name": "Falconsai NSFW image detection (ViT)", "hub_id": "Falconsai/nsfw_image_detection", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 on the repo; a community fine-tune of a ViT base, with no published training-data audit.", "release_date": "2023-10", "paper_url": "https://arxiv.org/abs/2010.11929", "repo_url": "https://huggingface.co/Falconsai/nsfw_image_detection" }, "task": { "category": "classification", "modalities_in": [ "image", "video" ], "modalities_out": [ "labels", "scores" ], "best_at": "Fast binary normal/nsfw screening of images, and of video by the max score over sparse sampled frames.", "worst_at": "Graded or policy-specific safety (violence, drugs, suggestive but allowed content): it has two labels only; one bad frame between samples can be missed." }, "io": { "input": { "image": { "resolution": "224x224 (ViTImageProcessor)", "color": "RGB" }, "video": "sparse frames sampled by engine.inference.frame_sampling" }, "output": { "labels": [ "normal", "nsfw" ], "scores": "nsfw_score in [0, 1]; is_nsfw = nsfw_score >= threshold (default 0.5)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "classify" ], "composes_with": [ "classify task=nsfw calls mixpeek/content-classifier-v1 per document (image or video URL); text documents go to the DistilBERT text classifier in the same app" ] }, "run": { "status": "served", "status_reason": "Image and video half of the mixpeek__content_classifier_v1 Serve app.", "serve_apps": [ "mixpeek__content_classifier_v1" ], "runtime": "transformers pipeline(\"image-classification\") (ViTForImageClassification)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 343.0, "params_millions": 85.8, "preprocessing": "Flatten to RGB, pipeline image processor (resize 224, normalize); video sampled to sparse frames.", "postprocessing": "Softmax over both labels (top_k=None); nsfw_score is the max over labels matching nsfw/unsafe/porn/explicit/questionable, max over frames for video.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "memory_gb": 2, "num_cpus": 2.0, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Falconsai/nsfw_image_detection@96cb0d0342c7afb80cab76ecc58b265fa44da256" ], "formats": [ { "ref": "hf://Falconsai/nsfw_image_detection@96cb0d0342c7afb80cab76ecc58b265fa44da256", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://Falconsai/nsfw_image_detection@96cb0d0342c7afb80cab76ecc58b265fa44da256", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://Falconsai/nsfw_image_detection@96cb0d0342c7afb80cab76ecc58b265fa44da256", "format": "pytorch_pt", "file": "2 files, e.g. falconsai_yolov9_nsfw_model_quantized.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "moderation", "classification" ], "recommended": { "value": true, "for": [ "moderation" ], "reason": "Served CPU-only image safety classifier behind the content classifier app." }, "demos": [], "seo": { "title": "Falconsai NSFW image detection (ViT): classification model", "description": "Falconsai NSFW image detection (ViT) is a image and video classification model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "Falconsai NSFW image detection (ViT)", "vit", "classification", "Falconsai/nsfw_image_detection", "moderation" ], "faqs": [ { "q": "What is Falconsai NSFW image detection (ViT) best at?", "a": "Fast binary normal/nsfw screening of images, and of video by the max score over sparse sampled frames." }, { "q": "What are the limits of Falconsai NSFW image detection (ViT)?", "a": "Graded or policy-specific safety (violence, drugs, suggestive but allowed content): it has two labels only; one bad frame between samples can be missed." }, { "q": "Can Falconsai NSFW image detection (ViT) be used commercially?", "a": "Yes. Apache-2.0 on the repo; a community fine-tune of a ViT base, with no published training-data audit." }, { "q": "How does Falconsai NSFW image detection (ViT) run on Mixpeek?", "a": "Image and video half of the mixpeek__content_classifier_v1 Serve app." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Falconsai/nsfw_image_detection", "https://huggingface.co/api/models/Falconsai/nsfw_image_detection", "https://arxiv.org/abs/2010.11929", "server/engine/inference/nsfw/services.py", "server/shared/inference/nsfw/models.py", "server/shared/retrievers/stages/enrich/classify/services.py" ], "registry": { "mixpeek__content_classifier_v1": { "deployment": "mixpeek__content_classifier_v1", "service_id": "mixpeek/content-classifier-v1", "routes": "engine/inference/nsfw/routes.py", "route_prefix": "/mixpeek_content_classifier/v1", "enabled": true, "description": "Multimodal content (NSFW) classifier. Classifies text, images, and video (via sparse frame sampling) as safe/unsafe. Powers the tenant upload gate and the retriever 'classify' stage. CPU-only, no GPU required.", "capabilities": { "category": "classification", "modality_input": [ "text", "image", "video" ], "modality_output": "structured", "supports_batch": false, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "target_ongoing_requests": 5, "num_cpus": 2.0, "num_gpus": 0, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.nsfw.models.InferenceRequest", "output": "shared.inference.nsfw.models.InferenceResponse", "config": "shared.inference.nsfw.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "classification", "family": "vit", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "video" ], "modalities_out": [ "labels", "scores" ], "use_cases": [ "moderation", "classification" ], "recommended": true, "release_date": "2023-10", "params_millions": 85.8, "size_on_disk_mb": 343.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "pytorch_bin", "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 4.0 }, "rank_score": 4.0 }, { "id": "whisper-large-v3-turbo", "identity": { "family": "whisper", "display_name": "Whisper large-v3-turbo", "hub_id": "openai/whisper-large-v3-turbo", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights and code; no use restrictions.", "release_date": "2024-10", "paper_url": "https://arxiv.org/abs/2212.04356", "repo_url": "https://github.com/openai/whisper" }, "task": { "category": "transcription", "modalities_in": [ "audio", "video" ], "modalities_out": [ "text" ], "best_at": "Multilingual speech-to-text on GPU at large-v3 quality with the decoder cut from 32 to 4 layers, so decoding is much faster than large-v3.", "worst_at": "Word-level timing through the transformers pipeline (Live Nation measured 9.4 GB peak and wrong times); CPU-only clusters (the code forces the whisper-1 API when there is no GPU); music and non-speech audio." }, "io": { "input": { "audio": { "sample_rate": 16000, "long_form": "return_timestamps=True (MVS_WHISPER_RETURN_TIMESTAMPS), needed past 30 s" }, "language": "optional forced language; auto-detect when unset" }, "output": { "text": "transcript per audio chunk" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v1", "selected_by": "run_transcription=true and HARDWARE_WHISPER_USE_API=false on a GPU cluster" }, { "extractor": "multimodal_extractor@v2", "selected_by": "run_transcription=true and HARDWARE_WHISPER_USE_API=false on a GPU cluster" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "video -> audio chunks -> whisper transcript -> E5 transcription vector (multimodal_extractor_v*_transcription_embedding)", "customers/livenation/manifest-loop ln_shots plugin: 30 s windows with segment timestamps, then Qwen3-Embedding-0.6B text vectors" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry. multimodal_extractor loads it in-process (WhisperLocalTranscriptionBatch, GPU) only when the whisper-1 API is off; prod runs HARDWARE_WHISPER_USE_API=true per infra/hardware/config.py comments.", "serve_apps": [ "openai__whisper_large_v3_turbo" ], "runtime": "transformers AutoModelForSpeechSeq2Seq + AutoProcessor + pipeline(\"automatic-speech-recognition\")", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1618.0, "params_millions": 809.0, "preprocessing": "Decode and resample audio to 16 kHz mono; fp16 on GPU, fp32 otherwise.", "postprocessing": "Keep result[\"text\"]; timestamps are computed and discarded.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": null, "num_cpus": 0.5, "num_gpus": { "expr": "float(os.getenv('HARDWARE_WHISPER_NUM_GPUS', '0.5')) if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed full measurement. Code comment (BACKE-740): return_timestamps cuts L4 throughput from 2.54 to 1.30 (units not stated)." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://openai/whisper-large-v3-turbo@41f01f3fe87f28c78e2fbf8b568835947dd65ed9" ], "formats": [ { "ref": "hf://openai/whisper-large-v3-turbo@41f01f3fe87f28c78e2fbf8b568835947dd65ed9", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "transcripts" ], "recommended": { "value": true, "for": [ "transcripts" ], "reason": "The served speech model behind multimodal_extractor transcription." }, "demos": [], "seo": { "title": "Whisper large-v3-turbo: speech-to-text model", "description": "Whisper large-v3-turbo is a audio and video speech-to-text model (MIT). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "Whisper large-v3-turbo", "whisper", "speech-to-text", "openai/whisper-large-v3-turbo", "transcripts" ], "faqs": [ { "q": "What is Whisper large-v3-turbo best at?", "a": "Multilingual speech-to-text on GPU at large-v3 quality with the decoder cut from 32 to 4 layers, so decoding is much faster than large-v3." }, { "q": "What are the limits of Whisper large-v3-turbo?", "a": "Word-level timing through the transformers pipeline (Live Nation measured 9.4 GB peak and wrong times); CPU-only clusters (the code forces the whisper-1 API when there is no GPU); music and non-speech audio." }, { "q": "Can Whisper large-v3-turbo be used commercially?", "a": "Yes. MIT weights and code; no use restrictions." }, { "q": "How does Whisper large-v3-turbo run on Mixpeek?", "a": "Ray Serve app in the inference registry. multimodal_extractor loads it in-process (WhisperLocalTranscriptionBatch, GPU) only when the whisper-1 API is off; prod runs HARDWARE_WHISPER_USE_API=true per infra/hardware/config.py comments." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/openai/whisper-large-v3-turbo", "https://huggingface.co/api/models/openai/whisper-large-v3-turbo", "https://arxiv.org/abs/2212.04356", "server/engine/inference/openai/whisper_large_v3_turbo/services.py", "server/engine/extractors/multimodal_extractor/v2/pipeline.py", "server/infra/hardware/config.py", "customers/livenation/manifest-loop/README.md", "customers/livenation/manifest-loop/plugin/ln_shots/v1/models.py" ], "registry": { "openai__whisper_large_v3_turbo": { "deployment": "openai__whisper_large_v3_turbo", "service_id": "openai/whisper-large-v3-turbo", "routes": "engine/inference/openai/whisper_large_v3_turbo/routes.py", "route_prefix": "/openai/whisper_large_v3_turbo/v1", "enabled": { "expr": "not _whisper_use_api" }, "description": "OpenAI Whisper Large V3 Turbo speech recognition model. Transcribes audio to text with state-of-the-art accuracy. Supports multiple languages and audio formats.", "capabilities": { "category": "transcription", "modality_input": [ "audio" ], "modality_output": "text", "supports_batch": false, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "target_ongoing_requests": 5, "num_cpus": 0.5, "num_gpus": { "expr": "float(os.getenv('HARDWARE_WHISPER_NUM_GPUS', '0.5')) if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.openai.whisper_large_v3_turbo.models.InferenceRequest", "output": "shared.inference.openai.whisper_large_v3_turbo.models.InferenceResponse", "config": "shared.inference.openai.whisper_large_v3_turbo.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "transcription", "family": "whisper", "license": "MIT", "commercial_use": true, "modalities_in": [ "audio", "video" ], "modalities_out": [ "text" ], "use_cases": [ "transcripts" ], "recommended": true, "release_date": "2024-10", "params_millions": 809.0, "size_on_disk_mb": 1618.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": null, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 4.0 }, "rank_score": 4.0 }, { "id": "gemini-2-5-flash", "identity": { "family": "gemini", "display_name": "Gemini 2.5 Flash", "hub_id": null, "api_name": "gemini-2.5-flash", "revision": null, "variants": [], "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "commercial_use_note": "Paid Google API under Google's terms; tenants can bring their own key.", "release_date": "2025-06", "paper_url": "https://arxiv.org/abs/2507.06261", "repo_url": null }, "task": { "category": "generation", "modalities_in": [ "text", "image", "video", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "The extractors' default for structured extraction and vision work: video description and OCR in multimodal_extractor v2, web page extraction, PDF layout (document_graph), scrolling-text OCR.", "worst_at": "It is a thinking model: the thinking budget can eat the output tokens on long extractions (llm_extractor.py comments); slower and costlier than gemini-3.5-flash-lite." }, "io": { "input": { "text": "prompt plus content", "image": "page images, frames, panoramas", "video": "segments up to 120 s" }, "output": { "text": "free text, or JSON against response_shape / response_schema" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v2", "selected_by": "fixed (description and ocr_text when run_video_description or run_ocr)" }, { "extractor": "web_scraper@v1", "selected_by": "llm_model unset (default), also forced when a blocked expensive model is named" }, { "extractor": "document_graph_extractor@v1", "selected_by": "vlm_model unset (default gemini-2.5-flash)" }, { "extractor": "text_extractor@v1", "selected_by": "response_shape set and llm_model unset (llm_provider google)" }, { "extractor": "scrolling_text_extractor@v1", "selected_by": "fixed (panorama OCR via CentralLLMService)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "multimodal_extractor v2: Gemini description and OCR text -> E5 description and OCR vectors, and the description feeds the Gemini Embedding 2 vector" ] }, "run": { "status": "api", "status_reason": "Provider API, called from extractor batch steps through CentralLLMService or the Gemini video client; universal__api routes it for POST /v1/inference.", "serve_apps": [ "universal__api" ], "runtime": "Gemini developer API or Vertex AI (GoogleModel.GEMINI_2_5_FLASH, GoogleProvider)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Prompt per extractor; structured output forced to JSON when OCR and description must be split.", "postprocessing": "JSON parsed into columns (description, ocr_text, extracted_data).", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Billed per token by Google; no committed throughput and cost measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-2.5-flash" ], "formats": [ { "ref": "api://google/gemini-2.5-flash", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 2.5 Flash: language model", "description": "Gemini 2.5 Flash is a language model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 2.5 Flash", "gemini", "language", "gemini-2.5-flash", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 2.5 Flash best at?", "a": "The extractors' default for structured extraction and vision work: video description and OCR in multimodal_extractor v2, web page extraction, PDF layout (document_graph), scrolling-text OCR." }, { "q": "What are the limits of Gemini 2.5 Flash?", "a": "It is a thinking model: the thinking budget can eat the output tokens on long extractions (llm_extractor.py comments); slower and costlier than gemini-3.5-flash-lite." }, { "q": "Can Gemini 2.5 Flash be used commercially?", "a": "Yes. Paid Google API under Google's terms; tenants can bring their own key." }, { "q": "How does Gemini 2.5 Flash run on Mixpeek?", "a": "Provider API, called from extractor batch steps through CentralLLMService or the Gemini video client; universal__api routes it for POST /v1/inference." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/models", "https://arxiv.org/abs/2507.06261", "server/engine/extractors/multimodal_extractor/v2/pipeline.py", "server/engine/extractors/web_scraper/v1/pipeline.py", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/document_graph_extractor/v1/definition.py", "server/engine/extractors/text_extractor/v1/pipeline.py", "server/engine/extractors/scrolling_text_extractor/v1/processors/scrolling_text_batch.py" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [ "footage-intelligence" ], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 1, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "generation", "family": "gemini", "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "modalities_in": [ "text", "image", "video", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": "2025-06", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "batch", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 1, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.5 }, "rank_score": 3.5 }, { "id": "clap-htsat-tiny", "identity": { "family": "laion-clap", "display_name": "LAION CLAP HTSAT-tiny (630k-audioset-best)", "hub_id": "lukewys/laion_clap", "api_name": null, "revision": "630k-audioset-best.pt", "variants": [], "license": "CC0-1.0", "commercial_use": true, "commercial_use_note": "The checkpoint repo is CC0-1.0 (public domain dedication). Training data includes AudioSet and LAION-Audio-630K; no license restriction on the weights.", "release_date": "2023-02", "paper_url": "https://arxiv.org/abs/2211.06687", "repo_url": "https://github.com/LAION-AI/CLAP" }, "task": { "category": "embedding", "modalities_in": [ "audio", "text" ], "modalities_out": [ "vector" ], "best_at": "Sound-alike search over short audio windows (jingles, music beds, effects) and text-to-audio search (\"upbeat jingle with bells\") in one 512-d space.", "worst_at": "Speech content (use whisper + E5); exact duplicate detection where an acoustic fingerprint algorithm is cheaper; audio longer than the 5 s window is seen only as overlapping windows." }, "io": { "input": { "audio": { "sample_rate": 48000, "window_s": 5.0, "hop_s": 2.5, "channels": "mono" }, "text": "free-text description of a sound" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "audio_fingerprint_extractor@v1", "selected_by": "fixed" } ], "feature_uris": [ "mixpeek://audio_fingerprint_extractor@v1/laion__clap_htsat_tiny" ], "vector_indexes": [ "audio_fingerprint_extractor_v1_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "audio or video -> 5 s windows with 2.5 s hop -> CLAP vector per window -> feature_search by audio clip or text" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry (query embeds); audio_fingerprint_extractor loads the same checkpoint in-process (AudioFingerprintBatchService).", "serve_apps": [ "laion__clap_htsat_tiny" ], "runtime": "laion_clap.CLAP_Module(enable_fusion=False) + load_ckpt (HTSAT-tiny audio encoder, RoBERTa text encoder)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 1864.0, "params_millions": null, "preprocessing": "Extract audio from video, resample to 48 kHz, slice 5 s windows every 2.5 s; text goes through the CLAP text tower.", "postprocessing": "get_audio_embedding_from_filelist / get_text_embedding (512 dims), normalized when normalize_embeddings is true.", "serving_shape": "both", "batch_size": 8, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "memory_gb": 2, "num_cpus": 1.0, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://lukewys/laion_clap@630k-audioset-best.pt" ], "formats": [ { "ref": "hf://lukewys/laion_clap@630k-audioset-best.pt", "format": "pytorch_pt", "file": "7 files, e.g. 630k-audioset-best.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "audio-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "LAION CLAP HTSAT-tiny (630k-audioset-best): embedding model", "description": "LAION CLAP HTSAT-tiny (630k-audioset-best) is a audio and text embedding model (CC0-1.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "LAION CLAP HTSAT-tiny (630k-audioset-best)", "laion-clap", "embedding", "lukewys/laion_clap", "audio-search" ], "faqs": [ { "q": "What is LAION CLAP HTSAT-tiny (630k-audioset-best) best at?", "a": "Sound-alike search over short audio windows (jingles, music beds, effects) and text-to-audio search (\"upbeat jingle with bells\") in one 512-d space." }, { "q": "What are the limits of LAION CLAP HTSAT-tiny (630k-audioset-best)?", "a": "Speech content (use whisper + E5); exact duplicate detection where an acoustic fingerprint algorithm is cheaper; audio longer than the 5 s window is seen only as overlapping windows." }, { "q": "Can LAION CLAP HTSAT-tiny (630k-audioset-best) be used commercially?", "a": "Yes. The checkpoint repo is CC0-1.0 (public domain dedication). Training data includes AudioSet and LAION-Audio-630K; no license restriction on the weights." }, { "q": "How does LAION CLAP HTSAT-tiny (630k-audioset-best) run on Mixpeek?", "a": "Ray Serve app in the inference registry (query embeds); audio_fingerprint_extractor loads the same checkpoint in-process (AudioFingerprintBatchService)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/lukewys/laion_clap", "https://huggingface.co/api/models/lukewys/laion_clap", "https://arxiv.org/abs/2211.06687", "https://github.com/LAION-AI/CLAP", "server/engine/inference/laion/clap_htsat_tiny/services.py", "server/engine/extractors/audio_fingerprint_extractor/v1/definition.py", "server/engine/extractors/audio_fingerprint_extractor/v1/processors/processors.py" ], "registry": { "laion__clap_htsat_tiny": { "deployment": "laion__clap_htsat_tiny", "service_id": "laion/clap-htsat-tiny", "routes": "engine/inference/laion/clap_htsat_tiny/routes.py", "route_prefix": "/audio", "enabled": { "expr": "importlib.util.find_spec('laion_clap') is not None" }, "description": "CLAP HTSAT-tiny audio embedding service. Generates 512-d L2-normalized audio embeddings for audio fingerprint matching and similarity search. Also embeds TEXT queries into the same space (contrastive text-audio model) for natural-language soundtrack search.", "capabilities": { "category": "embedding", "modality_input": [ "audio", "text" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 512, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "target_ongoing_requests": 8, "num_cpus": 1.0, "num_gpus": 0, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "engine.inference.laion.clap_htsat_tiny.models.AudioEmbedRequest", "output": "engine.inference.laion.clap_htsat_tiny.models.AudioEmbedResponse", "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "laion-clap", "license": "CC0-1.0", "commercial_use": true, "modalities_in": [ "audio", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "audio-search" ], "recommended": false, "release_date": "2023-02", "params_millions": null, "size_on_disk_mb": 1864.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "clip-vit-l-14-laion2b-s32b-b82k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-L/14 LAION-2B s32B b82K", "hub_id": "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT on the Hugging Face repo. The model card calls the release research-oriented and LAION-2B is uncurated web data; no license restriction on use.", "release_date": "2022-09", "paper_url": "https://arxiv.org/abs/2212.07143", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image" ], "modalities_out": [ "vector" ], "best_at": "Image-to-image similarity from a large web-trained ViT-L/14 image tower.", "worst_at": "Text-to-image search: the Serve app loads only CLIPVisionModel and returns the raw CLS hidden state, which is not in CLIP's shared text-image space. No extractor writes this vector." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean/std (CLIPImageProcessor)" } }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "Realtime image embedding through POST /v1/inference only; laion_clip_vit_l_14_v1 is a named inference in the vector models but no extractor declares an index for it" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry, image modality only; no extractor uses it.", "serve_apps": [ "laion__CLIP_ViT_L_14_laion2B_s32B_b82K" ], "runtime": "transformers CLIPVisionModel + CLIPImageProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1711.0, "params_millions": 428.0, "preprocessing": "CLIPImageProcessor (resize, center crop 224, CLIP normalization).", "postprocessing": "last_hidden_state[:, 0] (CLS before post_layernorm and before the visual projection), L2-normalized. The checkpoint config gives vision hidden_size 1024 and projection_dim 768, so this vector is 1024-d while the registry declares 768.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": null, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-L-14-laion2B-s32B-b82K@1627032197142fbe2a7cfec626f4ced3ae60d07a" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-L-14-laion2B-s32B-b82K@1627032197142fbe2a7cfec626f4ced3ae60d07a", "format": "safetensors", "file": "2 files, e.g. model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://laion/CLIP-ViT-L-14-laion2B-s32B-b82K@1627032197142fbe2a7cfec626f4ced3ae60d07a", "format": "pytorch_bin", "file": "2 files, e.g. open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-L/14 LAION-2B s32B b82K: embedding model", "description": "OpenCLIP ViT-L/14 LAION-2B s32B b82K is a image embedding model (MIT). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "OpenCLIP ViT-L/14 LAION-2B s32B b82K", "openclip", "embedding", "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-L/14 LAION-2B s32B b82K best at?", "a": "Image-to-image similarity from a large web-trained ViT-L/14 image tower." }, { "q": "What are the limits of OpenCLIP ViT-L/14 LAION-2B s32B b82K?", "a": "Text-to-image search: the Serve app loads only CLIPVisionModel and returns the raw CLS hidden state, which is not in CLIP's shared text-image space. No extractor writes this vector." }, { "q": "Can OpenCLIP ViT-L/14 LAION-2B s32B b82K be used commercially?", "a": "Yes. MIT on the Hugging Face repo. The model card calls the release research-oriented and LAION-2B is uncurated web data; no license restriction on use." }, { "q": "How does OpenCLIP ViT-L/14 LAION-2B s32B b82K run on Mixpeek?", "a": "Ray Serve app in the inference registry, image modality only; no extractor uses it." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "https://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K/resolve/main/config.json", "https://huggingface.co/api/models/laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "https://arxiv.org/abs/2212.07143", "server/engine/inference/laion/clip_vit_l_14/services.py", "server/engine/inference/laion/clip_vit_l_14/routes.py", "server/shared/collection/features/vectors/models.py" ], "registry": { "laion__CLIP_ViT_L_14_laion2B_s32B_b82K": { "deployment": "laion__CLIP_ViT_L_14_laion2B_s32B_b82K", "service_id": "laion/CLIP-ViT-L-14-laion2B-s32B-b82K", "routes": "engine/inference/laion/clip_vit_l_14/routes.py", "route_prefix": "/laion_clip_vit_l_14/v1", "enabled": false, "description": "LAION CLIP ViT-L-14 vision transformer for image embeddings. Generates 768-dimensional embeddings optimized for educational content with better text-in-image understanding (slides, code screenshots).", "capabilities": { "category": "embedding", "modality_input": [ "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 768, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "target_ongoing_requests": 5, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.laion.clip_vit_l_14.models.InferenceRequest", "output": "shared.inference.laion.clip_vit_l_14.models.InferenceResponse", "config": "shared.inference.laion.clip_vit_l_14.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2022-09", "params_millions": 428.0, "size_on_disk_mb": 1711.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "dinov2-base", "identity": { "family": "dinov2", "display_name": "DINOv2 Base", "hub_id": "facebook/dinov2-base", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2023-04", "paper_url": "https://arxiv.org/abs/2304.07193", "repo_url": "https://github.com/facebookresearch/dinov2" }, "task": { "category": "embedding", "modalities_in": [ "image" ], "modalities_out": [ "vector" ], "best_at": "Image-to-image similarity on visual structure (layout, near-duplicates, the same object or scene) with no text in the loop.", "worst_at": "Text queries (no text tower, so text-to-image search is impossible); semantic matches across different-looking images, where SigLIP is better." }, "io": { "input": { "image": { "resolution": "resize shortest side 256, center crop 224", "color": "RGB", "normalize": "ImageNet mean/std" } }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "web_scraper@v1", "selected_by": "generate_structure_embeddings=true (default true); image documents only" } ], "feature_uris": [ "mixpeek://web_scraper@v1/facebook__dinov2_base" ], "vector_indexes": [ "facebook__dinov2_base" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "web_scraper: DINOv2 structure vector beside SigLIP image vectors for the same page images" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry (image queries); web_scraper loads facebook/dinov2-base in-process (DocTypeStructureEmbedder).", "serve_apps": [ "facebook__dinov2_base" ], "runtime": "transformers AutoModel + AutoImageProcessor (Dinov2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 346.0, "params_millions": 86.6, "preprocessing": "AutoImageProcessor defaults for the checkpoint (resize, center crop 224, ImageNet normalization).", "postprocessing": "CLS token of last_hidden_state, L2-normalized (768 dims).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "memory_gb": 1, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/dinov2-base@f9e44c814b77203eaa57a6bdbbd535f21ede1415", "mixpeek://system/dinov2_vit_b_14@1.0.0" ], "formats": [ { "ref": "hf://facebook/dinov2-base@f9e44c814b77203eaa57a6bdbbd535f21ede1415", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/dinov2-base@f9e44c814b77203eaa57a6bdbbd535f21ede1415", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "web-pages" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "DINOv2 Base: embedding model", "description": "DINOv2 Base is a image embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "DINOv2 Base", "dinov2", "embedding", "facebook/dinov2-base", "visual-search", "web-pages" ], "faqs": [ { "q": "What is DINOv2 Base best at?", "a": "Image-to-image similarity on visual structure (layout, near-duplicates, the same object or scene) with no text in the loop." }, { "q": "What are the limits of DINOv2 Base?", "a": "Text queries (no text tower, so text-to-image search is impossible); semantic matches across different-looking images, where SigLIP is better." }, { "q": "Can DINOv2 Base be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does DINOv2 Base run on Mixpeek?", "a": "Ray Serve app in the inference registry (image queries); web_scraper loads facebook/dinov2-base in-process (DocTypeStructureEmbedder)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/dinov2-base", "https://huggingface.co/api/models/facebook/dinov2-base", "https://arxiv.org/abs/2304.07193", "server/engine/inference/facebook/dinov2/services.py", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/web_scraper/v1/processors/doctype_embedders.py" ], "registry": { "facebook__dinov2_base": { "deployment": "facebook__dinov2_base", "service_id": "facebook/dinov2-base", "routes": "engine/inference/facebook/dinov2/routes.py", "route_prefix": "/facebook_dinov2_base/v1", "enabled": true, "description": "Facebook DINOv2 Base self-supervised vision model. Generates 768-dimensional embeddings for fine-grained visual structure comparison. Captures layout, spatial arrangement, and visual patterns rather than semantic content. Use alongside SigLIP for complementary visual analysis in drift detection.", "capabilities": { "category": "embedding", "modality_input": [ "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 768, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "target_ongoing_requests": 5, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 1, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.facebook.dinov2.models.InferenceRequest", "output": "shared.inference.facebook.dinov2.models.InferenceResponse", "config": "shared.inference.facebook.dinov2.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "dinov2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "web-pages" ], "recommended": false, "release_date": "2023-04", "params_millions": 86.6, "size_on_disk_mb": 346.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf", "mixpeek" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "distilbert-nsfw-text-classifier", "identity": { "family": "distilbert", "display_name": "DistilBERT NSFW text classifier", "hub_id": "eliasalbouzidi/distilbert-nsfw-text-classifier", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 on the repo; a community fine-tune of DistilBERT.", "release_date": "2024-04", "paper_url": "https://arxiv.org/abs/2412.00064", "repo_url": "https://huggingface.co/eliasalbouzidi/distilbert-nsfw-text-classifier" }, "task": { "category": "classification", "modalities_in": [ "text" ], "modalities_out": [ "labels", "scores" ], "best_at": "Cheap binary safe/nsfw screening of English text such as captions, titles and transcripts.", "worst_at": "Non-English text, sarcasm and context-dependent policy categories; it scores sexual content and does not cover other harms." }, "io": { "input": { "text": { "language": "English", "tokenizer": "distilbert" } }, "output": { "labels": [ "safe", "nsfw" ], "scores": "nsfw_score in [0, 1]; is_nsfw = nsfw_score >= threshold (default 0.5)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "classify" ], "composes_with": [ "classify task=nsfw on text documents; images and video go to the ViT classifier in the same app" ] }, "run": { "status": "served", "status_reason": "Text half of the mixpeek__content_classifier_v1 Serve app.", "serve_apps": [ "mixpeek__content_classifier_v1" ], "runtime": "transformers pipeline(\"text-classification\") (DistilBertForSequenceClassification)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 268.0, "params_millions": 67.0, "preprocessing": "Pipeline tokenizer defaults.", "postprocessing": "Softmax over labels (top_k=None); nsfw_score is the max over labels matching nsfw/unsafe/porn/explicit/questionable.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "memory_gb": 2, "num_cpus": 2.0, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://eliasalbouzidi/distilbert-nsfw-text-classifier@c0abb7a10abea1f50528885e9e38d51cca935516" ], "formats": [ { "ref": "hf://eliasalbouzidi/distilbert-nsfw-text-classifier@c0abb7a10abea1f50528885e9e38d51cca935516", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "moderation", "classification" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "DistilBERT NSFW text classifier: classification model", "description": "DistilBERT NSFW text classifier is a text classification model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "DistilBERT NSFW text classifier", "distilbert", "classification", "eliasalbouzidi/distilbert-nsfw-text-classifier", "moderation" ], "faqs": [ { "q": "What is DistilBERT NSFW text classifier best at?", "a": "Cheap binary safe/nsfw screening of English text such as captions, titles and transcripts." }, { "q": "What are the limits of DistilBERT NSFW text classifier?", "a": "Non-English text, sarcasm and context-dependent policy categories; it scores sexual content and does not cover other harms." }, { "q": "Can DistilBERT NSFW text classifier be used commercially?", "a": "Yes. Apache-2.0 on the repo; a community fine-tune of DistilBERT." }, { "q": "How does DistilBERT NSFW text classifier run on Mixpeek?", "a": "Text half of the mixpeek__content_classifier_v1 Serve app." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/eliasalbouzidi/distilbert-nsfw-text-classifier", "https://huggingface.co/api/models/eliasalbouzidi/distilbert-nsfw-text-classifier", "server/engine/inference/nsfw/services.py", "server/shared/inference/nsfw/models.py", "server/shared/retrievers/stages/enrich/classify/services.py" ], "registry": { "mixpeek__content_classifier_v1": { "deployment": "mixpeek__content_classifier_v1", "service_id": "mixpeek/content-classifier-v1", "routes": "engine/inference/nsfw/routes.py", "route_prefix": "/mixpeek_content_classifier/v1", "enabled": true, "description": "Multimodal content (NSFW) classifier. Classifies text, images, and video (via sparse frame sampling) as safe/unsafe. Powers the tenant upload gate and the retriever 'classify' stage. CPU-only, no GPU required.", "capabilities": { "category": "classification", "modality_input": [ "text", "image", "video" ], "modality_output": "structured", "supports_batch": false, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 300, "target_ongoing_requests": 5, "num_cpus": 2.0, "num_gpus": 0, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.nsfw.models.InferenceRequest", "output": "shared.inference.nsfw.models.InferenceResponse", "config": "shared.inference.nsfw.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "classification", "family": "distilbert", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "labels", "scores" ], "use_cases": [ "moderation", "classification" ], "recommended": false, "release_date": "2024-04", "params_millions": 67.0, "size_on_disk_mb": 268.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "gemini-3-5-flash-lite", "identity": { "family": "gemini", "display_name": "Gemini 3.5 Flash-Lite", "hub_id": null, "api_name": "gemini-3.5-flash-lite", "revision": null, "variants": [], "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "commercial_use_note": "Paid Google API under Google's terms; tenants can bring their own key.", "release_date": null, "paper_url": null, "repo_url": null }, "task": { "category": "generation", "modalities_in": [ "text", "image", "video", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "The platform's default LLM (DEFAULT_GEMINI_MODEL) for cheap per-document work in retriever stages (enrich, filter, summarize) and for video-segment description and OCR in multimodal_extractor v1.", "worst_at": "Multi-step reasoning and long structured extraction, where the code picks gemini-2.5-flash or escalates to gemini-2.5-pro; per-document cost grows with candidate count in llm_filter and llm_enrich." }, "io": { "input": { "text": "prompt plus document fields", "image": "URLs or inline data", "video": "segments (multimodal_extractor v1)" }, "output": { "text": "free text, or JSON when response_shape / response_schema is set" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v1", "selected_by": "fixed (video description and OCR when run_video_description or run_ocr)" }, { "extractor": "universal_extractor@v1", "selected_by": "fixed (GENERATION_MODEL)" }, { "extractor": "scrolling_text_extractor@v1", "selected_by": "fixed (direct Gemini OCR fallback, _OCR_FALLBACK_MODEL)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize", "agent_search", "agentic_enrich", "rerank" ], "composes_with": [ "feature_search -> llm_filter or llm_enrich (default model_name) -> summarize", "rerank with inference_name mixpeek__decision and decision_model=gemini-3.5-flash-lite" ] }, "run": { "status": "api", "status_reason": "Provider API. Retriever stages and extractors call it through CentralLLMService or google-genai directly; universal__api routes any gemini-* model for POST /v1/inference.", "serve_apps": [ "universal__api" ], "runtime": "Gemini developer API or Vertex AI (clients.providers.google.gemini_backend, shared.models.llm GoogleProvider)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Prompt templates per stage; media passed as URLs or inline data.", "postprocessing": "Text, or JSON parsed against the requested schema.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Billed per token by Google; no committed throughput and cost measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-3.5-flash-lite" ], "formats": [ { "ref": "api://google/gemini-3.5-flash-lite", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": true, "for": [ "tagging", "captioning", "ocr" ], "reason": "Platform default Gemini model since 2026-10-10 (shared/models/llm/gemini_models.py)." }, "demos": [], "seo": { "title": "Gemini 3.5 Flash-Lite: language model", "description": "Gemini 3.5 Flash-Lite is a language model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 3.5 Flash-Lite", "gemini", "language", "gemini-3.5-flash-lite", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 3.5 Flash-Lite best at?", "a": "The platform's default LLM (DEFAULT_GEMINI_MODEL) for cheap per-document work in retriever stages (enrich, filter, summarize) and for video-segment description and OCR in multimodal_extractor v1." }, { "q": "What are the limits of Gemini 3.5 Flash-Lite?", "a": "Multi-step reasoning and long structured extraction, where the code picks gemini-2.5-flash or escalates to gemini-2.5-pro; per-document cost grows with candidate count in llm_filter and llm_enrich." }, { "q": "Can Gemini 3.5 Flash-Lite be used commercially?", "a": "Yes. Paid Google API under Google's terms; tenants can bring their own key." }, { "q": "How does Gemini 3.5 Flash-Lite run on Mixpeek?", "a": "Provider API. Retriever stages and extractors call it through CentralLLMService or google-genai directly; universal__api routes any gemini-* model for POST /v1/inference." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash-lite", "server/shared/models/llm/gemini_models.py", "server/shared/retrievers/stages/enrich/llm/models.py", "server/shared/retrievers/stages/filter/llm/models.py", "server/shared/retrievers/stages/reduce/summarize/models.py", "server/shared/retrievers/stages/enrich/agentic/models.py", "server/engine/extractors/multimodal_extractor/v1/pipeline.py", "server/engine/extractors/universal_extractor/v1/processors/gemini_client.py", "server/engine/extractors/scrolling_text_extractor/v1/processors/scrolling_text_batch.py", "server/engine/inference/llm/routes.py" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "generation", "family": "gemini", "license": "Google Gemini API / Vertex AI terms of service", "commercial_use": true, "modalities_in": [ "text", "image", "video", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": true, "release_date": null, "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "insightface-buffalo-l", "identity": { "family": "insightface", "display_name": "InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K)", "hub_id": "insightface/buffalo_l", "api_name": null, "revision": null, "variants": [], "license": "InsightFace pretrained models, non-commercial research only (library code MIT)", "commercial_use": false, "commercial_use_note": "The insightface README and model_zoo state every pretrained model, downloaded automatically or manually, is for non-commercial research only; the MIT license covers the library code. face_identity_extractor bills customers for this pack, so a commercial license from InsightFace is needed.", "release_date": null, "paper_url": "https://arxiv.org/abs/1801.07698", "repo_url": "https://github.com/deepinsight/insightface" }, "task": { "category": "face", "modalities_in": [ "image", "video" ], "modalities_out": [ "boxes", "keypoints", "vector" ], "best_at": "Face detection, 5-point alignment and 512-d face identity vectors for finding the same person across images and video frames.", "worst_at": "Commercial use (license); tiny or heavily occluded faces below min_face_size; the parameter names do not match the weights: scrfd_2.5g and scrfd_10g both load SCRFD-10GF, and arcface_r100 / magface_r100 load a ResNet50 (w600k_r50), not an R100." }, "io": { "input": { "image": { "det_size": "640x640", "align": "5-point affine to 112x112" }, "video": "frames sampled by the extractor, optional dedup of the same face" }, "output": { "boxes": "face bounding boxes with detection scores", "keypoints": "5 landmarks per face (2d106 and 3d68 landmark models also in the pack)", "labels": "gender and age from the pack's genderage head", "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "face_identity_extractor@v1", "selected_by": "detection_model=scrfd_2.5g (default) or scrfd_10g; embedding fixed arcface_r100" } ], "feature_uris": [ "mixpeek://face_identity_extractor@v1/insightface__arcface" ], "vector_indexes": [ "face_identity_extractor_v1_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "image or video -> SCRFD boxes -> aligned 112x112 crops -> ArcFace 512-d vector per face -> feature_search with a face photo" ] }, "run": { "status": "served", "status_reason": "insightface__arcface Serve app; face_identity_extractor batch calls it (FaceProcessingBatchService, strict_serve, no local fallback).", "serve_apps": [ "insightface__arcface" ], "runtime": "insightface.app.FaceAnalysis(name=\"buffalo_l\") on onnxruntime, prepare(ctx_id from providers, det_size=(640, 640))", "weights_format": "onnx", "trust_remote_code": false, "size_on_disk_mb": 326.0, "params_millions": null, "preprocessing": "SCRFD detection at 640x640, 5-point affine alignment to 112x112.", "postprocessing": "ArcFace recognition embedding per aligned face (512 dims), L2-normalized; optional quality scoring and video dedup.", "serving_shape": "both", "batch_size": 8, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "memory_gb": 2, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement. The definition's detection latencies (2-3, 5-7, 10-15 ms) are descriptions, not committed measurements." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://deepinsight/insightface/models/buffalo_l.zip@v0.7" ], "formats": [], "use_cases": [ "face-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K)", "description": "InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K) is a face recognition model. Served on Mixpeek, scaling to zero when idle.", "keywords": [ "InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K)", "insightface", "face recognition", "insightface/buffalo_l", "face-search" ], "faqs": [ { "q": "What is InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K) best at?", "a": "Face detection, 5-point alignment and 512-d face identity vectors for finding the same person across images and video frames." }, { "q": "What are the limits of InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K)?", "a": "Commercial use (license); tiny or heavily occluded faces below min_face_size; the parameter names do not match the weights: scrfd_2.5g and scrfd_10g both load SCRFD-10GF, and arcface_r100 / magface_r100 load a ResNet50 (w600k_r50), not an R100." }, { "q": "Can InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K) be used commercially?", "a": "No. The insightface README and model_zoo state every pretrained model, downloaded automatically or manually, is for non-commercial research only; the MIT license covers the library code. face_identity_extractor bills customers for this pack, so a commercial license from InsightFace is needed." }, { "q": "How does InsightFace buffalo_l (SCRFD-10GF + ArcFace ResNet50 WebFace600K) run on Mixpeek?", "a": "insightface__arcface Serve app; face_identity_extractor batch calls it (FaceProcessingBatchService, strict_serve, no local fallback)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://github.com/deepinsight/insightface/blob/master/model_zoo/README.md", "https://github.com/deepinsight/insightface/blob/master/python-package/README.md", "https://arxiv.org/abs/1801.07698", "https://arxiv.org/abs/2105.04714", "server/engine/inference/insightface/routes.py", "server/engine/inference/insightface/scrfd/services.py", "server/engine/inference/insightface/arcface/services.py", "server/engine/extractors/face_identity_extractor/v1/definition.py", "server/engine/extractors/face_identity_extractor/v1/pipeline.py", "server/engine/extractors/face_identity_extractor/v1/processors/processors.py" ], "registry": { "insightface__arcface": { "deployment": "insightface__arcface", "service_id": "insightface/arcface", "routes": "engine/inference/insightface/routes.py", "route_prefix": "/face", "enabled": true, "description": "InsightFace ArcFace R100 face embedding service. Executes complete SOTA face recognition pipeline: SCRFD detection → 5-point affine alignment → ArcFace 512-d embedding generation. 99.83% accuracy on LFW benchmark.", "capabilities": { "category": "embedding", "modality_input": [ "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 512, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "target_ongoing_requests": 8, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "engine.inference.insightface.models.FaceEmbedRequest", "output": "engine.inference.insightface.models.FaceEmbedResponse", "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "face", "family": "insightface", "license": "InsightFace pretrained models, non-commercial research only (library code MIT)", "commercial_use": false, "modalities_in": [ "image", "video" ], "modalities_out": [ "boxes", "keypoints", "vector" ], "use_cases": [ "face-search" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": 326.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "onnx", "formats": [], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "insightface-buffalo-sc", "identity": { "family": "insightface", "display_name": "InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K)", "hub_id": "insightface/buffalo_sc", "api_name": null, "revision": null, "variants": [ "insightface/buffalo_s" ], "license": "InsightFace pretrained models, non-commercial research only (library code MIT)", "commercial_use": false, "commercial_use_note": "Same terms as every InsightFace pretrained pack: non-commercial research only per the insightface README and model_zoo.", "release_date": null, "paper_url": "https://arxiv.org/abs/2105.04714", "repo_url": "https://github.com/deepinsight/insightface" }, "task": { "category": "face", "modalities_in": [ "image", "video" ], "modalities_out": [ "boxes", "keypoints", "vector" ], "best_at": "The fastest face detector option (SCRFD-500MF) for high-volume frames where small faces matter less.", "worst_at": "Recall on small faces and identity accuracy: model_zoo lists buffalo_s/sc well below buffalo_l on IJB-C and MR-ALL. Commercial use is not licensed." }, "io": { "input": { "image": { "det_size": "640x640" } }, "output": { "boxes": "face bounding boxes with detection scores", "keypoints": "5 landmarks per face", "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "face_identity_extractor@v1", "selected_by": "detection_model=scrfd_500m (buffalo_sc detector; embeddings still from buffalo_l)" } ], "feature_uris": [ "mixpeek://face_identity_extractor@v1/insightface__arcface" ], "vector_indexes": [ "face_identity_extractor_v1_embedding" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "buffalo_s (variant) is loaded only for the Serve app's embedding_model=arcface_r50, a MobileFaceNet whose vectors are not comparable with buffalo_l's" ] }, "run": { "status": "served", "status_reason": "Loaded by the insightface__arcface Serve app when a request names detection_model=scrfd_500m or embedding_model=arcface_r50.", "serve_apps": [ "insightface__arcface" ], "runtime": "insightface.app.FaceAnalysis(name=\"buffalo_sc\") on onnxruntime (buffalo_s for arcface_r50)", "weights_format": "onnx", "trust_remote_code": false, "size_on_disk_mb": 16.0, "params_millions": null, "preprocessing": "SCRFD detection at 640x640.", "postprocessing": "Boxes and 5-point landmarks passed to alignment and the recognition model.", "serving_shape": "both", "batch_size": 8, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "memory_gb": 2, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://deepinsight/insightface/models/buffalo_sc.zip@v0.7" ], "formats": [], "use_cases": [ "face-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K)", "description": "InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K) is a face recognition model. Served on Mixpeek, scaling to zero when idle.", "keywords": [ "InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K)", "insightface", "face recognition", "insightface/buffalo_sc", "face-search" ], "faqs": [ { "q": "What is InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K) best at?", "a": "The fastest face detector option (SCRFD-500MF) for high-volume frames where small faces matter less." }, { "q": "What are the limits of InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K)?", "a": "Recall on small faces and identity accuracy: model_zoo lists buffalo_s/sc well below buffalo_l on IJB-C and MR-ALL. Commercial use is not licensed." }, { "q": "Can InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K) be used commercially?", "a": "No. Same terms as every InsightFace pretrained pack: non-commercial research only per the insightface README and model_zoo." }, { "q": "How does InsightFace buffalo_sc (SCRFD-500MF + MobileFaceNet WebFace600K) run on Mixpeek?", "a": "Loaded by the insightface__arcface Serve app when a request names detection_model=scrfd_500m or embedding_model=arcface_r50." } ] }, "last_verified": "2026-11-05", "sources": [ "https://github.com/deepinsight/insightface/blob/master/model_zoo/README.md", "https://github.com/deepinsight/insightface/blob/master/python-package/README.md", "https://arxiv.org/abs/2105.04714", "server/engine/inference/insightface/scrfd/services.py", "server/engine/inference/insightface/arcface/services.py", "server/engine/extractors/face_identity_extractor/v1/definition.py" ], "registry": { "insightface__arcface": { "deployment": "insightface__arcface", "service_id": "insightface/arcface", "routes": "engine/inference/insightface/routes.py", "route_prefix": "/face", "enabled": true, "description": "InsightFace ArcFace R100 face embedding service. Executes complete SOTA face recognition pipeline: SCRFD detection → 5-point affine alignment → ArcFace 512-d embedding generation. 99.83% accuracy on LFW benchmark.", "capabilities": { "category": "embedding", "modality_input": [ "image" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 512, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "target_ongoing_requests": 8, "num_cpus": 0.5, "num_gpus": { "expr": "hardware_config.gpu_fraction if hardware_config.gpu_concurrency > 0 else 0" }, "memory_gb": 2, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "engine.inference.insightface.models.FaceEmbedRequest", "output": "engine.inference.insightface.models.FaceEmbedResponse", "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "face", "family": "insightface", "license": "InsightFace pretrained models, non-commercial research only (library code MIT)", "commercial_use": false, "modalities_in": [ "image", "video" ], "modalities_out": [ "boxes", "keypoints", "vector" ], "use_cases": [ "face-search" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": 16.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "onnx", "formats": [], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "jina-embeddings-v2-base-code", "identity": { "family": "jina-embeddings-v2", "display_name": "Jina Embeddings v2 Base Code", "hub_id": "jinaai/jina-embeddings-v2-base-code", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; the modeling code is loaded from the hub (remote code).", "release_date": "2023-11", "paper_url": "https://arxiv.org/abs/2310.19923", "repo_url": "https://huggingface.co/jinaai/jina-embeddings-v2-base-code" }, "task": { "category": "embedding", "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "best_at": "Code snippets and natural-language-to-code search across about 30 programming languages, with up to 8192-token inputs (ALiBi).", "worst_at": "General prose (E5 is the text default); the loader must run hub code, so a hub change can change behavior without a version bump here." }, "io": { "input": { "text": { "max_tokens": 8192, "truncation": true }, "prompts": "none" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "web_scraper@v1", "selected_by": "generate_code_embeddings=true (default true); code blocks" } ], "feature_uris": [ "mixpeek://web_scraper@v1/jinaai__jina_embeddings_v2_base_code" ], "vector_indexes": [ "jinaai__jina_embeddings_v2_base_code" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "web_scraper: code-block vectors beside E5 page text vectors" ] }, "run": { "status": "served", "status_reason": "Ray Serve app in the inference registry (query embeds); web_scraper loads the same repo in-process (DocTypeCodeEmbedder).", "serve_apps": [ "jinaai__jina_embeddings_v2_base_code" ], "runtime": "transformers AutoModel + AutoTokenizer with trust_remote_code=True (JinaBert hub code), mean pooling", "weights_format": "safetensors", "trust_remote_code": true, "size_on_disk_mb": 322.0, "params_millions": 161.0, "preprocessing": "Tokenize with truncation at max_length (default 8192); fp16 on GPU.", "postprocessing": "Mean pool over the attention mask, L2-normalize (768 dims).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 8, "idle_timeout_s": 300, "memory_gb": 1, "num_cpus": 0.25, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement for this model." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://jinaai/jina-embeddings-v2-base-code@516f4baf13dec4ddddda8631e019b5737c8bc250" ], "formats": [ { "ref": "hf://jinaai/jina-embeddings-v2-base-code@516f4baf13dec4ddddda8631e019b5737c8bc250", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://jinaai/jina-embeddings-v2-base-code@516f4baf13dec4ddddda8631e019b5737c8bc250", "format": "onnx", "file": "3 files, e.g. onnx/model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://jinaai/jina-embeddings-v2-base-code@516f4baf13dec4ddddda8631e019b5737c8bc250", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "code-search", "web-pages" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Jina Embeddings v2 Base Code: embedding model", "description": "Jina Embeddings v2 Base Code is a text embedding model (Apache-2.0). Served on Mixpeek, scaling to zero when idle.", "keywords": [ "Jina Embeddings v2 Base Code", "jina-embeddings-v2", "embedding", "jinaai/jina-embeddings-v2-base-code", "code-search", "web-pages" ], "faqs": [ { "q": "What is Jina Embeddings v2 Base Code best at?", "a": "Code snippets and natural-language-to-code search across about 30 programming languages, with up to 8192-token inputs (ALiBi)." }, { "q": "What are the limits of Jina Embeddings v2 Base Code?", "a": "General prose (E5 is the text default); the loader must run hub code, so a hub change can change behavior without a version bump here." }, { "q": "Can Jina Embeddings v2 Base Code be used commercially?", "a": "Yes. Apache-2.0 weights; the modeling code is loaded from the hub (remote code)." }, { "q": "How does Jina Embeddings v2 Base Code run on Mixpeek?", "a": "Ray Serve app in the inference registry (query embeds); web_scraper loads the same repo in-process (DocTypeCodeEmbedder)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/jinaai/jina-embeddings-v2-base-code", "https://huggingface.co/api/models/jinaai/jina-embeddings-v2-base-code", "https://arxiv.org/abs/2310.19923", "server/engine/inference/jinaai/jina_embeddings_v2_base_code/services.py", "server/engine/extractors/web_scraper/v1/processors/doctype_embedders.py", "server/engine/extractors/web_scraper/v1/definition.py" ], "registry": { "jinaai__jina_embeddings_v2_base_code": { "deployment": "jinaai__jina_embeddings_v2_base_code", "service_id": "jinaai/jina-embeddings-v2-base-code", "routes": "engine/inference/jinaai/jina_embeddings_v2_base_code/routes.py", "route_prefix": "/jina_embeddings_v2_base_code/v1", "enabled": true, "description": "Jina Embeddings V2 Base Code embedding model. Generates 768-dimensional dense embeddings for code. Supports 8192 token context with code-optimized representations.", "capabilities": { "category": "embedding", "modality_input": [ "text" ], "modality_output": "vector", "supports_batch": true, "reranking_mode": null }, "vector_config": { "dimensions": 768, "distance": "cosine", "vector_type": "dense" }, "serve": { "min_replicas": 0, "max_replicas": 8, "idle_timeout_s": 300, "target_ongoing_requests": 3, "num_cpus": 0.25, "num_gpus": 0, "memory_gb": 1, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": "shared.inference.jinaai.jina_embeddings_v2_base_code.models.InferenceRequest", "output": "shared.inference.jinaai.jina_embeddings_v2_base_code.models.InferenceResponse", "config": "shared.inference.jinaai.jina_embeddings_v2_base_code.models.InferenceConfigs" } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "served", "category": "embedding", "family": "jina-embeddings-v2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "use_cases": [ "code-search", "web-pages" ], "recommended": false, "release_date": "2023-11", "params_millions": 161.0, "size_on_disk_mb": 322.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": true, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "siglip2-so400m-patch14-384", "identity": { "family": "siglip2", "display_name": "SigLIP 2 so400m patch14 (224/384)", "hub_id": "google/siglip2-so400m-patch14-384", "api_name": null, "revision": null, "variants": [ "google/siglip2-so400m-patch14-224" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-17", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Logos, signage, brands and venue shots in concert footage (Live Nation visual arm); strongest SigLIP 2 below giant.", "worst_at": "Alone it trails text retrieval on speech and moment queries (Live Nation visual F1@10 0.348 vs text 0.414); 1.1B params, GPU in practice." }, "io": { "input": { "image": { "resolution": "384x384 (224 sibling)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 1152, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "ln_shots@v1", "selected_by": "fixed (customer plugin, customers/livenation/manifest-loop)" } ], "feature_uris": [ "mixpeek://ln_shots@v1/ln_visual" ], "vector_indexes": [ "ln_visual" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "shots -> one to three frames per shot -> SigLIP 2 vectors (mean) -> feature_search", "SigLIP 2 visual + Qwen3-Embedding text + BM25 on search_text -> DBSF fusion (Live Nation tri retriever)" ] }, "run": { "status": "plugin", "status_reason": "Runs today as the Live Nation customer plugin ln_shots@v1 in the standalone image (safetensors, local_files_only, trust_remote_code false); not a built-in Serve app.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4544.0, "params_millions": 1136.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 8.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "Live Nation marketing clips, 10 clips, 263 shots, 56 queries", "metric": "F1@10, visual arm alone", "k": 10, "score": 0.348, "baseline": "text arm alone (Qwen3-Embedding-0.6B)", "baseline_score": 0.414, "held_out": false, "date": "2026-11-04", "source": "customers/livenation/manifest-loop/README.md" }, { "dataset": "Live Nation held-out clips, 5 clips, 28 queries", "metric": "F1@10, SYSTEM: tri retriever (SigLIP 2 visual + Qwen3 text + BM25, DBSF)", "k": 10, "score": 0.406, "baseline": null, "baseline_score": null, "held_out": true, "date": "2026-11-04", "source": "customers/livenation/manifest-loop/DRIVER-RUN.md" } ] }, "refs": [ "hf://google/siglip2-so400m-patch14-384@e8e487298228002f3d8a82e0cd5c8ea9c567f57f" ], "formats": [ { "ref": "hf://google/siglip2-so400m-patch14-384@e8e487298228002f3d8a82e0cd5c8ea9c567f57f", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "brand-detection", "multilingual-search" ], "recommended": { "value": true, "for": [ "visual-search", "brand-detection" ], "reason": "Apache-2.0, safetensors, no remote code; carried brand and venue queries in the Live Nation held-out eval." }, "demos": [], "seo": { "title": "SigLIP 2 so400m patch14 (224/384): embedding model", "description": "SigLIP 2 so400m patch14 (224/384) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 so400m patch14 (224/384)", "siglip2", "embedding", "google/siglip2-so400m-patch14-384", "visual-search", "brand-detection", "multilingual-search" ], "faqs": [ { "q": "What is SigLIP 2 so400m patch14 (224/384) best at?", "a": "Logos, signage, brands and venue shots in concert footage (Live Nation visual arm); strongest SigLIP 2 below giant." }, { "q": "What are the limits of SigLIP 2 so400m patch14 (224/384)?", "a": "Alone it trails text retrieval on speech and moment queries (Live Nation visual F1@10 0.348 vs text 0.414); 1.1B params, GPU in practice." }, { "q": "Can SigLIP 2 so400m patch14 (224/384) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 so400m patch14 (224/384) run on Mixpeek?", "a": "Runs today as the Live Nation customer plugin ln_shots@v1 in the standalone image (safetensors, local_files_only, trust_remote_code false); not a built-in Serve app." }, { "q": "What does SigLIP 2 so400m patch14 (224/384) output?", "a": "A 1152-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-so400m-patch14-384", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2", "customers/livenation/manifest-loop/plugin/ln_shots/v1/models.py", "customers/livenation/manifest-loop/README.md", "customers/livenation/manifest-loop/DRIVER-RUN.md" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "brand-detection", "multilingual-search" ], "recommended": true, "release_date": "2025-02-17", "params_millions": 1136.0, "size_on_disk_mb": 4544.0, "vector_dims": 1152, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 8.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 3.0 }, "rank_score": 3.0 }, { "id": "claude-sonnet-4-5", "identity": { "family": "claude", "display_name": "Claude Sonnet 4.5", "hub_id": null, "api_name": "claude-sonnet-4-5-20250929", "revision": "20250929", "variants": [], "license": "Anthropic commercial terms of service", "commercial_use": true, "commercial_use_note": "Paid Anthropic API under Anthropic's commercial terms; agent sessions bill on the platform key.", "release_date": "2025-09-29", "paper_url": null, "repo_url": null }, "task": { "category": "generation", "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "best_at": "Multi-step tool use and reasoning; the default model of agent sessions (DEFAULT_AGENT_MODEL) and of agentic_enrich when no provider or model is given.", "worst_at": "Per-document cost: agentic_enrich makes 3 to 15 calls per document, so it needs a when filter; no video or audio input." }, "io": { "input": { "text": "messages and tool results", "image": "image URLs" }, "output": { "text": "messages, tool calls, structured fields" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "agentic_enrich" ], "composes_with": [ "feature_search -> agentic_enrich (Claude Sonnet 4.5 by default, gemini-2.5-pro as escalation model)", "agent sessions (api.agents) call Anthropic only" ] }, "run": { "status": "api", "status_reason": "Provider API. Agent sessions and agentic_enrich call Anthropic directly; universal__api routes claude-* models for POST /v1/inference.", "serve_apps": [ "universal__api" ], "runtime": "Anthropic Messages API (shared.models.llm Anthropic provider, api.agents.services)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Agent or stage prompt with tool definitions.", "postprocessing": "Tool calls executed by the runtime; final text parsed into fields.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Billed per token by Anthropic; no committed measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://anthropic/claude-sonnet-4-5-20250929" ], "formats": [ { "ref": "api://anthropic/claude-sonnet-4-5-20250929", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "tagging", "captioning" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Claude Sonnet 4.5: language model", "description": "Claude Sonnet 4.5 is a text and image language model (Anthropic commercial terms of service). Called on Mixpeek through the provider API.", "keywords": [ "Claude Sonnet 4.5", "claude", "language", "claude-sonnet-4-5-20250929", "tagging", "captioning" ], "faqs": [ { "q": "What is Claude Sonnet 4.5 best at?", "a": "Multi-step tool use and reasoning; the default model of agent sessions (DEFAULT_AGENT_MODEL) and of agentic_enrich when no provider or model is given." }, { "q": "What are the limits of Claude Sonnet 4.5?", "a": "Per-document cost: agentic_enrich makes 3 to 15 calls per document, so it needs a when filter; no video or audio input." }, { "q": "Can Claude Sonnet 4.5 be used commercially?", "a": "Yes. Paid Anthropic API under Anthropic's commercial terms; agent sessions bill on the platform key." }, { "q": "How does Claude Sonnet 4.5 run on Mixpeek?", "a": "Provider API. Agent sessions and agentic_enrich call Anthropic directly; universal__api routes claude-* models for POST /v1/inference." } ] }, "last_verified": "2026-11-05", "sources": [ "https://www.anthropic.com/news/claude-sonnet-4-5", "server/shared/agents/models.py", "server/api/agents/services.py", "server/shared/retrievers/stages/enrich/agentic/models.py", "server/engine/inference/llm/routes.py" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "generation", "family": "claude", "license": "Anthropic commercial terms of service", "commercial_use": true, "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "tagging", "captioning" ], "recommended": false, "release_date": "2025-09-29", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "d-fine-l-coco", "identity": { "family": "d-fine", "display_name": "D-FINE L (COCO)", "hub_id": "ustc-community/dfine-large-coco", "api_name": null, "revision": null, "variants": [ "ustc-community/dfine-small-coco (10M)", "ustc-community/dfine-xlarge-obj2coco (63M, Objects365 pretrain)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-10", "paper_url": "https://arxiv.org/abs/2410.13842", "repo_url": "https://github.com/Peterande/D-FINE" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Real-time DETR with fine-grained box refinement; among the highest COCO AP per latency of the permissive detectors.", "worst_at": "Closed COCO label set." }, "io": { "input": { "image": { "resolution": "640x640", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels (post_process_object_detection)", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "plugin", "status_reason": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops.", "serve_apps": [], "runtime": "transformers DFineForObjectDetection + RTDetrImageProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 126.0, "params_millions": 31.0, "preprocessing": "Resize to 640x640, rescale to 0..1 (image processor).", "postprocessing": "Sigmoid class scores, top-k queries, threshold; no NMS (one-to-one matching).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://ustc-community/dfine-large-coco@aa8f8211545e4cff5adf52ea512f9ac74a9824ec" ], "formats": [ { "ref": "hf://ustc-community/dfine-large-coco@aa8f8211545e4cff5adf52ea512f9ac74a9824ec", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "object-detection" ], "recommended": { "value": true, "for": [ "object-detection" ], "reason": "Apache-2.0 detector with safetensors and transformers support: the permissive choice over AGPL YOLO. Not yet evaluated by Mixpeek." }, "demos": [], "seo": { "title": "D-FINE L (COCO): object detection model", "description": "D-FINE L (COCO) is a image object detection model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "D-FINE L (COCO)", "d-fine", "object detection", "ustc-community/dfine-large-coco", "object-detection" ], "faqs": [ { "q": "What is D-FINE L (COCO) best at?", "a": "Real-time DETR with fine-grained box refinement; among the highest COCO AP per latency of the permissive detectors." }, { "q": "What are the limits of D-FINE L (COCO)?", "a": "Closed COCO label set." }, { "q": "Can D-FINE L (COCO) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does D-FINE L (COCO) run on Mixpeek?", "a": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/ustc-community/dfine-large-coco", "https://arxiv.org/abs/2410.13842", "https://github.com/Peterande/D-FINE" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "detection", "family": "d-fine", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": true, "release_date": "2024-10", "params_millions": 31.0, "size_on_disk_mb": 126.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gemini-2-5-flash-lite", "identity": { "family": "gemini-flash", "display_name": "Gemini 2.5 Flash-Lite", "hub_id": null, "api_name": "gemini-2.5-flash-lite", "revision": null, "variants": [], "license": "Proprietary (Google API terms)", "commercial_use": true, "commercial_use_note": "Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models.", "release_date": "2025-07-22", "paper_url": null, "repo_url": null }, "task": { "category": "video_qa", "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "Cheapest Gemini video tagging and captioning at 1 fps.", "worst_at": "Lower reasoning quality; no longer served to new developer API keys." }, "io": { "input": { "video": { "frames": "1 frame per second by default (static mode); custom fps via the processing config", "tokens_per_frame": "258 at default media_resolution, 66 at low", "audio": "32 tokens per second of audio track", "max_length": "about 1 hour at default resolution or 3 hours at low resolution in a 1M-token context", "videos_per_request": "up to 10 (Gemini 2.5 and later)", "formats": "mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp; public YouTube URLs (preview)" }, "prompts": "free-form instructions; response_schema for JSON output", "max_tokens": "1M-token input context (Flash family)" }, "output": { "text": "free-form or JSON matching response_schema (tags, descriptions, timestamps MM:SS)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize" ], "composes_with": [ "scene splits -> Gemini Flash per scene -> tags and descriptions -> E5 text vectors + BM25", "retriever results -> llm_enrich with this model -> structured fields" ] }, "run": { "status": "api", "status_reason": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). The developer API stopped serving it to new keys (platform default moved 2026-10-10).", "serve_apps": [], "runtime": "Google Gen AI SDK (Gemini API / Vertex AI) via shared/models/llm/providers/google.py", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Upload video through the Files API or pass a URI; set fps and media_resolution as needed.", "postprocessing": "Parse JSON when response_schema is set; meter tokens by modality (shared/billing/model_prices.py).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Provider price, not a measurement: input 0.10 (video 0.10, audio 0.30), output 0.40 USD per 1M tokens (server/shared/billing/model_prices.py). Per-video cost is unmeasured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-2.5-flash-lite" ], "formats": [ { "ref": "api://google/gemini-2.5-flash-lite", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 2.5 Flash-Lite: video understanding model", "description": "Gemini 2.5 Flash-Lite is a video understanding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 2.5 Flash-Lite", "gemini-flash", "video understanding", "gemini-2.5-flash-lite", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 2.5 Flash-Lite best at?", "a": "Cheapest Gemini video tagging and captioning at 1 fps." }, { "q": "What are the limits of Gemini 2.5 Flash-Lite?", "a": "Lower reasoning quality; no longer served to new developer API keys." }, { "q": "Can Gemini 2.5 Flash-Lite be used commercially?", "a": "Yes. Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models." }, { "q": "How does Gemini 2.5 Flash-Lite run on Mixpeek?", "a": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). The developer API stopped serving it to new keys (platform default moved 2026-10-10)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/video-understanding", "https://ai.google.dev/gemini-api/docs/models", "server/shared/models/llm/gemini_models.py", "server/shared/billing/model_prices.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "video_qa", "family": "gemini-flash", "license": "Proprietary (Google API terms)", "commercial_use": true, "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": "2025-07-22", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gemini-3-1-flash-lite", "identity": { "family": "gemini-flash", "display_name": "Gemini 3.1 Flash-Lite", "hub_id": null, "api_name": "gemini-3.1-flash-lite", "revision": null, "variants": [], "license": "Proprietary (Google API terms)", "commercial_use": true, "commercial_use_note": "Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models.", "release_date": null, "paper_url": null, "repo_url": null }, "task": { "category": "video_qa", "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "Low-cost bulk tagging of scenes and frames.", "worst_at": "Weaker than 3.x Flash on long or multi-step video questions." }, "io": { "input": { "video": { "frames": "1 frame per second by default (static mode); custom fps via the processing config", "tokens_per_frame": "258 at default media_resolution, 66 at low", "audio": "32 tokens per second of audio track", "max_length": "about 1 hour at default resolution or 3 hours at low resolution in a 1M-token context", "videos_per_request": "up to 10 (Gemini 2.5 and later)", "formats": "mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp; public YouTube URLs (preview)" }, "prompts": "free-form instructions; response_schema for JSON output", "max_tokens": "1M-token input context (Flash family)" }, "output": { "text": "free-form or JSON matching response_schema (tags, descriptions, timestamps MM:SS)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize" ], "composes_with": [ "scene splits -> Gemini Flash per scene -> tags and descriptions -> E5 text vectors + BM25", "retriever results -> llm_enrich with this model -> structured fields" ] }, "run": { "status": "api", "status_reason": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Stable.", "serve_apps": [], "runtime": "Google Gen AI SDK (Gemini API / Vertex AI) via shared/models/llm/providers/google.py", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Upload video through the Files API or pass a URI; set fps and media_resolution as needed.", "postprocessing": "Parse JSON when response_schema is set; meter tokens by modality (shared/billing/model_prices.py).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Provider price, not a measurement: input 0.25 (audio 0.50), output 1.50 USD per 1M tokens (server/shared/billing/model_prices.py). Per-video cost is unmeasured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-3.1-flash-lite" ], "formats": [ { "ref": "api://google/gemini-3.1-flash-lite", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 3.1 Flash-Lite: video understanding model", "description": "Gemini 3.1 Flash-Lite is a video understanding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 3.1 Flash-Lite", "gemini-flash", "video understanding", "gemini-3.1-flash-lite", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 3.1 Flash-Lite best at?", "a": "Low-cost bulk tagging of scenes and frames." }, { "q": "What are the limits of Gemini 3.1 Flash-Lite?", "a": "Weaker than 3.x Flash on long or multi-step video questions." }, { "q": "Can Gemini 3.1 Flash-Lite be used commercially?", "a": "Yes. Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models." }, { "q": "How does Gemini 3.1 Flash-Lite run on Mixpeek?", "a": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Stable." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/video-understanding", "https://ai.google.dev/gemini-api/docs/models", "server/shared/models/llm/gemini_models.py", "server/shared/billing/model_prices.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "video_qa", "family": "gemini-flash", "license": "Proprietary (Google API terms)", "commercial_use": true, "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gemini-3-6-flash", "identity": { "family": "gemini-flash", "display_name": "Gemini 3.6 Flash", "hub_id": null, "api_name": "gemini-3.6-flash", "revision": null, "variants": [], "license": "Proprietary (Google API terms)", "commercial_use": true, "commercial_use_note": "Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models.", "release_date": null, "paper_url": null, "repo_url": null }, "task": { "category": "video_qa", "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "High-quality video QA and structured tagging.", "worst_at": "Five times the input price of 3.5 Flash-Lite." }, "io": { "input": { "video": { "frames": "1 frame per second by default (static mode); custom fps via the processing config", "tokens_per_frame": "258 at default media_resolution, 66 at low", "audio": "32 tokens per second of audio track", "max_length": "about 1 hour at default resolution or 3 hours at low resolution in a 1M-token context", "videos_per_request": "up to 10 (Gemini 2.5 and later)", "formats": "mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp; public YouTube URLs (preview)" }, "prompts": "free-form instructions; response_schema for JSON output", "max_tokens": "1M-token input context (Flash family)" }, "output": { "text": "free-form or JSON matching response_schema (tags, descriptions, timestamps MM:SS)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize" ], "composes_with": [ "scene splits -> Gemini Flash per scene -> tags and descriptions -> E5 text vectors + BM25", "retriever results -> llm_enrich with this model -> structured fields" ] }, "run": { "status": "api", "status_reason": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Stable.", "serve_apps": [], "runtime": "Google Gen AI SDK (Gemini API / Vertex AI) via shared/models/llm/providers/google.py", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Upload video through the Files API or pass a URI; set fps and media_resolution as needed.", "postprocessing": "Parse JSON when response_schema is set; meter tokens by modality (shared/billing/model_prices.py).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Provider price, not a measurement: input 1.50, output 7.50 USD per 1M tokens (server/shared/billing/model_prices.py). Per-video cost is unmeasured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-3.6-flash" ], "formats": [ { "ref": "api://google/gemini-3.6-flash", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 3.6 Flash: video understanding model", "description": "Gemini 3.6 Flash is a video understanding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 3.6 Flash", "gemini-flash", "video understanding", "gemini-3.6-flash", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 3.6 Flash best at?", "a": "High-quality video QA and structured tagging." }, { "q": "What are the limits of Gemini 3.6 Flash?", "a": "Five times the input price of 3.5 Flash-Lite." }, { "q": "Can Gemini 3.6 Flash be used commercially?", "a": "Yes. Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models." }, { "q": "How does Gemini 3.6 Flash run on Mixpeek?", "a": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Stable." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/video-understanding", "https://ai.google.dev/gemini-api/docs/models", "server/shared/models/llm/gemini_models.py", "server/shared/billing/model_prices.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "video_qa", "family": "gemini-flash", "license": "Proprietary (Google API terms)", "commercial_use": true, "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gemini-3-8-flash", "identity": { "family": "gemini-flash", "display_name": "Gemini 3.8 Flash", "hub_id": null, "api_name": "gemini-3.8-flash", "revision": null, "variants": [], "license": "Proprietary (Google API terms)", "commercial_use": true, "commercial_use_note": "Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models.", "release_date": null, "paper_url": null, "repo_url": null }, "task": { "category": "video_qa", "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "Brand, artist, venue and moment tags per shot (the Live Nation ln_shots plugin default tag_model).", "worst_at": "Price matches 3.6 Flash; no Mixpeek evaluation of tag quality on its own." }, "io": { "input": { "video": { "frames": "1 frame per second by default (static mode); custom fps via the processing config", "tokens_per_frame": "258 at default media_resolution, 66 at low", "audio": "32 tokens per second of audio track", "max_length": "about 1 hour at default resolution or 3 hours at low resolution in a 1M-token context", "videos_per_request": "up to 10 (Gemini 2.5 and later)", "formats": "mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp; public YouTube URLs (preview)" }, "prompts": "free-form instructions; response_schema for JSON output", "max_tokens": "1M-token input context (Flash family)" }, "output": { "text": "free-form or JSON matching response_schema (tags, descriptions, timestamps MM:SS)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize" ], "composes_with": [ "scene splits -> Gemini Flash per scene -> tags and descriptions -> E5 text vectors + BM25", "retriever results -> llm_enrich with this model -> structured fields" ] }, "run": { "status": "api", "status_reason": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Newest Flash on the models page (read 2026-11-05).", "serve_apps": [], "runtime": "Google Gen AI SDK (Gemini API / Vertex AI) via shared/models/llm/providers/google.py", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Upload video through the Files API or pass a URI; set fps and media_resolution as needed.", "postprocessing": "Parse JSON when response_schema is set; meter tokens by modality (shared/billing/model_prices.py).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Provider price, not a measurement: input 1.50, output 7.50 USD per 1M tokens (server/shared/billing/model_prices.py). Per-video cost is unmeasured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-3.8-flash" ], "formats": [ { "ref": "api://google/gemini-3.8-flash", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 3.8 Flash: video understanding model", "description": "Gemini 3.8 Flash is a video understanding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 3.8 Flash", "gemini-flash", "video understanding", "gemini-3.8-flash", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 3.8 Flash best at?", "a": "Brand, artist, venue and moment tags per shot (the Live Nation ln_shots plugin default tag_model)." }, { "q": "What are the limits of Gemini 3.8 Flash?", "a": "Price matches 3.6 Flash; no Mixpeek evaluation of tag quality on its own." }, { "q": "Can Gemini 3.8 Flash be used commercially?", "a": "Yes. Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models." }, { "q": "How does Gemini 3.8 Flash run on Mixpeek?", "a": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Newest Flash on the models page (read 2026-11-05)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/video-understanding", "https://ai.google.dev/gemini-api/docs/models", "server/shared/models/llm/gemini_models.py", "server/shared/billing/model_prices.py", "customers/livenation/manifest-loop/plugin/ln_shots/v1/definition.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "video_qa", "family": "gemini-flash", "license": "Proprietary (Google API terms)", "commercial_use": true, "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": null, "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gemini-3-flash-preview", "identity": { "family": "gemini-flash", "display_name": "Gemini 3 Flash (preview)", "hub_id": null, "api_name": "gemini-3-flash-preview", "revision": null, "variants": [], "license": "Proprietary (Google API terms)", "commercial_use": true, "commercial_use_note": "Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models.", "release_date": "2025-12", "paper_url": null, "repo_url": null }, "task": { "category": "video_qa", "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "best_at": "Stronger video reasoning than 2.5 Flash at Flash latency.", "worst_at": "Preview: may change or be withdrawn." }, "io": { "input": { "video": { "frames": "1 frame per second by default (static mode); custom fps via the processing config", "tokens_per_frame": "258 at default media_resolution, 66 at low", "audio": "32 tokens per second of audio track", "max_length": "about 1 hour at default resolution or 3 hours at low resolution in a 1M-token context", "videos_per_request": "up to 10 (Gemini 2.5 and later)", "formats": "mp4, mpeg, mov, avi, x-flv, mpg, webm, wmv, 3gpp; public YouTube URLs (preview)" }, "prompts": "free-form instructions; response_schema for JSON output", "max_tokens": "1M-token input context (Flash family)" }, "output": { "text": "free-form or JSON matching response_schema (tags, descriptions, timestamps MM:SS)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "llm_enrich", "llm_filter", "summarize" ], "composes_with": [ "scene splits -> Gemini Flash per scene -> tags and descriptions -> E5 text vectors + BM25", "retriever results -> llm_enrich with this model -> structured fields" ] }, "run": { "status": "api", "status_reason": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Preview model.", "serve_apps": [], "runtime": "Google Gen AI SDK (Gemini API / Vertex AI) via shared/models/llm/providers/google.py", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Upload video through the Files API or pass a URI; set fps and media_resolution as needed.", "postprocessing": "Parse JSON when response_schema is set; meter tokens by modality (shared/billing/model_prices.py).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Provider price, not a measurement: input 0.50 (audio 1.00), output 3.00 USD (gemini-3-flash row) per 1M tokens (server/shared/billing/model_prices.py). Per-video cost is unmeasured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://google/gemini-3-flash-preview" ], "formats": [ { "ref": "api://google/gemini-3-flash-preview", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Gemini 3 Flash (preview): video understanding model", "description": "Gemini 3 Flash (preview) is a video understanding model. Called on Mixpeek through the provider API.", "keywords": [ "Gemini 3 Flash (preview)", "gemini-flash", "video understanding", "gemini-3-flash-preview", "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "faqs": [ { "q": "What is Gemini 3 Flash (preview) best at?", "a": "Stronger video reasoning than 2.5 Flash at Flash latency." }, { "q": "What are the limits of Gemini 3 Flash (preview)?", "a": "Preview: may change or be withdrawn." }, { "q": "Can Gemini 3 Flash (preview) be used commercially?", "a": "Yes. Google Gemini API paid-tier terms; outputs usable commercially; paid-tier data is not used to train Google models." }, { "q": "How does Gemini 3 Flash (preview) run on Mixpeek?", "a": "Provider API; accepted by name wherever the platform takes a Gemini model (shared/models/llm/gemini_models.py GEMINI_GENERATION_MODELS). Preview model." } ] }, "last_verified": "2026-11-05", "sources": [ "https://ai.google.dev/gemini-api/docs/video-understanding", "https://ai.google.dev/gemini-api/docs/models", "server/shared/models/llm/gemini_models.py", "server/shared/billing/model_prices.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "video_qa", "family": "gemini-flash", "license": "Proprietary (Google API terms)", "commercial_use": true, "modalities_in": [ "video", "audio", "image", "text", "document" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "video-understanding", "tagging", "captioning", "ocr", "brand-detection" ], "recommended": false, "release_date": "2025-12", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "gpt-4o-mini", "identity": { "family": "gpt-4o", "display_name": "GPT-4o mini", "hub_id": null, "api_name": "gpt-4o-mini", "revision": null, "variants": [], "license": "OpenAI API terms of use", "commercial_use": true, "commercial_use_note": "Paid OpenAI API under OpenAI's terms; tenants can bring their own key.", "release_date": "2024-07", "paper_url": null, "repo_url": null }, "task": { "category": "generation", "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "best_at": "Cheap short generations such as query expansion (query_expand's default expansion_model) and the fallback chat model of the universal inference endpoint.", "worst_at": "Video and audio input (needs another model); the stage's 3 s query_expand budget leaves little room for slow provider responses." }, "io": { "input": { "text": "chat messages", "image": "image URLs (universal__api converts base64 media to data URIs)" }, "output": { "text": "chat completion" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "query_expand", "llm_enrich" ], "composes_with": [ "query_expand (gpt-4o-mini) -> several feature_search calls fused", "llm_enrich with legacy inference_name openai:gpt-4o-mini resolves to universal/api with model gpt-4o-mini" ] }, "run": { "status": "api", "status_reason": "Provider API. universal__api uses it when a request names no model (params.get('model', 'gpt-4o-mini')); query_expand calls it through CentralLLMService.", "serve_apps": [ "universal__api" ], "runtime": "OpenAI chat completions (shared.models.llm OpenAIProvider; universal__api _handle_chat)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Stage prompt templates ({{QUERY}}, {{NUM_EXPANSIONS}} for query_expand).", "postprocessing": "Text, parsed into expansions or enrichment fields.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Billed per token by OpenAI; no committed measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://openai/gpt-4o-mini" ], "formats": [ { "ref": "api://openai/gpt-4o-mini", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "tagging", "captioning" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "GPT-4o mini: language model", "description": "GPT-4o mini is a text and image language model (OpenAI API terms of use). Called on Mixpeek through the provider API.", "keywords": [ "GPT-4o mini", "gpt-4o", "language", "gpt-4o-mini", "tagging", "captioning" ], "faqs": [ { "q": "What is GPT-4o mini best at?", "a": "Cheap short generations such as query expansion (query_expand's default expansion_model) and the fallback chat model of the universal inference endpoint." }, { "q": "What are the limits of GPT-4o mini?", "a": "Video and audio input (needs another model); the stage's 3 s query_expand budget leaves little room for slow provider responses." }, { "q": "Can GPT-4o mini be used commercially?", "a": "Yes. Paid OpenAI API under OpenAI's terms; tenants can bring their own key." }, { "q": "How does GPT-4o mini run on Mixpeek?", "a": "Provider API. universal__api uses it when a request names no model (params.get('model', 'gpt-4o-mini')); query_expand calls it through CentralLLMService." } ] }, "last_verified": "2026-11-05", "sources": [ "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/", "server/engine/inference/llm/routes.py", "server/shared/retrievers/stages/filter/query_expand/models.py", "server/shared/retrievers/inference/resolver.py", "server/shared/retrievers/stages/enrich/llm/models.py" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "generation", "family": "gpt-4o", "license": "OpenAI API terms of use", "commercial_use": true, "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "tagging", "captioning" ], "recommended": false, "release_date": "2024-07", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "qwen3-embedding-0-6b", "identity": { "family": "qwen3-embedding", "display_name": "Qwen3 Embedding 0.6B", "hub_id": "Qwen/Qwen3-Embedding-0.6B", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; no use restrictions.", "release_date": "2025-06", "paper_url": "https://arxiv.org/abs/2506.05176", "repo_url": "https://github.com/QwenLM/Qwen3-Embedding" }, "task": { "category": "embedding", "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "best_at": "Instruction-aware multilingual text vectors from a small model (0.6B, 1024 dims) on CPU; in the Live Nation plugin it embeds Gemini tags plus the words spoken over each shot.", "worst_at": "Speech-only queries with one relevant shot (Live Nation speech category F1@10 0.249); it needs the Instruct/Query prompt on the query side, so a plain query embed underperforms." }, "io": { "input": { "text": { "max_tokens": "32768 (model limit)" }, "prompts": "queries: 'Instruct: \nQuery: '; documents: no prefix" }, "output": { "vector": { "dims": 1024, "distance": "cosine", "pooling": "last token", "normalized": true } } }, "mixpeek_wiring": { "extractors": [ { "extractor": "ln_shots@v1", "selected_by": "fixed (customer _local plugin in customers/livenation/manifest-loop)" } ], "feature_uris": [ "mixpeek://ln_shots@v1/ln_text" ], "vector_indexes": [ "ln_shots_v1_ln_text" ], "retriever_stages": [ "feature_search" ], "composes_with": [ "ln_shots: whisper-large-v3-turbo speech + Gemini tags -> Qwen3 text vector, beside a SigLIP 2 so400m visual vector and BM25 on search_text, fused by DBSF" ] }, "run": { "status": "plugin", "status_reason": "Runs only inside the committed Live Nation _local plugin (ln_shots v1), loaded from /data/models as safetensors with remote code off; no Serve app or built-in extractor uses it.", "serve_apps": [], "runtime": "sentence_transformers.SentenceTransformer (Qwen3Model, last-token pooling, Normalize), device cpu", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1192.0, "params_millions": 596.0, "preprocessing": "Documents are the shot's tags and spoken sentences; queries get the Instruct/Query template with the Live Nation task.", "postprocessing": "Last-token pooling and L2 normalization (sentence-transformers modules), batch 16.", "serving_shape": "both", "batch_size": 16, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": null, "num_cpus": null, "num_gpus": null, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed throughput or cost measurement." }, "evidence": { "unmeasured": false, "items": [ { "dataset": "Live Nation demo, 10 clips, 263 shots, 56 queries", "metric": "macro F1@10, variant 'text' (Qwen3 text vector alone, feature_search)", "k": 10, "score": 0.414, "baseline": "the demo's native pipeline", "baseline_score": 0.412, "held_out": false, "date": "2026-11-04", "source": "customers/livenation/manifest-loop/README.md" }, { "dataset": "Live Nation held-out clips, 28 queries the driver never saw", "metric": "macro F1@10, SYSTEM number: tri_dbsf_p200 baseline = SigLIP 2 so400m visual + Qwen3 text + BM25, DBSF fusion", "k": 10, "score": 0.406, "baseline": null, "baseline_score": null, "held_out": true, "date": "2026-11-04", "source": "customers/livenation/manifest-loop/DRIVER-RUN.md" } ] }, "refs": [ "hf://Qwen/Qwen3-Embedding-0.6B@97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3" ], "formats": [ { "ref": "hf://Qwen/Qwen3-Embedding-0.6B@97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "text-search", "multilingual-search", "transcripts" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Qwen3 Embedding 0.6B: embedding model", "description": "Qwen3 Embedding 0.6B is a text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "Qwen3 Embedding 0.6B", "qwen3-embedding", "embedding", "Qwen/Qwen3-Embedding-0.6B", "text-search", "multilingual-search", "transcripts" ], "faqs": [ { "q": "What is Qwen3 Embedding 0.6B best at?", "a": "Instruction-aware multilingual text vectors from a small model (0.6B, 1024 dims) on CPU; in the Live Nation plugin it embeds Gemini tags plus the words spoken over each shot." }, { "q": "What are the limits of Qwen3 Embedding 0.6B?", "a": "Speech-only queries with one relevant shot (Live Nation speech category F1@10 0.249); it needs the Instruct/Query prompt on the query side, so a plain query embed underperforms." }, { "q": "Can Qwen3 Embedding 0.6B be used commercially?", "a": "Yes. Apache-2.0 weights; no use restrictions." }, { "q": "How does Qwen3 Embedding 0.6B run on Mixpeek?", "a": "Runs only inside the committed Live Nation _local plugin (ln_shots v1), loaded from /data/models as safetensors with remote code off; no Serve app or built-in extractor uses it." }, { "q": "What does Qwen3 Embedding 0.6B output?", "a": "A 1024-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "https://huggingface.co/api/models/Qwen/Qwen3-Embedding-0.6B", "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B/resolve/main/1_Pooling/config.json", "https://arxiv.org/abs/2506.05176", "customers/livenation/manifest-loop/README.md", "customers/livenation/manifest-loop/DRIVER-RUN.md", "customers/livenation/manifest-loop/plugin/ln_shots/v1/models.py", "customers/livenation/manifest-loop/plugin/ln_shots/v1/definition.py" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "qwen3-embedding", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "vector" ], "use_cases": [ "text-search", "multilingual-search", "transcripts" ], "recommended": false, "release_date": "2025-06", "params_millions": 596.0, "size_on_disk_mb": 1192.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": null, "memory_gb": null, "min_replicas": null, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": true, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "sam2-1-hiera-tiny", "identity": { "family": "sam2", "display_name": "SAM 2.1 Hiera Tiny (small, base-plus)", "hub_id": "facebook/sam2.1-hiera-tiny", "api_name": null, "revision": null, "variants": [ "facebook/sam2.1-hiera-small (46M)", "facebook/sam2.1-hiera-base-plus (81M)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-09-24", "paper_url": "https://arxiv.org/abs/2408.00714", "repo_url": "https://github.com/facebookresearch/sam2" }, "task": { "category": "segmentation", "modalities_in": [ "image", "video" ], "modalities_out": [ "masks", "scores", "tracks" ], "best_at": "Fast promptable masks; the size to try on CPU for image masks from detector boxes.", "worst_at": "Lower mask quality than Large on fine boundaries; video tracking is slow on CPU." }, "io": { "input": { "image": { "resolution": "1024x1024 (longest side resized)" }, "video": { "frames": "streamed frame by frame with a memory bank" }, "prompts": "points (positive/negative), boxes, or a mask; no text" }, "output": { "masks": "binary masks per prompt (3 candidates with IoU scores for ambiguous prompts)", "scores": "predicted IoU", "tracks": "object ids across video frames" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes (RT-DETR, D-FINE) -> SAM 2 box prompts -> masks -> masked crops -> SigLIP vectors", "masks per frame -> tracks -> object documents spanning a time range" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin with transformers Sam2Model (safetensors, no remote code), prompted by boxes from a permissive detector.", "serve_apps": [], "runtime": "transformers Sam2Model + Sam2Processor (Sam2VideoModel for video)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 156.0, "params_millions": 39.0, "preprocessing": "Resize longest side to 1024, pad, normalize ImageNet mean/std; prompts rescaled to the same frame.", "postprocessing": "Upsample low-res mask logits, threshold at 0, pick the best mask by predicted IoU.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/sam2.1-hiera-tiny@de431c4043854a71d8101e17995dfe596bf101a5" ], "formats": [ { "ref": "hf://facebook/sam2.1-hiera-tiny@de431c4043854a71d8101e17995dfe596bf101a5", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/sam2.1-hiera-tiny@de431c4043854a71d8101e17995dfe596bf101a5", "format": "pytorch_pt", "file": "sam2.1_hiera_tiny.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection", "tracking" ], "recommended": { "value": true, "for": [ "segmentation" ], "reason": "Apache-2.0 promptable masks small enough to try on CPU. Not yet evaluated by Mixpeek." }, "demos": [], "seo": { "title": "SAM 2.1 Hiera Tiny (small, base-plus): segmentation model", "description": "SAM 2.1 Hiera Tiny (small, base-plus) is a image and video segmentation model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SAM 2.1 Hiera Tiny (small, base-plus)", "sam2", "segmentation", "facebook/sam2.1-hiera-tiny", "object-detection", "tracking" ], "faqs": [ { "q": "What is SAM 2.1 Hiera Tiny (small, base-plus) best at?", "a": "Fast promptable masks; the size to try on CPU for image masks from detector boxes." }, { "q": "What are the limits of SAM 2.1 Hiera Tiny (small, base-plus)?", "a": "Lower mask quality than Large on fine boundaries; video tracking is slow on CPU." }, { "q": "Can SAM 2.1 Hiera Tiny (small, base-plus) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SAM 2.1 Hiera Tiny (small, base-plus) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin with transformers Sam2Model (safetensors, no remote code), prompted by boxes from a permissive detector." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/sam2.1-hiera-tiny", "https://arxiv.org/abs/2408.00714", "https://github.com/facebookresearch/sam2", "https://huggingface.co/docs/transformers/model_doc/sam2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "segmentation", "family": "sam2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "video" ], "modalities_out": [ "masks", "scores", "tracks" ], "use_cases": [ "segmentation", "object-detection", "tracking" ], "recommended": true, "release_date": "2024-09-24", "params_millions": 39.0, "size_on_disk_mb": 156.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "typesafe-jev", "identity": { "family": "typesafe-system-one", "display_name": "TypeSafe Jev", "hub_id": null, "api_name": "jev-latest", "revision": null, "variants": [ "jev-preview", "jev-1.13.0" ], "license": "TypeSafe AI API terms", "commercial_use": true, "commercial_use_note": "Paid TypeSafe API (input tokens only, 0.042 USD per million per client.py); used under TypeSafe's terms.", "release_date": "2026-09", "paper_url": null, "repo_url": null }, "task": { "category": "classification", "modalities_in": [ "text", "structured" ], "modalities_out": [ "labels", "scores", "structured" ], "best_at": "Typed decisions with calibrated probabilities over a JSON state: yes/no relevance for rerank, label probabilities for classify, closed-set schema fields for text_extractor; the platform's DECISION_MODEL_DEFAULT.", "worst_at": "Free-text generation and open-ended fields (closed-set answers only); no image, video or audio input; 32k-token state limit (MAX_STATE_CHARS 100,000)." }, "io": { "input": { "state": "JSON object or text, up to about 100,000 characters", "questions": "typed questions (choice, yes/no, schema fields)" }, "output": { "structured": "answers with per-option probabilities; rerank -> {scores}, classify -> {labels: [{label, confidence}]}" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "text_extractor@v1", "selected_by": "llm_provider=typesafe or llm_model starting with jev (response_shape set)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "rerank", "llm_enrich", "classify" ], "composes_with": [ "feature_search -> rerank inference_name=typesafe__jev (or mixpeek__decision, default jev-latest)" ] }, "run": { "status": "api", "status_reason": "TypeSafe System One HTTP API. typesafe__jev registers the name; on the planes mixpeek__api_inference hosts it as an API target.", "serve_apps": [ "typesafe__jev" ], "runtime": "httpx client to https://api.typesafe.ai/v1/systemone (clients.providers.typesafe.client)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "One question per (query, document) pair for rerank; labels become a choice question for classify; schema fields become typed questions.", "postprocessing": "Yes-probability as the rerank score; label probabilities sorted highest first.", "serving_shape": "realtime", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Price per client.py is 0.042 USD per million input tokens, output free; no committed throughput or latency measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://typesafe/jev-latest" ], "formats": [ { "ref": "api://typesafe/jev-latest", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "classification", "reranking", "moderation" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "TypeSafe Jev: classification model", "description": "TypeSafe Jev is a text and structured classification model (TypeSafe AI API terms). Called on Mixpeek through the provider API.", "keywords": [ "TypeSafe Jev", "typesafe-system-one", "classification", "jev-latest", "reranking", "moderation" ], "faqs": [ { "q": "What is TypeSafe Jev best at?", "a": "Typed decisions with calibrated probabilities over a JSON state: yes/no relevance for rerank, label probabilities for classify, closed-set schema fields for text_extractor; the platform's DECISION_MODEL_DEFAULT." }, { "q": "What are the limits of TypeSafe Jev?", "a": "Free-text generation and open-ended fields (closed-set answers only); no image, video or audio input; 32k-token state limit (MAX_STATE_CHARS 100,000)." }, { "q": "Can TypeSafe Jev be used commercially?", "a": "Yes. Paid TypeSafe API (input tokens only, 0.042 USD per million per client.py); used under TypeSafe's terms." }, { "q": "How does TypeSafe Jev run on Mixpeek?", "a": "TypeSafe System One HTTP API. typesafe__jev registers the name; on the planes mixpeek__api_inference hosts it as an API target." } ] }, "last_verified": "2026-11-05", "sources": [ "https://docs.typesafe.ai/api", "https://www.infoq.com/news/2026/10/typesafe-ai-jev-released", "server/clients/providers/typesafe/client.py", "server/engine/inference/typesafe/jev/routes.py", "server/engine/inference/typesafe/jev/services.py", "server/shared/retrievers/stages/sort/rerank/models.py", "server/engine/extractors/text_extractor/v1/pipeline.py", "server/configs/settings.py" ], "registry": { "typesafe__jev": { "deployment": "typesafe__jev", "service_id": "typesafe/jev", "routes": "engine/inference/typesafe/jev/routes.py", "route_prefix": "/typesafe/jev", "enabled": true, "description": "TypeSafe Jev evaluation model. Answers typed questions about text with calibrated probabilities (choice, score, yes/no). As a reranker it scores {pairs} and returns {scores}; it also classifies {text} against parameters.labels, answers a closed-set {schema}, or passes raw {state, questions} through.", "capabilities": { "category": "reranking", "modality_input": [ "text" ], "modality_output": "score", "supports_batch": true, "reranking_mode": "text-to-text" }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 4, "idle_timeout_s": 600, "target_ongoing_requests": 10, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "classification", "family": "typesafe-system-one", "license": "TypeSafe AI API terms", "commercial_use": true, "modalities_in": [ "text", "structured" ], "modalities_out": [ "labels", "scores", "structured" ], "use_cases": [ "classification", "reranking", "moderation" ], "recommended": false, "release_date": "2026-09", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "realtime", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "whisper-1", "identity": { "family": "whisper", "display_name": "OpenAI Whisper API (whisper-1)", "hub_id": null, "api_name": "whisper-1", "revision": null, "variants": [], "license": "OpenAI API terms of use", "commercial_use": true, "commercial_use_note": "Paid OpenAI API under OpenAI's terms.", "release_date": "2023-03", "paper_url": "https://arxiv.org/abs/2212.04356", "repo_url": null }, "task": { "category": "transcription", "modalities_in": [ "audio", "video" ], "modalities_out": [ "text" ], "best_at": "Speech-to-text for video and audio with no GPU in the cluster; the production default for multimodal_extractor transcription.", "worst_at": "Provider rate limits (a Primis batch lost its transcripts to a 429), per-minute cost, and audio sent off-box." }, "io": { "input": { "audio": "audio chunks from AudioParallelChunking", "language": "optional forced language" }, "output": { "text": "transcript, or verbose_json segments when segment timestamps are wanted" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "multimodal_extractor@v1", "selected_by": "run_transcription=true and use_whisper_api (HARDWARE_WHISPER_USE_API=true, or forced when the cluster has no GPU)" }, { "extractor": "multimodal_extractor@v2", "selected_by": "run_transcription=true and use_whisper_api (HARDWARE_WHISPER_USE_API=true, or forced when the cluster has no GPU)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "video -> audio chunks -> whisper-1 transcript -> E5 transcription vector" ] }, "run": { "status": "api", "status_reason": "OpenAI audio transcriptions API, called from WhisperTranscriptionBatch; universal__api also maps whisper-1 for POST /v1/inference.", "serve_apps": [ "universal__api" ], "runtime": "OpenAI audio.transcriptions.create (engine.inference.openai.whisper_api.services)", "weights_format": "api", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "ffmpeg audio extraction and chunking per segment.", "postprocessing": "Text, or verbose_json segments mapped to timestamps.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "memory_gb": null, "num_cpus": 0.1, "num_gpus": 0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "Billed per audio minute by OpenAI; no committed measurement." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "api://openai/whisper-1" ], "formats": [ { "ref": "api://openai/whisper-1", "format": "api", "file": null, "loader": "provider SDK, called by model name", "supported_today": true } ], "use_cases": [ "transcripts" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenAI Whisper API (whisper-1): speech-to-text model", "description": "OpenAI Whisper API (whisper-1) is a audio and video speech-to-text model (OpenAI API terms of use). Called on Mixpeek through the provider API.", "keywords": [ "OpenAI Whisper API (whisper-1)", "whisper", "speech-to-text", "whisper-1", "transcripts" ], "faqs": [ { "q": "What is OpenAI Whisper API (whisper-1) best at?", "a": "Speech-to-text for video and audio with no GPU in the cluster; the production default for multimodal_extractor transcription." }, { "q": "What are the limits of OpenAI Whisper API (whisper-1)?", "a": "Provider rate limits (a Primis batch lost its transcripts to a 429), per-minute cost, and audio sent off-box." }, { "q": "Can OpenAI Whisper API (whisper-1) be used commercially?", "a": "Yes. Paid OpenAI API under OpenAI's terms." }, { "q": "How does OpenAI Whisper API (whisper-1) run on Mixpeek?", "a": "OpenAI audio transcriptions API, called from WhisperTranscriptionBatch; universal__api also maps whisper-1 for POST /v1/inference." } ] }, "last_verified": "2026-11-05", "sources": [ "https://platform.openai.com/docs/guides/speech-to-text", "server/engine/inference/openai/whisper_api/services.py", "server/engine/extractors/multimodal_extractor/v2/pipeline.py", "server/infra/hardware/config.py", "server/engine/inference/llm/routes.py", "customers/primis/BENCHMARK_RESULTS.md" ], "registry": { "universal__api": { "deployment": "universal__api", "service_id": "universal/api", "routes": "engine/inference/llm/routes.py", "route_prefix": "/universal/api", "enabled": true, "description": "Universal inference endpoint routing to multiple providers and modalities. Supports: OpenAI (GPT, embeddings, Whisper), Google (Gemini, Vertex embeddings), Anthropic (Claude). Routes based on model parameter.", "capabilities": { "category": "generation", "modality_input": [ "text" ], "modality_output": "text", "supports_batch": true, "reranking_mode": null }, "vector_config": null, "serve": { "min_replicas": 0, "max_replicas": 6, "idle_timeout_s": 600, "target_ongoing_requests": 25, "num_cpus": 0.1, "num_gpus": 0, "memory_gb": null, "pinnable": false, "placement": "worker", "enabled": true }, "interactive_twin": false, "interactive_serve": null, "schemas": { "input": null, "output": null, "config": null } } }, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "api", "category": "transcription", "family": "whisper", "license": "OpenAI API terms of use", "commercial_use": true, "modalities_in": [ "audio", "video" ], "modalities_out": [ "text" ], "use_cases": [ "transcripts" ], "recommended": false, "release_date": "2023-03", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "api", "formats": [ "api" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": null, "min_replicas": 0.0, "serving_shape": "both", "served": true, "ref_schemes": [ "api" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 2.0 }, "rank_score": 2.0 }, { "id": "clip-vit-b-16-laion2b-s34b-b88k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-B/16 LAION-2B", "hub_id": "laion/CLIP-ViT-B-16-laion2B-s34B-b88K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2023-01", "paper_url": "https://arxiv.org/abs/2210.08402", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Better than B/32 on fine detail at modest cost.", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 599.0, "params_millions": 150.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-B-16-laion2B-s34B-b88K@7288da5a0d6f0b51c4a2b27c624837a9236d0112" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-B-16-laion2B-s34B-b88K@7288da5a0d6f0b51c4a2b27c624837a9236d0112", "format": "safetensors", "file": "open_clip_model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://laion/CLIP-ViT-B-16-laion2B-s34B-b88K@7288da5a0d6f0b51c4a2b27c624837a9236d0112", "format": "pytorch_bin", "file": "open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-B/16 LAION-2B: embedding model", "description": "OpenCLIP ViT-B/16 LAION-2B is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "OpenCLIP ViT-B/16 LAION-2B", "openclip", "embedding", "laion/CLIP-ViT-B-16-laion2B-s34B-b88K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-B/16 LAION-2B best at?", "a": "Better than B/32 on fine detail at modest cost." }, { "q": "What are the limits of OpenCLIP ViT-B/16 LAION-2B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-B/16 LAION-2B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-B/16 LAION-2B run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false." }, { "q": "What does OpenCLIP ViT-B/16 LAION-2B output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-B-16-laion2B-s34B-b88K", "https://arxiv.org/abs/2210.08402", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2023-01", "params_millions": 150.0, "size_on_disk_mb": 599.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "clip-vit-b-32-laion2b-s34b-b79k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-B/32 LAION-2B", "hub_id": "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2022-09", "paper_url": "https://arxiv.org/abs/2210.08402", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Cheapest strong CLIP for CPU bulk frame embedding.", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 605.0, "params_millions": 151.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-B-32-laion2B-s34B-b79K@1a25a446712ba5ee05982a381eed697ef9b435cf" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-B-32-laion2B-s34B-b79K@1a25a446712ba5ee05982a381eed697ef9b435cf", "format": "safetensors", "file": "2 files, e.g. model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://laion/CLIP-ViT-B-32-laion2B-s34B-b79K@1a25a446712ba5ee05982a381eed697ef9b435cf", "format": "pytorch_bin", "file": "2 files, e.g. open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-B/32 LAION-2B: embedding model", "description": "OpenCLIP ViT-B/32 LAION-2B is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "OpenCLIP ViT-B/32 LAION-2B", "openclip", "embedding", "laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-B/32 LAION-2B best at?", "a": "Cheapest strong CLIP for CPU bulk frame embedding." }, { "q": "What are the limits of OpenCLIP ViT-B/32 LAION-2B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-B/32 LAION-2B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-B/32 LAION-2B run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false." }, { "q": "What does OpenCLIP ViT-B/32 LAION-2B output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-B-32-laion2B-s34B-b79K", "https://arxiv.org/abs/2210.08402", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2022-09", "params_millions": 151.0, "size_on_disk_mb": 605.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "clip-vit-bigg-14-laion2b-39b-b160k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-bigG/14 LAION-2B", "hub_id": "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2023-01", "paper_url": "https://arxiv.org/abs/2212.07143", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Largest LAION CLIP; best zero-shot accuracy in the LAION family.", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 1280, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false. GPU recommended at this size.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 10158.0, "params_millions": 2540.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 16.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-bigG-14-laion2B-39B-b160k@743c27bd53dfe508a0ade0f50698f99b39d03bec" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-bigG-14-laion2B-39B-b160k@743c27bd53dfe508a0ade0f50698f99b39d03bec", "format": "safetensors", "file": "3 files, e.g. open_clip_model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://laion/CLIP-ViT-bigG-14-laion2B-39B-b160k@743c27bd53dfe508a0ade0f50698f99b39d03bec", "format": "pytorch_bin", "file": "3 files, e.g. open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-bigG/14 LAION-2B: embedding model", "description": "OpenCLIP ViT-bigG/14 LAION-2B is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "OpenCLIP ViT-bigG/14 LAION-2B", "openclip", "embedding", "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-bigG/14 LAION-2B best at?", "a": "Largest LAION CLIP; best zero-shot accuracy in the LAION family." }, { "q": "What are the limits of OpenCLIP ViT-bigG/14 LAION-2B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-bigG/14 LAION-2B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-bigG/14 LAION-2B run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false. GPU recommended at this size." }, { "q": "What does OpenCLIP ViT-bigG/14 LAION-2B output?", "a": "A 1280-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "https://arxiv.org/abs/2212.07143", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2023-01", "params_millions": 2540.0, "size_on_disk_mb": 10158.0, "vector_dims": 1280, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 16.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "clip-vit-h-14-laion2b-s32b-b79k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-H/14 LAION-2B", "hub_id": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2022-09", "paper_url": "https://arxiv.org/abs/2210.08402", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "High-accuracy CLIP for offline catalogs (the text encoder family used by Stable Diffusion 2).", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 1024, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false. GPU recommended at this size.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 3945.0, "params_millions": 986.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-H-14-laion2B-s32B-b79K@1c2b8495b28150b8a4922ee1c8edee224c284c0c" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-H-14-laion2B-s32B-b79K@1c2b8495b28150b8a4922ee1c8edee224c284c0c", "format": "safetensors", "file": "2 files, e.g. model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://laion/CLIP-ViT-H-14-laion2B-s32B-b79K@1c2b8495b28150b8a4922ee1c8edee224c284c0c", "format": "pytorch_bin", "file": "2 files, e.g. open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-H/14 LAION-2B: embedding model", "description": "OpenCLIP ViT-H/14 LAION-2B is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "OpenCLIP ViT-H/14 LAION-2B", "openclip", "embedding", "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-H/14 LAION-2B best at?", "a": "High-accuracy CLIP for offline catalogs (the text encoder family used by Stable Diffusion 2)." }, { "q": "What are the limits of OpenCLIP ViT-H/14 LAION-2B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-H/14 LAION-2B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-H/14 LAION-2B run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading open_clip_model.safetensors via open_clip hf-hub with trust_remote_code false. GPU recommended at this size." }, { "q": "What does OpenCLIP ViT-H/14 LAION-2B output?", "a": "A 1024-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "https://arxiv.org/abs/2210.08402", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2022-09", "params_millions": 986.0, "size_on_disk_mb": 3945.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "clip-vit-large-patch14", "identity": { "family": "clip", "display_name": "OpenAI CLIP ViT-L/14", "hub_id": "openai/clip-vit-large-patch14", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call.", "release_date": "2021-01", "paper_url": "https://arxiv.org/abs/2103.00020", "repo_url": "https://github.com/openai/CLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "General zero-shot labels and text-to-image search; the reference baseline most papers report against.", "worst_at": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; BPE tokenizer, 77 tokens max" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers CLIPModel + CLIPProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1711.0, "params_millions": 428.0, "preprocessing": "Resize shortest side and center-crop to 224x224, normalize with CLIP mean/std; text to 77 tokens.", "postprocessing": "L2-normalize image and text embeddings; logits = 100 * cosine for zero-shot.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 3.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://openai/clip-vit-large-patch14@32bd64288804d66eefd0ccbe215aa642df71cc41" ], "formats": [ { "ref": "hf://openai/clip-vit-large-patch14@32bd64288804d66eefd0ccbe215aa642df71cc41", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://openai/clip-vit-large-patch14@32bd64288804d66eefd0ccbe215aa642df71cc41", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://openai/clip-vit-large-patch14@32bd64288804d66eefd0ccbe215aa642df71cc41", "format": "jax", "file": "flax_model.msgpack", "loader": "JAX/Flax; not in Mixpeek's runtime", "supported_today": false }, { "ref": "hf://openai/clip-vit-large-patch14@32bd64288804d66eefd0ccbe215aa642df71cc41", "format": "tensorflow", "file": "tf_model.h5", "loader": "TensorFlow; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenAI CLIP ViT-L/14: embedding model", "description": "OpenAI CLIP ViT-L/14 is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "OpenAI CLIP ViT-L/14", "clip", "embedding", "openai/clip-vit-large-patch14", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenAI CLIP ViT-L/14 best at?", "a": "General zero-shot labels and text-to-image search; the reference baseline most papers report against." }, { "q": "What are the limits of OpenAI CLIP ViT-L/14?", "a": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, { "q": "Can OpenAI CLIP ViT-L/14 be used commercially?", "a": "Yes. MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call." }, { "q": "How does OpenAI CLIP ViT-L/14 run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does OpenAI CLIP ViT-L/14 output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/openai/clip-vit-large-patch14", "https://arxiv.org/abs/2103.00020", "https://github.com/openai/CLIP/blob/main/LICENSE", "https://github.com/openai/CLIP/blob/main/model-card.md" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "clip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2021-01", "params_millions": 428.0, "size_on_disk_mb": 1711.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "jax", "pytorch_bin", "safetensors", "tensorflow" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 3.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "fashion-clip", "identity": { "family": "fashion-clip", "display_name": "FashionCLIP (2.0)", "hub_id": "patrickjohncyh/fashion-clip", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT license on the repo and weights.", "release_date": "2023-02", "paper_url": "https://www.nature.com/articles/s41598-022-23052-9", "repo_url": "https://github.com/patrickjohncyh/fashion-clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Product attributes in fashion catalogs: garment type, color, pattern, material, style words.", "worst_at": "General scenes, people in context, and anything outside apparel; English only." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "product-style captions, e.g. 'a red midi dress with floral print'" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "product images -> FashionCLIP vectors -> feature_search with attribute filters on price and brand", "detector person boxes -> garment crops -> FashionCLIP vectors" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers CLIPModel + CLIPProcessor (ViT-B/32 backbone, LAION base for 2.0)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 605.0, "params_millions": 151.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std; text to 77 tokens.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://patrickjohncyh/fashion-clip@7e3ba62ce16b379a1ab479346b66f192e76f51b7" ], "formats": [ { "ref": "hf://patrickjohncyh/fashion-clip@7e3ba62ce16b379a1ab479346b66f192e76f51b7", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://patrickjohncyh/fashion-clip@7e3ba62ce16b379a1ab479346b66f192e76f51b7", "format": "onnx", "file": "onnx/model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://patrickjohncyh/fashion-clip@7e3ba62ce16b379a1ab479346b66f192e76f51b7", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "FashionCLIP (2.0): embedding model", "description": "FashionCLIP (2.0) is a image and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "FashionCLIP (2.0)", "fashion-clip", "embedding", "patrickjohncyh/fashion-clip", "fashion", "product-search", "visual-search" ], "faqs": [ { "q": "What is FashionCLIP (2.0) best at?", "a": "Product attributes in fashion catalogs: garment type, color, pattern, material, style words." }, { "q": "What are the limits of FashionCLIP (2.0)?", "a": "General scenes, people in context, and anything outside apparel; English only." }, { "q": "Can FashionCLIP (2.0) be used commercially?", "a": "Yes. MIT license on the repo and weights." }, { "q": "How does FashionCLIP (2.0) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does FashionCLIP (2.0) output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/patrickjohncyh/fashion-clip", "https://www.nature.com/articles/s41598-022-23052-9", "https://github.com/patrickjohncyh/fashion-clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "fashion-clip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": false, "release_date": "2023-02", "params_millions": 151.0, "size_on_disk_mb": 605.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "marqo-fashionclip", "identity": { "family": "marqo-fashion", "display_name": "Marqo-FashionCLIP", "hub_id": "Marqo/marqo-fashionCLIP", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-08", "paper_url": null, "repo_url": "https://github.com/marqo-ai/marqo-FashionCLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Fashion product search from text or image (ViT-B/16 CLIP fine-tuned with generalized contrastive learning on product titles, categories and attributes); Marqo reports gains over FashionCLIP 2.0 on its fashion benchmarks.", "worst_at": "Non-fashion content; the transformers path needs remote code (use open_clip instead)." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "product titles and attributes" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "product images -> vectors -> feature_search; attributes as filters" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. Load through open_clip ('hf-hub:Marqo/marqo-fashionCLIP', open_clip_model.safetensors); the repo's transformers path is custom code and is not used.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 599.0, "params_millions": 150.0, "preprocessing": "open_clip preprocess for the checkpoint (resize, center-crop 224, normalize).", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Marqo/marqo-fashionCLIP@44f4c655124ed71e90cbf528d82f508d69d8a81b" ], "formats": [ { "ref": "hf://Marqo/marqo-fashionCLIP@44f4c655124ed71e90cbf528d82f508d69d8a81b", "format": "safetensors", "file": "2 files, e.g. model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://Marqo/marqo-fashionCLIP@44f4c655124ed71e90cbf528d82f508d69d8a81b", "format": "onnx", "file": "16 files, e.g. onnx/text_model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://Marqo/marqo-fashionCLIP@44f4c655124ed71e90cbf528d82f508d69d8a81b", "format": "pytorch_bin", "file": "open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Marqo-FashionCLIP: embedding model", "description": "Marqo-FashionCLIP is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "Marqo-FashionCLIP", "marqo-fashion", "embedding", "Marqo/marqo-fashionCLIP", "fashion", "product-search", "visual-search" ], "faqs": [ { "q": "What is Marqo-FashionCLIP best at?", "a": "Fashion product search from text or image (ViT-B/16 CLIP fine-tuned with generalized contrastive learning on product titles, categories and attributes); Marqo reports gains over FashionCLIP 2.0 on its fashion benchmarks." }, { "q": "What are the limits of Marqo-FashionCLIP?", "a": "Non-fashion content; the transformers path needs remote code (use open_clip instead)." }, { "q": "Can Marqo-FashionCLIP be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does Marqo-FashionCLIP run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. Load through open_clip ('hf-hub:Marqo/marqo-fashionCLIP', open_clip_model.safetensors); the repo's transformers path is custom code and is not used." }, { "q": "What does Marqo-FashionCLIP output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Marqo/marqo-fashionCLIP", "https://github.com/marqo-ai/marqo-FashionCLIP", "https://www.marqo.ai/blog/search-model-for-fashion" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "marqo-fashion", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": false, "release_date": "2024-08", "params_millions": 150.0, "size_on_disk_mb": 599.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "marqo-fashionsiglip", "identity": { "family": "marqo-fashion", "display_name": "Marqo-FashionSigLIP", "hub_id": "Marqo/marqo-fashionSigLIP", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-08", "paper_url": null, "repo_url": "https://github.com/marqo-ai/marqo-FashionCLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Fashion product search from text or image (ViT-B/16 SigLIP fine-tuned with generalized contrastive learning on product titles, categories and attributes); Marqo reports gains over FashionCLIP 2.0 on its fashion benchmarks.", "worst_at": "Non-fashion content; the transformers path needs remote code (use open_clip instead)." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "product titles and attributes" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "product images -> vectors -> feature_search; attributes as filters" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. Load through open_clip ('hf-hub:Marqo/marqo-fashionSigLIP', open_clip_model.safetensors); the repo's transformers path is custom code and is not used.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 813.0, "params_millions": 203.0, "preprocessing": "open_clip preprocess for the checkpoint (resize, center-crop 224, normalize).", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Marqo/marqo-fashionSigLIP@c56244cc94f92419e8369fa71efdaf403b124ce8" ], "formats": [ { "ref": "hf://Marqo/marqo-fashionSigLIP@c56244cc94f92419e8369fa71efdaf403b124ce8", "format": "safetensors", "file": "2 files, e.g. model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://Marqo/marqo-fashionSigLIP@c56244cc94f92419e8369fa71efdaf403b124ce8", "format": "onnx", "file": "16 files, e.g. onnx/text_model.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true }, { "ref": "hf://Marqo/marqo-fashionSigLIP@c56244cc94f92419e8369fa71efdaf403b124ce8", "format": "pytorch_bin", "file": "open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Marqo-FashionSigLIP: embedding model", "description": "Marqo-FashionSigLIP is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "Marqo-FashionSigLIP", "marqo-fashion", "embedding", "Marqo/marqo-fashionSigLIP", "fashion", "product-search", "visual-search" ], "faqs": [ { "q": "What is Marqo-FashionSigLIP best at?", "a": "Fashion product search from text or image (ViT-B/16 SigLIP fine-tuned with generalized contrastive learning on product titles, categories and attributes); Marqo reports gains over FashionCLIP 2.0 on its fashion benchmarks." }, { "q": "What are the limits of Marqo-FashionSigLIP?", "a": "Non-fashion content; the transformers path needs remote code (use open_clip instead)." }, { "q": "Can Marqo-FashionSigLIP be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does Marqo-FashionSigLIP run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. Load through open_clip ('hf-hub:Marqo/marqo-fashionSigLIP', open_clip_model.safetensors); the repo's transformers path is custom code and is not used." }, { "q": "What does Marqo-FashionSigLIP output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Marqo/marqo-fashionSigLIP", "https://github.com/marqo-ai/marqo-FashionCLIP", "https://www.marqo.ai/blog/search-model-for-fashion" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "marqo-fashion", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "fashion", "product-search", "visual-search" ], "recommended": false, "release_date": "2024-08", "params_millions": 203.0, "size_on_disk_mb": 813.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "onnx", "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "rf-detr-base", "identity": { "family": "rf-detr", "display_name": "RF-DETR Base", "hub_id": "Roboflow/rf-detr-base", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-03", "paper_url": "https://arxiv.org/abs/2511.09554", "repo_url": "https://github.com/roboflow/rf-detr" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "DINOv2-backbone real-time detector that fine-tunes well on small custom datasets (RF100-VL); Nano to Large are Apache-2.0.", "worst_at": "XL and 2XL sizes are under Roboflow's PML 1.0, not Apache; check the transformers version supports the architecture." }, "io": { "input": { "image": { "resolution": "640x640", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels (post_process_object_detection)", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "plugin", "status_reason": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops.", "serve_apps": [], "runtime": "transformers (RF-DETR, library_name transformers on the hub) or the rfdetr package", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 129.0, "params_millions": 32.0, "preprocessing": "Resize to 640x640, rescale to 0..1 (image processor).", "postprocessing": "Sigmoid class scores, top-k queries, threshold; no NMS (one-to-one matching).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Roboflow/rf-detr-base@7b95b089788e6c7db56d5ea9b0a07ca08ea6ac0a" ], "formats": [ { "ref": "hf://Roboflow/rf-detr-base@7b95b089788e6c7db56d5ea9b0a07ca08ea6ac0a", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "RF-DETR Base: object detection model", "description": "RF-DETR Base is a image object detection model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "RF-DETR Base", "rf-detr", "object detection", "Roboflow/rf-detr-base", "object-detection" ], "faqs": [ { "q": "What is RF-DETR Base best at?", "a": "DINOv2-backbone real-time detector that fine-tunes well on small custom datasets (RF100-VL); Nano to Large are Apache-2.0." }, { "q": "What are the limits of RF-DETR Base?", "a": "XL and 2XL sizes are under Roboflow's PML 1.0, not Apache; check the transformers version supports the architecture." }, { "q": "Can RF-DETR Base be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does RF-DETR Base run on Mixpeek?", "a": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Roboflow/rf-detr-base", "https://arxiv.org/abs/2511.09554", "https://github.com/roboflow/rf-detr", "https://arxiv.org/abs/2511.09554", "https://rfdetr.roboflow.com/latest/", "https://playground.roboflow.com/models/roboflow" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "detection", "family": "rf-detr", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2025-03", "params_millions": 32.0, "size_on_disk_mb": 129.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "rtdetr-r50vd", "identity": { "family": "rt-detr", "display_name": "RT-DETR R50vd", "hub_id": "PekingU/rtdetr_r50vd", "api_name": null, "revision": null, "variants": [ "PekingU/rtdetr_r18vd (20M)", "PekingU/rtdetr_r101vd (77M)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2023-04", "paper_url": "https://arxiv.org/abs/2304.08069", "repo_url": "https://github.com/lyuwenyu/RT-DETR" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Real-time end-to-end detection with no NMS step; Apache-2.0, a drop-in for YOLO detect.", "worst_at": "Closed COCO label set; heavier than YOLO n on CPU." }, "io": { "input": { "image": { "resolution": "640x640", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels (post_process_object_detection)", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "plugin", "status_reason": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops.", "serve_apps": [], "runtime": "transformers RTDetrForObjectDetection + RTDetrImageProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 172.0, "params_millions": 43.0, "preprocessing": "Resize to 640x640, rescale to 0..1 (image processor).", "postprocessing": "Sigmoid class scores, top-k queries, threshold; no NMS (one-to-one matching).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://PekingU/rtdetr_r50vd@df939e661d8c52e80608d1ec566561aabd25a4e7" ], "formats": [ { "ref": "hf://PekingU/rtdetr_r50vd@df939e661d8c52e80608d1ec566561aabd25a4e7", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "RT-DETR R50vd: object detection model", "description": "RT-DETR R50vd is a image object detection model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "RT-DETR R50vd", "rt-detr", "object detection", "PekingU/rtdetr_r50vd", "object-detection" ], "faqs": [ { "q": "What is RT-DETR R50vd best at?", "a": "Real-time end-to-end detection with no NMS step; Apache-2.0, a drop-in for YOLO detect." }, { "q": "What are the limits of RT-DETR R50vd?", "a": "Closed COCO label set; heavier than YOLO n on CPU." }, { "q": "Can RT-DETR R50vd be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does RT-DETR R50vd run on Mixpeek?", "a": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/PekingU/rtdetr_r50vd", "https://arxiv.org/abs/2304.08069", "https://github.com/lyuwenyu/RT-DETR" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "detection", "family": "rt-detr", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2023-04", "params_millions": 43.0, "size_on_disk_mb": 172.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "rtdetr-v2-r50vd", "identity": { "family": "rt-detr", "display_name": "RT-DETRv2 R50vd", "hub_id": "PekingU/rtdetr_v2_r50vd", "api_name": null, "revision": null, "variants": [ "PekingU/rtdetr_v2_r18vd" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-07", "paper_url": "https://arxiv.org/abs/2407.17140", "repo_url": "https://github.com/lyuwenyu/RT-DETR" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "RT-DETR with a better training recipe (bag of freebies); same cost, higher COCO AP.", "worst_at": "Closed COCO label set." }, "io": { "input": { "image": { "resolution": "640x640", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels (post_process_object_detection)", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "plugin", "status_reason": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops.", "serve_apps": [], "runtime": "transformers RTDetrV2ForObjectDetection + RTDetrImageProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 172.0, "params_millions": 43.0, "preprocessing": "Resize to 640x640, rescale to 0..1 (image processor).", "postprocessing": "Sigmoid class scores, top-k queries, threshold; no NMS (one-to-one matching).", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://PekingU/rtdetr_v2_r50vd@282494075698cab9faa1096ae26856890030c817" ], "formats": [ { "ref": "hf://PekingU/rtdetr_v2_r50vd@282494075698cab9faa1096ae26856890030c817", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "RT-DETRv2 R50vd: object detection model", "description": "RT-DETRv2 R50vd is a image object detection model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "RT-DETRv2 R50vd", "rt-detr", "object detection", "PekingU/rtdetr_v2_r50vd", "object-detection" ], "faqs": [ { "q": "What is RT-DETRv2 R50vd best at?", "a": "RT-DETR with a better training recipe (bag of freebies); same cost, higher COCO AP." }, { "q": "What are the limits of RT-DETRv2 R50vd?", "a": "Closed COCO label set." }, { "q": "Can RT-DETRv2 R50vd be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does RT-DETRv2 R50vd run on Mixpeek?", "a": "Not served. Permissive alternative to Ultralytics YOLO. Would run as a custom extractor plugin loading safetensors with transformers (trust_remote_code false), emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/PekingU/rtdetr_v2_r50vd", "https://arxiv.org/abs/2407.17140", "https://github.com/lyuwenyu/RT-DETR" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "detection", "family": "rt-detr", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2024-07", "params_millions": 43.0, "size_on_disk_mb": 172.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "sam2-1-hiera-large", "identity": { "family": "sam2", "display_name": "SAM 2.1 Hiera Large", "hub_id": "facebook/sam2.1-hiera-large", "api_name": null, "revision": null, "variants": [ "facebook/sam2-hiera-large (SAM 2.0)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-09-24", "paper_url": "https://arxiv.org/abs/2408.00714", "repo_url": "https://github.com/facebookresearch/sam2" }, "task": { "category": "segmentation", "modalities_in": [ "image", "video" ], "modalities_out": [ "masks", "scores", "tracks" ], "best_at": "Highest-quality promptable masks in images and video, with memory-based tracking of an object across frames.", "worst_at": "GPU in practice for video; needs points or boxes (no text prompts)." }, "io": { "input": { "image": { "resolution": "1024x1024 (longest side resized)" }, "video": { "frames": "streamed frame by frame with a memory bank" }, "prompts": "points (positive/negative), boxes, or a mask; no text" }, "output": { "masks": "binary masks per prompt (3 candidates with IoU scores for ambiguous prompts)", "scores": "predicted IoU", "tracks": "object ids across video frames" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes (RT-DETR, D-FINE) -> SAM 2 box prompts -> masks -> masked crops -> SigLIP vectors", "masks per frame -> tracks -> object documents spanning a time range" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin with transformers Sam2Model (safetensors, no remote code), prompted by boxes from a permissive detector.", "serve_apps": [], "runtime": "transformers Sam2Model + Sam2Processor (Sam2VideoModel for video)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 898.0, "params_millions": 224.0, "preprocessing": "Resize longest side to 1024, pad, normalize ImageNet mean/std; prompts rescaled to the same frame.", "postprocessing": "Upsample low-res mask logits, threshold at 0, pick the best mask by predicted IoU.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/sam2.1-hiera-large@665f8e2ad61cf5f53d65644ff27c8ee525124610" ], "formats": [ { "ref": "hf://facebook/sam2.1-hiera-large@665f8e2ad61cf5f53d65644ff27c8ee525124610", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/sam2.1-hiera-large@665f8e2ad61cf5f53d65644ff27c8ee525124610", "format": "pytorch_pt", "file": "sam2.1_hiera_large.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection", "tracking" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SAM 2.1 Hiera Large: segmentation model", "description": "SAM 2.1 Hiera Large is a image and video segmentation model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SAM 2.1 Hiera Large", "sam2", "segmentation", "facebook/sam2.1-hiera-large", "object-detection", "tracking" ], "faqs": [ { "q": "What is SAM 2.1 Hiera Large best at?", "a": "Highest-quality promptable masks in images and video, with memory-based tracking of an object across frames." }, { "q": "What are the limits of SAM 2.1 Hiera Large?", "a": "GPU in practice for video; needs points or boxes (no text prompts)." }, { "q": "Can SAM 2.1 Hiera Large be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SAM 2.1 Hiera Large run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin with transformers Sam2Model (safetensors, no remote code), prompted by boxes from a permissive detector." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/sam2.1-hiera-large", "https://arxiv.org/abs/2408.00714", "https://github.com/facebookresearch/sam2", "https://huggingface.co/docs/transformers/model_doc/sam2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "segmentation", "family": "sam2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "video" ], "modalities_out": [ "masks", "scores", "tracks" ], "use_cases": [ "segmentation", "object-detection", "tracking" ], "recommended": false, "release_date": "2024-09-24", "params_millions": 224.0, "size_on_disk_mb": 898.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip-base-patch16-256-multilingual", "identity": { "family": "siglip", "display_name": "SigLIP Base patch16 256 multilingual", "hub_id": "google/siglip-base-patch16-256-multilingual", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-01-08", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Non-English text queries against images in one model (multilingual text tower, 250k vocabulary).", "worst_at": "English-only retrieval quality is a little below the English base model; larger text tower." }, "io": { "input": { "image": { "resolution": "256x256", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "short captions; the card uses 'This is a photo of {label}.'; text padded to max_length 64" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1483.0, "params_millions": 371.0, "preprocessing": "Resize to 256x256, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 3.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip-base-patch16-256-multilingual@8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6" ], "formats": [ { "ref": "hf://google/siglip-base-patch16-256-multilingual@8952a4eafcde3cb7ab46b1dd629b33f8784ca9c6", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "multilingual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP Base patch16 256 multilingual: embedding model", "description": "SigLIP Base patch16 256 multilingual is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP Base patch16 256 multilingual", "siglip", "embedding", "google/siglip-base-patch16-256-multilingual", "visual-search", "multilingual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP Base patch16 256 multilingual best at?", "a": "Non-English text queries against images in one model (multilingual text tower, 250k vocabulary)." }, { "q": "What are the limits of SigLIP Base patch16 256 multilingual?", "a": "English-only retrieval quality is a little below the English base model; larger text tower." }, { "q": "Can SigLIP Base patch16 256 multilingual be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP Base patch16 256 multilingual run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP Base patch16 256 multilingual output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-base-patch16-256-multilingual", "https://arxiv.org/abs/2303.15343", "https://github.com/google-research/big_vision", "https://huggingface.co/docs/transformers/model_doc/siglip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "multilingual-search", "product-search" ], "recommended": false, "release_date": "2024-01-08", "params_millions": 371.0, "size_on_disk_mb": 1483.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 3.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip-base-patch16-384", "identity": { "family": "siglip", "display_name": "SigLIP Base patch16 (256/384/512)", "hub_id": "google/siglip-base-patch16-384", "api_name": null, "revision": null, "variants": [ "google/siglip-base-patch16-256", "google/siglip-base-patch16-512" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-01-08", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Same space as the served siglip-base-patch16-224 at higher resolution: small text and fine detail in frames.", "worst_at": "Costs about 3x the compute of 224 for 384; no gain on simple product or scene search." }, "io": { "input": { "image": { "resolution": "384x384 (256 and 512 siblings)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "short captions; the card uses 'This is a photo of {label}.'; text padded to max_length 64" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 814.0, "params_millions": 203.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip-base-patch16-384@41aec1c83b32e0a6fca20ad88ba058aa5b5ea394" ], "formats": [ { "ref": "hf://google/siglip-base-patch16-384@41aec1c83b32e0a6fca20ad88ba058aa5b5ea394", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP Base patch16 (256/384/512): embedding model", "description": "SigLIP Base patch16 (256/384/512) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP Base patch16 (256/384/512)", "siglip", "embedding", "google/siglip-base-patch16-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP Base patch16 (256/384/512) best at?", "a": "Same space as the served siglip-base-patch16-224 at higher resolution: small text and fine detail in frames." }, { "q": "What are the limits of SigLIP Base patch16 (256/384/512)?", "a": "Costs about 3x the compute of 224 for 384; no gain on simple product or scene search." }, { "q": "Can SigLIP Base patch16 (256/384/512) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP Base patch16 (256/384/512) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP Base patch16 (256/384/512) output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-base-patch16-384", "https://arxiv.org/abs/2303.15343", "https://github.com/google-research/big_vision", "https://huggingface.co/docs/transformers/model_doc/siglip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2024-01-08", "params_millions": 203.0, "size_on_disk_mb": 814.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip-large-patch16-384", "identity": { "family": "siglip", "display_name": "SigLIP Large patch16 (256/384)", "hub_id": "google/siglip-large-patch16-384", "api_name": null, "revision": null, "variants": [ "google/siglip-large-patch16-256" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-01-08", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Higher zero-shot and retrieval accuracy than base when a GPU is available.", "worst_at": "Slow on CPU for bulk frames; 1024-d vectors double index memory against base." }, "io": { "input": { "image": { "resolution": "384x384 (256 sibling)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "short captions; the card uses 'This is a photo of {label}.'; text padded to max_length 64" }, "output": { "vector": { "dims": 1024, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 2610.0, "params_millions": 652.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 4.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip-large-patch16-384@ce005573a40965dfd21fd937fbdeeebf2439fc35" ], "formats": [ { "ref": "hf://google/siglip-large-patch16-384@ce005573a40965dfd21fd937fbdeeebf2439fc35", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP Large patch16 (256/384): embedding model", "description": "SigLIP Large patch16 (256/384) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP Large patch16 (256/384)", "siglip", "embedding", "google/siglip-large-patch16-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP Large patch16 (256/384) best at?", "a": "Higher zero-shot and retrieval accuracy than base when a GPU is available." }, { "q": "What are the limits of SigLIP Large patch16 (256/384)?", "a": "Slow on CPU for bulk frames; 1024-d vectors double index memory against base." }, { "q": "Can SigLIP Large patch16 (256/384) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP Large patch16 (256/384) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size." }, { "q": "What does SigLIP Large patch16 (256/384) output?", "a": "A 1024-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-large-patch16-384", "https://arxiv.org/abs/2303.15343", "https://github.com/google-research/big_vision", "https://huggingface.co/docs/transformers/model_doc/siglip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2024-01-08", "params_millions": 652.0, "size_on_disk_mb": 2610.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 4.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip-so400m-patch14-384", "identity": { "family": "siglip", "display_name": "SigLIP so400m patch14 (224/384)", "hub_id": "google/siglip-so400m-patch14-384", "api_name": null, "revision": null, "variants": [ "google/siglip-so400m-patch14-224" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-01-08", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Strongest SigLIP v1 image-text model; the vision tower behind many VLMs (PaliGemma).", "worst_at": "About 0.9B parameters: GPU in practice for video frames; 1152-d vectors." }, "io": { "input": { "image": { "resolution": "384x384 (224 sibling)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "short captions; the card uses 'This is a photo of {label}.'; text padded to max_length 64" }, "output": { "vector": { "dims": 1152, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 3512.0, "params_millions": 878.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip-so400m-patch14-384@9fdffc58afc957d1a03a25b10dba0329ab15c2a3" ], "formats": [ { "ref": "hf://google/siglip-so400m-patch14-384@9fdffc58afc957d1a03a25b10dba0329ab15c2a3", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP so400m patch14 (224/384): embedding model", "description": "SigLIP so400m patch14 (224/384) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP so400m patch14 (224/384)", "siglip", "embedding", "google/siglip-so400m-patch14-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP so400m patch14 (224/384) best at?", "a": "Strongest SigLIP v1 image-text model; the vision tower behind many VLMs (PaliGemma)." }, { "q": "What are the limits of SigLIP so400m patch14 (224/384)?", "a": "About 0.9B parameters: GPU in practice for video frames; 1152-d vectors." }, { "q": "Can SigLIP so400m patch14 (224/384) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP so400m patch14 (224/384) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size." }, { "q": "What does SigLIP so400m patch14 (224/384) output?", "a": "A 1152-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-so400m-patch14-384", "https://arxiv.org/abs/2303.15343", "https://github.com/google-research/big_vision", "https://huggingface.co/docs/transformers/model_doc/siglip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2024-01-08", "params_millions": 878.0, "size_on_disk_mb": 3512.0, "vector_dims": 1152, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip-so400m-patch16-256-i18n", "identity": { "family": "siglip", "display_name": "SigLIP so400m patch16 256 i18n", "hub_id": "google/siglip-so400m-patch16-256-i18n", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2024-10-21", "paper_url": "https://arxiv.org/abs/2303.15343", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Multilingual text-to-image search at so400m quality.", "worst_at": "Largest SigLIP v1 checkpoint (1.1B); GPU in practice." }, "io": { "input": { "image": { "resolution": "256x256", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "short captions; the card uses 'This is a photo of {label}.'; text padded to max_length 64" }, "output": { "vector": { "dims": 1152, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (SiglipModel)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4515.0, "params_millions": 1129.0, "preprocessing": "Resize to 256x256, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 8.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip-so400m-patch16-256-i18n@365d321c0cfdea96bc28e3a29787a11a062681a1" ], "formats": [ { "ref": "hf://google/siglip-so400m-patch16-256-i18n@365d321c0cfdea96bc28e3a29787a11a062681a1", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "multilingual-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP so400m patch16 256 i18n: embedding model", "description": "SigLIP so400m patch16 256 i18n is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP so400m patch16 256 i18n", "siglip", "embedding", "google/siglip-so400m-patch16-256-i18n", "visual-search", "multilingual-search" ], "faqs": [ { "q": "What is SigLIP so400m patch16 256 i18n best at?", "a": "Multilingual text-to-image search at so400m quality." }, { "q": "What are the limits of SigLIP so400m patch16 256 i18n?", "a": "Largest SigLIP v1 checkpoint (1.1B); GPU in practice." }, { "q": "Can SigLIP so400m patch16 256 i18n be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP so400m patch16 256 i18n run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. GPU recommended at this size." }, { "q": "What does SigLIP so400m patch16 256 i18n output?", "a": "A 1152-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip-so400m-patch16-256-i18n", "https://arxiv.org/abs/2303.15343", "https://github.com/google-research/big_vision", "https://huggingface.co/docs/transformers/model_doc/siglip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "multilingual-search" ], "recommended": false, "release_date": "2024-10-21", "params_millions": 1129.0, "size_on_disk_mb": 4515.0, "vector_dims": 1152, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 8.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-base-patch16-224", "identity": { "family": "siglip2", "display_name": "SigLIP 2 Base patch16 (224/256/384/512, patch32-256)", "hub_id": "google/siglip2-base-patch16-224", "api_name": null, "revision": null, "variants": [ "google/siglip2-base-patch16-256", "google/siglip2-base-patch16-384", "google/siglip2-base-patch16-512", "google/siglip2-base-patch32-256" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-17", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Drop-in upgrade over SigLIP base: multilingual text, better localization and dense features at the same 768-d.", "worst_at": "Text tower is larger than SigLIP v1 base (Gemma vocabulary), so query embedding costs more on CPU." }, "io": { "input": { "image": { "resolution": "224x224 (256, 384, 512 siblings)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1501.0, "params_millions": 375.0, "preprocessing": "Resize to 224x224, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 3.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-base-patch16-224@75de2d55ec2d0b4efc50b3e9ad70dba96a7b2fa2" ], "formats": [ { "ref": "hf://google/siglip2-base-patch16-224@75de2d55ec2d0b4efc50b3e9ad70dba96a7b2fa2", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 Base patch16 (224/256/384/512, patch32-256): embedding model", "description": "SigLIP 2 Base patch16 (224/256/384/512, patch32-256) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 Base patch16 (224/256/384/512, patch32-256)", "siglip2", "embedding", "google/siglip2-base-patch16-224", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP 2 Base patch16 (224/256/384/512, patch32-256) best at?", "a": "Drop-in upgrade over SigLIP base: multilingual text, better localization and dense features at the same 768-d." }, { "q": "What are the limits of SigLIP 2 Base patch16 (224/256/384/512, patch32-256)?", "a": "Text tower is larger than SigLIP v1 base (Gemma vocabulary), so query embedding costs more on CPU." }, { "q": "Can SigLIP 2 Base patch16 (224/256/384/512, patch32-256) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 Base patch16 (224/256/384/512, patch32-256) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP 2 Base patch16 (224/256/384/512, patch32-256) output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-base-patch16-224", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2025-02-17", "params_millions": 375.0, "size_on_disk_mb": 1501.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 3.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-base-patch16-naflex", "identity": { "family": "siglip2", "display_name": "SigLIP 2 Base patch16 NaFlex", "hub_id": "google/siglip2-base-patch16-naflex", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-18", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text", "document" ], "modalities_out": [ "vector" ], "best_at": "Documents, screenshots, receipts and wide or tall images: keeps the native aspect ratio instead of squashing to a square.", "worst_at": "Variable sequence length per image makes batching uneven; square photos gain nothing over the fixed-resolution model." }, "io": { "input": { "image": { "resolution": "native aspect ratio, resized so patches <= max_num_patches (default 256; 128 to 1024)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search" ], "composes_with": [ "PDF pages and screenshots -> NaFlex image vectors -> feature_search with a text query" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. The processor takes max_num_patches; batches carry a pixel attention mask.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model, NaFlex processor)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1501.0, "params_millions": 375.0, "preprocessing": "Resize keeping aspect ratio to fit max_num_patches of 16x16, normalize mean 0.5 std 0.5, pad patches and build pixel_attention_mask.", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 16, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 3.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-base-patch16-naflex@b53b807d3a2d5e2b3911292f2d69e5341cdc064c" ], "formats": [ { "ref": "hf://google/siglip2-base-patch16-naflex@b53b807d3a2d5e2b3911292f2d69e5341cdc064c", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "document-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 Base patch16 NaFlex: embedding model", "description": "SigLIP 2 Base patch16 NaFlex is a image, text and document embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 Base patch16 NaFlex", "siglip2", "embedding", "google/siglip2-base-patch16-naflex", "visual-search", "document-search" ], "faqs": [ { "q": "What is SigLIP 2 Base patch16 NaFlex best at?", "a": "Documents, screenshots, receipts and wide or tall images: keeps the native aspect ratio instead of squashing to a square." }, { "q": "What are the limits of SigLIP 2 Base patch16 NaFlex?", "a": "Variable sequence length per image makes batching uneven; square photos gain nothing over the fixed-resolution model." }, { "q": "Can SigLIP 2 Base patch16 NaFlex be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 Base patch16 NaFlex run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. The processor takes max_num_patches; batches carry a pixel attention mask." }, { "q": "What does SigLIP 2 Base patch16 NaFlex output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-base-patch16-naflex", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text", "document" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "document-search" ], "recommended": false, "release_date": "2025-02-18", "params_millions": 375.0, "size_on_disk_mb": 1501.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 3.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-giant-opt-patch16-384", "identity": { "family": "siglip2", "display_name": "SigLIP 2 giant-opt patch16 (256/384)", "hub_id": "google/siglip2-giant-opt-patch16-384", "api_name": null, "revision": null, "variants": [ "google/siglip2-giant-opt-patch16-256" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-17", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Best SigLIP 2 accuracy for offline catalog indexing.", "worst_at": "1.9B parameters and 1536-d vectors: heavy on GPU memory and index size; impractical on CPU." }, "io": { "input": { "image": { "resolution": "384x384 (256 sibling)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 1536, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 7488.0, "params_millions": 1872.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 12.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-giant-opt-patch16-384@a713301b217d38485fb2204c808367d10bc3cc40" ], "formats": [ { "ref": "hf://google/siglip2-giant-opt-patch16-384@a713301b217d38485fb2204c808367d10bc3cc40", "format": "safetensors", "file": "2 files, e.g. model-00001-of-00002.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 giant-opt patch16 (256/384): embedding model", "description": "SigLIP 2 giant-opt patch16 (256/384) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 giant-opt patch16 (256/384)", "siglip2", "embedding", "google/siglip2-giant-opt-patch16-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP 2 giant-opt patch16 (256/384) best at?", "a": "Best SigLIP 2 accuracy for offline catalog indexing." }, { "q": "What are the limits of SigLIP 2 giant-opt patch16 (256/384)?", "a": "1.9B parameters and 1536-d vectors: heavy on GPU memory and index size; impractical on CPU." }, { "q": "Can SigLIP 2 giant-opt patch16 (256/384) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 giant-opt patch16 (256/384) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP 2 giant-opt patch16 (256/384) output?", "a": "A 1536-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-giant-opt-patch16-384", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2025-02-17", "params_millions": 1872.0, "size_on_disk_mb": 7488.0, "vector_dims": 1536, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 12.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-large-patch16-384", "identity": { "family": "siglip2", "display_name": "SigLIP 2 Large patch16 (256/384/512)", "hub_id": "google/siglip2-large-patch16-384", "api_name": null, "revision": null, "variants": [ "google/siglip2-large-patch16-256", "google/siglip2-large-patch16-512" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-17", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Higher retrieval accuracy than base for catalogs and frames when a GPU is available.", "worst_at": "Slow on CPU for bulk video frames." }, "io": { "input": { "image": { "resolution": "384x384 (256, 512 siblings)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 1024, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 3528.0, "params_millions": 882.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 5.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-large-patch16-384@1b426889ea62b5a72bf9839009a1b184bfc9c178" ], "formats": [ { "ref": "hf://google/siglip2-large-patch16-384@1b426889ea62b5a72bf9839009a1b184bfc9c178", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 Large patch16 (256/384/512): embedding model", "description": "SigLIP 2 Large patch16 (256/384/512) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 Large patch16 (256/384/512)", "siglip2", "embedding", "google/siglip2-large-patch16-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP 2 Large patch16 (256/384/512) best at?", "a": "Higher retrieval accuracy than base for catalogs and frames when a GPU is available." }, { "q": "What are the limits of SigLIP 2 Large patch16 (256/384/512)?", "a": "Slow on CPU for bulk video frames." }, { "q": "Can SigLIP 2 Large patch16 (256/384/512) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 Large patch16 (256/384/512) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP 2 Large patch16 (256/384/512) output?", "a": "A 1024-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-large-patch16-384", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2025-02-17", "params_millions": 882.0, "size_on_disk_mb": 3528.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 5.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-so400m-patch16-384", "identity": { "family": "siglip2", "display_name": "SigLIP 2 so400m patch16 (256/384/512)", "hub_id": "google/siglip2-so400m-patch16-384", "api_name": null, "revision": null, "variants": [ "google/siglip2-so400m-patch16-256", "google/siglip2-so400m-patch16-512" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-17", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Patch16 so400m: fewer tokens than patch14 at the same resolution, so faster per frame.", "worst_at": "Same 1.1B parameter cost as patch14; GPU in practice." }, "io": { "input": { "image": { "resolution": "384x384 (256, 512 siblings)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 1152, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4544.0, "params_millions": 1136.0, "preprocessing": "Resize to 384x384, rescale to 0..1, normalize mean 0.5 std 0.5; text padded to 64 tokens (max_length).", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 8.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-so400m-patch16-384@dd658faac399427308559e2c3ac1e99cbe43845d" ], "formats": [ { "ref": "hf://google/siglip2-so400m-patch16-384@dd658faac399427308559e2c3ac1e99cbe43845d", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 so400m patch16 (256/384/512): embedding model", "description": "SigLIP 2 so400m patch16 (256/384/512) is a image and text embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 so400m patch16 (256/384/512)", "siglip2", "embedding", "google/siglip2-so400m-patch16-384", "visual-search", "product-search" ], "faqs": [ { "q": "What is SigLIP 2 so400m patch16 (256/384/512) best at?", "a": "Patch16 so400m: fewer tokens than patch14 at the same resolution, so faster per frame." }, { "q": "What are the limits of SigLIP 2 so400m patch16 (256/384/512)?", "a": "Same 1.1B parameter cost as patch14; GPU in practice." }, { "q": "Can SigLIP 2 so400m patch16 (256/384/512) be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 so400m patch16 (256/384/512) run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false." }, { "q": "What does SigLIP 2 so400m patch16 (256/384/512) output?", "a": "A 1152-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-so400m-patch16-384", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2025-02-17", "params_millions": 1136.0, "size_on_disk_mb": 4544.0, "vector_dims": 1152, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 8.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "siglip2-so400m-patch16-naflex", "identity": { "family": "siglip2", "display_name": "SigLIP 2 so400m patch16 NaFlex", "hub_id": "google/siglip2-so400m-patch16-naflex", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2025-02-18", "paper_url": "https://arxiv.org/abs/2502.14786", "repo_url": "https://github.com/google-research/big_vision" }, "task": { "category": "embedding", "modalities_in": [ "image", "text", "document" ], "modalities_out": [ "vector" ], "best_at": "Documents, screenshots, receipts and wide or tall images: keeps the native aspect ratio instead of squashing to a square.", "worst_at": "Variable sequence length per image makes batching uneven; square photos gain nothing over the fixed-resolution model." }, "io": { "input": { "image": { "resolution": "native aspect ratio, resized so patches <= max_num_patches (default 256; 128 to 1024)", "color": "RGB", "normalize": "mean 0.5, std 0.5" }, "text": { "max_tokens": 64 }, "prompts": "lowercase captions; the card uses 'this is a photo of {label}.'; Gemma tokenizer, padded to max_length 64" }, "output": { "vector": { "dims": 1152, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search" ], "composes_with": [ "PDF pages and screenshots -> NaFlex image vectors -> feature_search with a text query" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. The processor takes max_num_patches; batches carry a pixel attention mask.", "serve_apps": [], "runtime": "transformers AutoModel + AutoProcessor (Siglip2Model, NaFlex processor)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4543.0, "params_millions": 1136.0, "preprocessing": "Resize keeping aspect ratio to fit max_num_patches of 16x16, normalize mean 0.5 std 0.5, pad patches and build pixel_attention_mask.", "postprocessing": "L2-normalize pooled image and text embeddings; sigmoid of scaled logits for zero-shot scores.", "serving_shape": "both", "batch_size": 16, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 8.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/siglip2-so400m-patch16-naflex@cc24074f717b612951c2dead130904ab9b65a81e" ], "formats": [ { "ref": "hf://google/siglip2-so400m-patch16-naflex@cc24074f717b612951c2dead130904ab9b65a81e", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "visual-search", "document-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SigLIP 2 so400m patch16 NaFlex: embedding model", "description": "SigLIP 2 so400m patch16 NaFlex is a image, text and document embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "SigLIP 2 so400m patch16 NaFlex", "siglip2", "embedding", "google/siglip2-so400m-patch16-naflex", "visual-search", "document-search" ], "faqs": [ { "q": "What is SigLIP 2 so400m patch16 NaFlex best at?", "a": "Documents, screenshots, receipts and wide or tall images: keeps the native aspect ratio instead of squashing to a square." }, { "q": "What are the limits of SigLIP 2 so400m patch16 NaFlex?", "a": "Variable sequence length per image makes batching uneven; square photos gain nothing over the fixed-resolution model." }, { "q": "Can SigLIP 2 so400m patch16 NaFlex be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does SigLIP 2 so400m patch16 NaFlex run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin (engine/extractors//v1/ with realtime.py for query embedding), loading safetensors with trust_remote_code false. The processor takes max_num_patches; batches carry a pixel attention mask." }, { "q": "What does SigLIP 2 so400m patch16 NaFlex output?", "a": "A 1152-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/siglip2-so400m-patch16-naflex", "https://arxiv.org/abs/2502.14786", "https://huggingface.co/docs/transformers/model_doc/siglip2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "siglip2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image", "text", "document" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "document-search" ], "recommended": false, "release_date": "2025-02-18", "params_millions": 1136.0, "size_on_disk_mb": 4543.0, "vector_dims": 1152, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 8.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "vjepa2-vitg-fpc64-256", "identity": { "family": "vjepa2", "display_name": "V-JEPA 2 VITG", "hub_id": "facebook/vjepa2-vitg-fpc64-256", "api_name": null, "revision": null, "variants": [ "facebook/vjepa2-vitg-fpc64-384 (384px)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the hub tag.", "release_date": "2025-06-11", "paper_url": "https://arxiv.org/abs/2506.09985", "repo_url": "https://github.com/facebookresearch/vjepa2" }, "task": { "category": "embedding", "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "best_at": "Self-supervised video features that capture motion and physical dynamics; strong with attentive probes for action anticipation and classification.", "worst_at": "No text tower: cannot answer text queries without training a probe or alignment; 64-frame clips are heavy." }, "io": { "input": { "video": { "frames": 64, "resolution": "256x256" } }, "output": { "vector": { "dims": 1408, "distance": "cosine", "type": "dense", "note": "patch tokens; pool for a clip vector" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits -> V-JEPA 2 pooled clip vectors -> feature_search by example clip", "V-JEPA 2 features -> trained probe -> action labels -> attribute filter" ] }, "run": { "status": "plugin", "status_reason": "Not served. Safetensors with transformers VJEPA2Model (no remote code); would run as a GPU batch plugin producing clip vectors (too large for CPU scale to zero).", "serve_apps": [], "runtime": "transformers VJEPA2Model + AutoVideoProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 4138.0, "params_millions": 1035.0, "preprocessing": "Sample 64 frames, resize and center-crop, normalize ImageNet mean/std.", "postprocessing": "Mean-pool patch tokens (or attentive probe); L2-normalize.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 16.0, "num_cpus": 4.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/vjepa2-vitg-fpc64-256@875c192b7b704b87d1e1d99345769632dd5f739a" ], "formats": [ { "ref": "hf://facebook/vjepa2-vitg-fpc64-256@875c192b7b704b87d1e1d99345769632dd5f739a", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/vjepa2-vitg-fpc64-256@875c192b7b704b87d1e1d99345769632dd5f739a", "format": "pytorch_pt", "file": "original/model.pth", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "V-JEPA 2 VITG: embedding model", "description": "V-JEPA 2 VITG is a video embedding model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "V-JEPA 2 VITG", "vjepa2", "embedding", "facebook/vjepa2-vitg-fpc64-256", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is V-JEPA 2 VITG best at?", "a": "Self-supervised video features that capture motion and physical dynamics; strong with attentive probes for action anticipation and classification." }, { "q": "What are the limits of V-JEPA 2 VITG?", "a": "No text tower: cannot answer text queries without training a probe or alignment; 64-frame clips are heavy." }, { "q": "Can V-JEPA 2 VITG be used commercially?", "a": "Yes. Apache-2.0 per the hub tag." }, { "q": "How does V-JEPA 2 VITG run on Mixpeek?", "a": "Not served. Safetensors with transformers VJEPA2Model (no remote code); would run as a GPU batch plugin producing clip vectors (too large for CPU scale to zero)." }, { "q": "What does V-JEPA 2 VITG output?", "a": "A 1408-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/vjepa2-vitg-fpc64-256", "https://arxiv.org/abs/2506.09985", "https://github.com/facebookresearch/vjepa2", "https://huggingface.co/docs/transformers/model_doc/vjepa2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "vjepa2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2025-06-11", "params_millions": 1035.0, "size_on_disk_mb": 4138.0, "vector_dims": 1408, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 16.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "vjepa2-vitl-fpc64-256", "identity": { "family": "vjepa2", "display_name": "V-JEPA 2 VITL", "hub_id": "facebook/vjepa2-vitl-fpc64-256", "api_name": null, "revision": null, "variants": [ "facebook/vjepa2-vith-fpc64-256 (MIT, 654M, 1280-d)" ], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT per the hub tag.", "release_date": "2025-06-11", "paper_url": "https://arxiv.org/abs/2506.09985", "repo_url": "https://github.com/facebookresearch/vjepa2" }, "task": { "category": "embedding", "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "best_at": "Self-supervised video features that capture motion and physical dynamics; strong with attentive probes for action anticipation and classification.", "worst_at": "No text tower: cannot answer text queries without training a probe or alignment; 64-frame clips are heavy." }, "io": { "input": { "video": { "frames": 64, "resolution": "256x256" } }, "output": { "vector": { "dims": 1024, "distance": "cosine", "type": "dense", "note": "patch tokens; pool for a clip vector" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits -> V-JEPA 2 pooled clip vectors -> feature_search by example clip", "V-JEPA 2 features -> trained probe -> action labels -> attribute filter" ] }, "run": { "status": "plugin", "status_reason": "Not served. Safetensors with transformers VJEPA2Model (no remote code); would run as a GPU batch plugin producing clip vectors; CPU possible but slow at 64 frames.", "serve_apps": [], "runtime": "transformers VJEPA2Model + AutoVideoProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 1304.0, "params_millions": 326.0, "preprocessing": "Sample 64 frames, resize and center-crop, normalize ImageNet mean/std.", "postprocessing": "Mean-pool patch tokens (or attentive probe); L2-normalize.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 4.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/vjepa2-vitl-fpc64-256@b3c1679b7c34d3255ef3547f27c7b226aefab26f" ], "formats": [ { "ref": "hf://facebook/vjepa2-vitl-fpc64-256@b3c1679b7c34d3255ef3547f27c7b226aefab26f", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/vjepa2-vitl-fpc64-256@b3c1679b7c34d3255ef3547f27c7b226aefab26f", "format": "pytorch_pt", "file": "original/model.pth", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "V-JEPA 2 VITL: embedding model", "description": "V-JEPA 2 VITL is a video embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "V-JEPA 2 VITL", "vjepa2", "embedding", "facebook/vjepa2-vitl-fpc64-256", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is V-JEPA 2 VITL best at?", "a": "Self-supervised video features that capture motion and physical dynamics; strong with attentive probes for action anticipation and classification." }, { "q": "What are the limits of V-JEPA 2 VITL?", "a": "No text tower: cannot answer text queries without training a probe or alignment; 64-frame clips are heavy." }, { "q": "Can V-JEPA 2 VITL be used commercially?", "a": "Yes. MIT per the hub tag." }, { "q": "How does V-JEPA 2 VITL run on Mixpeek?", "a": "Not served. Safetensors with transformers VJEPA2Model (no remote code); would run as a GPU batch plugin producing clip vectors; CPU possible but slow at 64 frames." }, { "q": "What does V-JEPA 2 VITL output?", "a": "A 1024-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/vjepa2-vitl-fpc64-256", "https://arxiv.org/abs/2506.09985", "https://github.com/facebookresearch/vjepa2", "https://huggingface.co/docs/transformers/model_doc/vjepa2" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "vjepa2", "license": "MIT", "commercial_use": true, "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2025-06-11", "params_millions": 326.0, "size_on_disk_mb": 1304.0, "vector_dims": 1024, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "xclip-base-patch16", "identity": { "family": "xclip", "display_name": "X-CLIP base-patch16", "hub_id": "microsoft/xclip-base-patch16", "api_name": null, "revision": null, "variants": [ "microsoft/xclip-base-patch16-zero-shot (32 frames)" ], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT per the hub tag and repo.", "release_date": "2022-08", "paper_url": "https://arxiv.org/abs/2208.02816", "repo_url": "https://github.com/microsoft/VideoX/tree/master/X-CLIP" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained).", "worst_at": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, "io": { "input": { "video": { "frames": 8, "resolution": "224x224" }, "text": { "max_tokens": 77 }, "prompts": "action phrases" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search" ], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "plugin", "status_reason": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding.", "serve_apps": [], "runtime": "transformers XCLIPModel + XCLIPProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 780.0, "params_millions": 195.0, "preprocessing": "Sample 8 frames uniformly, resize and center-crop to 224, CLIP mean/std.", "postprocessing": "L2-normalize video and text embeddings.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://microsoft/xclip-base-patch16@d6184e3fd8780d04c85d0f1eabe5f94bf44d98f6" ], "formats": [ { "ref": "hf://microsoft/xclip-base-patch16@d6184e3fd8780d04c85d0f1eabe5f94bf44d98f6", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://microsoft/xclip-base-patch16@d6184e3fd8780d04c85d0f1eabe5f94bf44d98f6", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "X-CLIP base-patch16: embedding model", "description": "X-CLIP base-patch16 is a video and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "X-CLIP base-patch16", "xclip", "embedding", "microsoft/xclip-base-patch16", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is X-CLIP base-patch16 best at?", "a": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained)." }, { "q": "What are the limits of X-CLIP base-patch16?", "a": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, { "q": "Can X-CLIP base-patch16 be used commercially?", "a": "Yes. MIT per the hub tag and repo." }, { "q": "How does X-CLIP base-patch16 run on Mixpeek?", "a": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding." }, { "q": "What does X-CLIP base-patch16 output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/microsoft/xclip-base-patch16", "https://arxiv.org/abs/2208.02816", "https://huggingface.co/docs/transformers/model_doc/xclip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "xclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2022-08", "params_millions": 195.0, "size_on_disk_mb": 780.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "xclip-base-patch32", "identity": { "family": "xclip", "display_name": "X-CLIP base-patch32", "hub_id": "microsoft/xclip-base-patch32", "api_name": null, "revision": null, "variants": [ "microsoft/xclip-base-patch32-16-frames" ], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT per the hub tag and repo.", "release_date": "2022-08", "paper_url": "https://arxiv.org/abs/2208.02816", "repo_url": "https://github.com/microsoft/VideoX/tree/master/X-CLIP" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained).", "worst_at": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, "io": { "input": { "video": { "frames": 8, "resolution": "224x224" }, "text": { "max_tokens": 77 }, "prompts": "action phrases" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search" ], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "plugin", "status_reason": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding.", "serve_apps": [], "runtime": "transformers XCLIPModel + XCLIPProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 786.0, "params_millions": 197.0, "preprocessing": "Sample 8 frames uniformly, resize and center-crop to 224, CLIP mean/std.", "postprocessing": "L2-normalize video and text embeddings.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://microsoft/xclip-base-patch32@a2e27a78a2b5d802e894b8a1ef14f3a8ce490963" ], "formats": [ { "ref": "hf://microsoft/xclip-base-patch32@a2e27a78a2b5d802e894b8a1ef14f3a8ce490963", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://microsoft/xclip-base-patch32@a2e27a78a2b5d802e894b8a1ef14f3a8ce490963", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "X-CLIP base-patch32: embedding model", "description": "X-CLIP base-patch32 is a video and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "X-CLIP base-patch32", "xclip", "embedding", "microsoft/xclip-base-patch32", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is X-CLIP base-patch32 best at?", "a": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained)." }, { "q": "What are the limits of X-CLIP base-patch32?", "a": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, { "q": "Can X-CLIP base-patch32 be used commercially?", "a": "Yes. MIT per the hub tag and repo." }, { "q": "How does X-CLIP base-patch32 run on Mixpeek?", "a": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding." }, { "q": "What does X-CLIP base-patch32 output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/microsoft/xclip-base-patch32", "https://arxiv.org/abs/2208.02816", "https://huggingface.co/docs/transformers/model_doc/xclip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "xclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2022-08", "params_millions": 197.0, "size_on_disk_mb": 786.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "xclip-large-patch14", "identity": { "family": "xclip", "display_name": "X-CLIP large-patch14", "hub_id": "microsoft/xclip-large-patch14", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT per the hub tag and repo.", "release_date": "2022-08", "paper_url": "https://arxiv.org/abs/2208.02816", "repo_url": "https://github.com/microsoft/VideoX/tree/master/X-CLIP" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained).", "worst_at": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, "io": { "input": { "video": { "frames": 8, "resolution": "224x224" }, "text": { "max_tokens": 77 }, "prompts": "action phrases" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search" ], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "plugin", "status_reason": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding.", "serve_apps": [], "runtime": "transformers XCLIPModel + XCLIPProcessor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 2303.0, "params_millions": 576.0, "preprocessing": "Sample 8 frames uniformly, resize and center-crop to 224, CLIP mean/std.", "postprocessing": "L2-normalize video and text embeddings.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 4.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://microsoft/xclip-large-patch14@a9dd1429a16cf305df2aaea232d5e8dceba1c675" ], "formats": [ { "ref": "hf://microsoft/xclip-large-patch14@a9dd1429a16cf305df2aaea232d5e8dceba1c675", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://microsoft/xclip-large-patch14@a9dd1429a16cf305df2aaea232d5e8dceba1c675", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "X-CLIP large-patch14: embedding model", "description": "X-CLIP large-patch14 is a video and text embedding model (MIT). Can run on Mixpeek as a custom plugin.", "keywords": [ "X-CLIP large-patch14", "xclip", "embedding", "microsoft/xclip-large-patch14", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is X-CLIP large-patch14 best at?", "a": "Cheap video-text matching on short clips with a CLIP backbone; zero-shot action labels (Kinetics-trained)." }, { "q": "What are the limits of X-CLIP large-patch14?", "a": "Trained on Kinetics-400 action clips: weak on general scenes, logos, people and speech; 8 frames per clip." }, { "q": "Can X-CLIP large-patch14 be used commercially?", "a": "Yes. MIT per the hub tag and repo." }, { "q": "How does X-CLIP large-patch14 run on Mixpeek?", "a": "Not served. Safetensors with transformers XCLIPModel (no remote code); would run as a batch plugin over scene clips, with realtime text query embedding." }, { "q": "What does X-CLIP large-patch14 output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/microsoft/xclip-large-patch14", "https://arxiv.org/abs/2208.02816", "https://huggingface.co/docs/transformers/model_doc/xclip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "embedding", "family": "xclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2022-08", "params_millions": 576.0, "size_on_disk_mb": 2303.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_bin", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 4.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "yolox-s", "identity": { "family": "yolox", "display_name": "YOLOX-S", "hub_id": "yolox_s (Megvii release asset)", "api_name": null, "revision": null, "variants": [ "yolox_nano", "yolox_tiny", "yolox_m", "yolox_l", "yolox_x" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2021-07", "paper_url": "https://arxiv.org/abs/2107.08430", "repo_url": "https://github.com/Megvii-BaseDetection/YOLOX" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Mature Apache-2.0 YOLO-style detector with official ONNX exports; easy CPU deployment.", "worst_at": "Older (2021): lower accuracy than D-FINE or RT-DETRv2 at equal cost; closed COCO label set." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "BGR, no normalization (reference demo)" }, "prompts": "none" }, "output": { "boxes": "xyxy pixels after grid decode", "labels": "COCO 80 classes", "scores": "objectness x class" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "plugin", "status_reason": "Not served. Would run as a custom extractor plugin with onnxruntime on CPU using the official yolox_s.onnx release asset (no Python model code needed).", "serve_apps": [], "runtime": "onnxruntime (official ONNX release)", "weights_format": "onnx", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": 9.0, "preprocessing": "Letterbox to 640 with padding value 114.", "postprocessing": "Decode grid outputs, score threshold, NMS.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://Megvii-BaseDetection/YOLOX/yolox_s.onnx@0.1.1rc0" ], "formats": [ { "ref": "github://Megvii-BaseDetection/YOLOX/yolox_s.onnx@0.1.1rc0", "format": "onnx", "file": "yolox_s.onnx", "loader": "onnxruntime; Mixpeek model upload (api/models/loadability.py, ONNX IR <= 11)", "supported_today": true } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "YOLOX-S: object detection model", "description": "YOLOX-S is a image object detection model (Apache-2.0). Can run on Mixpeek as a custom plugin.", "keywords": [ "YOLOX-S", "yolox", "object detection", "yolox_s (Megvii release asset)", "object-detection" ], "faqs": [ { "q": "What is YOLOX-S best at?", "a": "Mature Apache-2.0 YOLO-style detector with official ONNX exports; easy CPU deployment." }, { "q": "What are the limits of YOLOX-S?", "a": "Older (2021): lower accuracy than D-FINE or RT-DETRv2 at equal cost; closed COCO label set." }, { "q": "Can YOLOX-S be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does YOLOX-S run on Mixpeek?", "a": "Not served. Would run as a custom extractor plugin with onnxruntime on CPU using the official yolox_s.onnx release asset (no Python model code needed)." } ] }, "last_verified": "2026-11-05", "sources": [ "https://github.com/Megvii-BaseDetection/YOLOX", "https://arxiv.org/abs/2107.08430", "https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "plugin", "category": "detection", "family": "yolox", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2021-07", "params_millions": 9.0, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "onnx", "formats": [ "onnx" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 1.0 }, "rank_score": 1.0 }, { "id": "clip-vit-b-16-datacomp-xl-s13b-b90k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-B/16 DataComp-1B", "hub_id": "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2023-05", "paper_url": "https://arxiv.org/abs/2304.14108", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Strong B/16 trained on the filtered DataComp-1B set.", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "not_runnable", "status_reason": "Only open_clip_pytorch_model.bin (pickle) on the hub; a plugin needs a one-time safetensors conversion. Otherwise loads with open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K').", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 599.0, "params_millions": 150.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K@d110532e8d4ff91c574ee60a342323f28468b287" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K@d110532e8d4ff91c574ee60a342323f28468b287", "format": "pytorch_bin", "file": "open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-B/16 DataComp-1B: embedding model", "description": "OpenCLIP ViT-B/16 DataComp-1B is a image and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "OpenCLIP ViT-B/16 DataComp-1B", "openclip", "embedding", "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-B/16 DataComp-1B best at?", "a": "Strong B/16 trained on the filtered DataComp-1B set." }, { "q": "What are the limits of OpenCLIP ViT-B/16 DataComp-1B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-B/16 DataComp-1B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-B/16 DataComp-1B run on Mixpeek?", "a": "Only open_clip_pytorch_model.bin (pickle) on the hub; a plugin needs a one-time safetensors conversion. Otherwise loads with open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K')." }, { "q": "What does OpenCLIP ViT-B/16 DataComp-1B output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K", "https://arxiv.org/abs/2304.14108", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2023-05", "params_millions": 150.0, "size_on_disk_mb": 599.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_bin" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "clip-vit-base-patch16", "identity": { "family": "clip", "display_name": "OpenAI CLIP ViT-B/16", "hub_id": "openai/clip-vit-base-patch16", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call.", "release_date": "2021-01", "paper_url": "https://arxiv.org/abs/2103.00020", "repo_url": "https://github.com/openai/CLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "General zero-shot labels and text-to-image search; the reference baseline most papers report against.", "worst_at": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; BPE tokenizer, 77 tokens max" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "not_runnable", "status_reason": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel).", "serve_apps": [], "runtime": "transformers CLIPModel + CLIPProcessor", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 599.0, "params_millions": 150.0, "preprocessing": "Resize shortest side and center-crop to 224x224, normalize with CLIP mean/std; text to 77 tokens.", "postprocessing": "L2-normalize image and text embeddings; logits = 100 * cosine for zero-shot.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://openai/clip-vit-base-patch16@57c216476eefef5ab752ec549e440a49ae4ae5f3" ], "formats": [ { "ref": "hf://openai/clip-vit-base-patch16@57c216476eefef5ab752ec549e440a49ae4ae5f3", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://openai/clip-vit-base-patch16@57c216476eefef5ab752ec549e440a49ae4ae5f3", "format": "jax", "file": "flax_model.msgpack", "loader": "JAX/Flax; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenAI CLIP ViT-B/16: embedding model", "description": "OpenAI CLIP ViT-B/16 is a image and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "OpenAI CLIP ViT-B/16", "clip", "embedding", "openai/clip-vit-base-patch16", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenAI CLIP ViT-B/16 best at?", "a": "General zero-shot labels and text-to-image search; the reference baseline most papers report against." }, { "q": "What are the limits of OpenAI CLIP ViT-B/16?", "a": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, { "q": "Can OpenAI CLIP ViT-B/16 be used commercially?", "a": "Yes. MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call." }, { "q": "How does OpenAI CLIP ViT-B/16 run on Mixpeek?", "a": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel)." }, { "q": "What does OpenAI CLIP ViT-B/16 output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/openai/clip-vit-base-patch16", "https://arxiv.org/abs/2103.00020", "https://github.com/openai/CLIP/blob/main/LICENSE", "https://github.com/openai/CLIP/blob/main/model-card.md" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "clip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2021-01", "params_millions": 150.0, "size_on_disk_mb": 599.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "jax", "pytorch_bin" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "clip-vit-base-patch32", "identity": { "family": "clip", "display_name": "OpenAI CLIP ViT-B/32", "hub_id": "openai/clip-vit-base-patch32", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call.", "release_date": "2021-01", "paper_url": "https://arxiv.org/abs/2103.00020", "repo_url": "https://github.com/openai/CLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "General zero-shot labels and text-to-image search; the reference baseline most papers report against.", "worst_at": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; BPE tokenizer, 77 tokens max" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "not_runnable", "status_reason": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel).", "serve_apps": [], "runtime": "transformers CLIPModel + CLIPProcessor", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 605.0, "params_millions": 151.0, "preprocessing": "Resize shortest side and center-crop to 224x224, normalize with CLIP mean/std; text to 77 tokens.", "postprocessing": "L2-normalize image and text embeddings; logits = 100 * cosine for zero-shot.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://openai/clip-vit-base-patch32@3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "mixpeek://system/clip_vit_b_32@1.0.0" ], "formats": [ { "ref": "hf://openai/clip-vit-base-patch32@3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://openai/clip-vit-base-patch32@3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "format": "jax", "file": "flax_model.msgpack", "loader": "JAX/Flax; not in Mixpeek's runtime", "supported_today": false }, { "ref": "hf://openai/clip-vit-base-patch32@3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268", "format": "tensorflow", "file": "tf_model.h5", "loader": "TensorFlow; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenAI CLIP ViT-B/32: embedding model", "description": "OpenAI CLIP ViT-B/32 is a image and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "OpenAI CLIP ViT-B/32", "clip", "embedding", "openai/clip-vit-base-patch32", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenAI CLIP ViT-B/32 best at?", "a": "General zero-shot labels and text-to-image search; the reference baseline most papers report against." }, { "q": "What are the limits of OpenAI CLIP ViT-B/32?", "a": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, { "q": "Can OpenAI CLIP ViT-B/32 be used commercially?", "a": "Yes. MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call." }, { "q": "How does OpenAI CLIP ViT-B/32 run on Mixpeek?", "a": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel)." }, { "q": "What does OpenAI CLIP ViT-B/32 output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/openai/clip-vit-base-patch32", "https://arxiv.org/abs/2103.00020", "https://github.com/openai/CLIP/blob/main/LICENSE", "https://github.com/openai/CLIP/blob/main/model-card.md" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "clip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2021-01", "params_millions": 151.0, "size_on_disk_mb": 605.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "jax", "pytorch_bin", "tensorflow" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf", "mixpeek" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "clip-vit-l-14-datacomp-xl-s13b-b90k", "identity": { "family": "openclip", "display_name": "OpenCLIP ViT-L/14 DataComp-1B", "hub_id": "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction.", "release_date": "2023-04", "paper_url": "https://arxiv.org/abs/2304.14108", "repo_url": "https://github.com/mlfoundations/open_clip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Best open L/14 CLIP of its time on ImageNet zero-shot (79.2% per the card).", "worst_at": "English-centric text tower (77 tokens); weak on text in images and counting." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; open_clip tokenizer, 77 tokens" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "not_runnable", "status_reason": "Only open_clip_pytorch_model.bin (pickle) on the hub; a plugin needs a one-time safetensors conversion. Otherwise loads with open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K'). GPU recommended at this size.", "serve_apps": [], "runtime": "open_clip create_model_and_transforms('hf-hub:')", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 1711.0, "params_millions": 428.0, "preprocessing": "Resize and center-crop to 224, CLIP mean/std normalize; tokenize to 77.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 4.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K@84c9828e63dc9a9351d1fe637c346d4c1c4db341" ], "formats": [ { "ref": "hf://laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K@84c9828e63dc9a9351d1fe637c346d4c1c4db341", "format": "pytorch_bin", "file": "2 files, e.g. open_clip_pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenCLIP ViT-L/14 DataComp-1B: embedding model", "description": "OpenCLIP ViT-L/14 DataComp-1B is a image and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "OpenCLIP ViT-L/14 DataComp-1B", "openclip", "embedding", "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenCLIP ViT-L/14 DataComp-1B best at?", "a": "Best open L/14 CLIP of its time on ImageNet zero-shot (79.2% per the card)." }, { "q": "What are the limits of OpenCLIP ViT-L/14 DataComp-1B?", "a": "English-centric text tower (77 tokens); weak on text in images and counting." }, { "q": "Can OpenCLIP ViT-L/14 DataComp-1B be used commercially?", "a": "Yes. MIT weights; LAION notes the datasets are for research, the model license itself has no use restriction." }, { "q": "How does OpenCLIP ViT-L/14 DataComp-1B run on Mixpeek?", "a": "Only open_clip_pytorch_model.bin (pickle) on the hub; a plugin needs a one-time safetensors conversion. Otherwise loads with open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K'). GPU recommended at this size." }, { "q": "What does OpenCLIP ViT-L/14 DataComp-1B output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "https://arxiv.org/abs/2304.14108", "https://github.com/mlfoundations/open_clip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "openclip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2023-04", "params_millions": 428.0, "size_on_disk_mb": 1711.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_bin" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 1.0, "memory_gb": 4.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "clip-vit-large-patch14-336", "identity": { "family": "clip", "display_name": "OpenAI CLIP ViT-L/14 336", "hub_id": "openai/clip-vit-large-patch14-336", "api_name": null, "revision": null, "variants": [], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call.", "release_date": "2022-04", "paper_url": "https://arxiv.org/abs/2103.00020", "repo_url": "https://github.com/openai/CLIP" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "General zero-shot labels and text-to-image search; the reference baseline most papers report against.", "worst_at": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, "io": { "input": { "image": { "resolution": "336x336", "color": "RGB", "normalize": "CLIP mean (0.4815, 0.4578, 0.4082), std (0.2686, 0.2613, 0.2758)" }, "text": { "max_tokens": 77 }, "prompts": "'a photo of a {label}.'; BPE tokenizer, 77 tokens max" }, "output": { "vector": { "dims": 768, "distance": "cosine", "type": "dense" }, "note": "image and text vectors share one space; L2-normalize before cosine" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [ "feature_search", "mmr" ], "composes_with": [ "video frames (VideoFrameExpander) -> image vectors -> feature_search with a text query", "detector boxes (RT-DETR, D-FINE) -> crops -> image vectors per object", "image vectors + E5 text vectors on captions -> feature_search fusion (dbsf)" ] }, "run": { "status": "not_runnable", "status_reason": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel).", "serve_apps": [], "runtime": "transformers CLIPModel + CLIPProcessor", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 1712.0, "params_millions": 428.0, "preprocessing": "Resize shortest side and center-crop to 336x336, normalize with CLIP mean/std; text to 77 tokens.", "postprocessing": "L2-normalize image and text embeddings; logits = 100 * cosine for zero-shot.", "serving_shape": "both", "batch_size": 32, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 3.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://openai/clip-vit-large-patch14-336@ce19dc912ca5cd21c8a653c79e251e808ccabcd1" ], "formats": [ { "ref": "hf://openai/clip-vit-large-patch14-336@ce19dc912ca5cd21c8a653c79e251e808ccabcd1", "format": "pytorch_bin", "file": "pytorch_model.bin", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "hf://openai/clip-vit-large-patch14-336@ce19dc912ca5cd21c8a653c79e251e808ccabcd1", "format": "tensorflow", "file": "tf_model.h5", "loader": "TensorFlow; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "OpenAI CLIP ViT-L/14 336: embedding model", "description": "OpenAI CLIP ViT-L/14 336 is a image and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "OpenAI CLIP ViT-L/14 336", "clip", "embedding", "openai/clip-vit-large-patch14-336", "visual-search", "product-search" ], "faqs": [ { "q": "What is OpenAI CLIP ViT-L/14 336 best at?", "a": "General zero-shot labels and text-to-image search; the reference baseline most papers report against." }, { "q": "What are the limits of OpenAI CLIP ViT-L/14 336?", "a": "English only; weaker than SigLIP and LAION OpenCLIP at equal size on retrieval; reads text in images poorly." }, { "q": "Can OpenAI CLIP ViT-L/14 336 be used commercially?", "a": "Yes. MIT license on code and weights; the OpenAI model card says deployed use cases are out of scope and calls the release research-oriented, so treat production use as a judgment call." }, { "q": "How does OpenAI CLIP ViT-L/14 336 run on Mixpeek?", "a": "Weights on the hub are pickle only (pytorch_model.bin, TF, Flax); a plugin needs a one-time conversion to safetensors hosted in our bucket. Otherwise runs like the L/14 record (transformers CLIPModel)." }, { "q": "What does OpenAI CLIP ViT-L/14 336 output?", "a": "A 768-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/openai/clip-vit-large-patch14-336", "https://arxiv.org/abs/2103.00020", "https://github.com/openai/CLIP/blob/main/LICENSE", "https://github.com/openai/CLIP/blob/main/model-card.md" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "clip", "license": "MIT", "commercial_use": true, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2022-04", "params_millions": 428.0, "size_on_disk_mb": 1712.0, "vector_dims": 768, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_bin", "tensorflow" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 3.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "efficientsam-vits", "identity": { "family": "efficientsam", "display_name": "EfficientSAM ViT-S", "hub_id": "yformer/EfficientSAM (efficient_sam_vits.pt)", "api_name": null, "revision": null, "variants": [ "efficient_sam_vitt.pt (ViT-Tiny)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights and code; no use restrictions.", "release_date": "2023-12", "paper_url": "https://arxiv.org/abs/2312.00863", "repo_url": "https://github.com/yformer/EfficientSAM" }, "task": { "category": "segmentation", "modalities_in": [ "image" ], "modalities_out": [ "masks", "scores" ], "best_at": "SAM-style point and box prompted masks at a fraction of SAM ViT-H cost (masked-image pretraining distillation).", "worst_at": "Images only (no video memory); below SAM 2 tiny in quality and tooling." }, "io": { "input": { "image": { "resolution": "1024x1024" }, "prompts": "points and boxes" }, "output": { "masks": "binary masks with IoU predictions" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> EfficientSAM masks -> masked crops -> SigLIP" ] }, "run": { "status": "not_runnable", "status_reason": "Weights ship as .pt pickles (ViT-S zipped) and the model code lives in the GitHub repo; needs a safetensors conversion and a vendored model definition (or the repo's ONNX export) before it can be a plugin. SAM 2.1 tiny covers the same need with transformers support.", "serve_apps": [], "runtime": "repo build_efficient_sam_vits() / ONNX export", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Resize to 1024, normalize.", "postprocessing": "Threshold mask logits, pick by predicted IoU.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://yformer/EfficientSAM/weights/efficient_sam_vits.pt.zip" ], "formats": [ { "ref": "github://yformer/EfficientSAM/weights/efficient_sam_vits.pt.zip", "format": "pytorch_pt", "file": "efficient_sam_vits.pt.zip", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "EfficientSAM ViT-S: segmentation model", "description": "EfficientSAM ViT-S is a image segmentation model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "EfficientSAM ViT-S", "efficientsam", "segmentation", "yformer/EfficientSAM (efficient_sam_vits.pt)", "object-detection" ], "faqs": [ { "q": "What is EfficientSAM ViT-S best at?", "a": "SAM-style point and box prompted masks at a fraction of SAM ViT-H cost (masked-image pretraining distillation)." }, { "q": "What are the limits of EfficientSAM ViT-S?", "a": "Images only (no video memory); below SAM 2 tiny in quality and tooling." }, { "q": "Can EfficientSAM ViT-S be used commercially?", "a": "Yes. Apache-2.0 weights and code; no use restrictions." }, { "q": "How does EfficientSAM ViT-S run on Mixpeek?", "a": "Weights ship as .pt pickles (ViT-S zipped) and the model code lives in the GitHub repo; needs a safetensors conversion and a vendored model definition (or the repo's ONNX export) before it can be a plugin. SAM 2.1 tiny covers the same need with transformers support." } ] }, "last_verified": "2026-11-05", "sources": [ "https://github.com/yformer/EfficientSAM", "https://arxiv.org/abs/2312.00863" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "efficientsam", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "masks", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2023-12", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "internvideo2-5-chat-8b", "identity": { "family": "internvideo2.5", "display_name": "InternVideo2.5 Chat 8B", "hub_id": "OpenGVLab/InternVideo2_5_Chat_8B", "api_name": null, "revision": null, "variants": [ "OpenGVLab/InternVideo2-Chat-8B (MIT, 2024)" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the hub tag; the language model inside (InternLM/Qwen based InternVL) carries its own terms, check before shipping.", "release_date": "2025-01-22", "paper_url": "https://arxiv.org/abs/2501.12386", "repo_url": "https://github.com/OpenGVLab/InternVideo" }, "task": { "category": "video_qa", "modalities_in": [ "video", "text" ], "modalities_out": [ "text" ], "best_at": "Long-video question answering and fine-grained temporal grounding with an open model (long and rich context modeling).", "worst_at": "8.4B parameters and remote code: GPU only, slow per clip; answers need checking against transcripts." }, "io": { "input": { "video": { "frames": "up to hundreds of frames (dynamic sampling)", "resolution": "448x448 tiles" }, "prompts": "free-form question", "max_tokens": "LLM context per the card" }, "output": { "text": "free-form answer or JSON when prompted" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits -> InternVideo2.5 descriptions and tags -> E5 text vectors + BM25" ] }, "run": { "status": "not_runnable", "status_reason": "Requires trust_remote_code (custom_code on the hub) and a 24 GB class GPU. Would run as a GPU batch plugin after vendoring the InternVL chat code.", "serve_apps": [], "runtime": "transformers AutoModel with trust_remote_code=True (InternVL chat)", "weights_format": "safetensors", "trust_remote_code": true, "size_on_disk_mb": 16802.0, "params_millions": 8401.0, "preprocessing": "Sample frames, tile to 448, normalize; build the chat prompt with frame tokens.", "postprocessing": "Decode text; parse JSON when requested.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 32.0, "num_cpus": 8.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://OpenGVLab/InternVideo2_5_Chat_8B@87680fe232af8681b55322f68541679b91f99df1" ], "formats": [ { "ref": "hf://OpenGVLab/InternVideo2_5_Chat_8B@87680fe232af8681b55322f68541679b91f99df1", "format": "safetensors", "file": "4 files, e.g. model-00001-of-00004.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "video-understanding", "captioning" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "InternVideo2.5 Chat 8B: video understanding model", "description": "InternVideo2.5 Chat 8B is a video and text video understanding model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "InternVideo2.5 Chat 8B", "internvideo2.5", "video understanding", "OpenGVLab/InternVideo2_5_Chat_8B", "video-understanding", "captioning" ], "faqs": [ { "q": "What is InternVideo2.5 Chat 8B best at?", "a": "Long-video question answering and fine-grained temporal grounding with an open model (long and rich context modeling)." }, { "q": "What are the limits of InternVideo2.5 Chat 8B?", "a": "8.4B parameters and remote code: GPU only, slow per clip; answers need checking against transcripts." }, { "q": "Can InternVideo2.5 Chat 8B be used commercially?", "a": "Yes. Apache-2.0 per the hub tag; the language model inside (InternLM/Qwen based InternVL) carries its own terms, check before shipping." }, { "q": "How does InternVideo2.5 Chat 8B run on Mixpeek?", "a": "Requires trust_remote_code (custom_code on the hub) and a 24 GB class GPU. Would run as a GPU batch plugin after vendoring the InternVL chat code." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/OpenGVLab/InternVideo2_5_Chat_8B", "https://arxiv.org/abs/2501.12386", "https://github.com/OpenGVLab/InternVideo" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "video_qa", "family": "internvideo2.5", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "text" ], "use_cases": [ "video-understanding", "captioning" ], "recommended": false, "release_date": "2025-01-22", "params_millions": 8401.0, "size_on_disk_mb": 16802.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": true, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 32.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "internvideo2-stage2-1b-224p-f4", "identity": { "family": "internvideo2", "display_name": "InternVideo2 Stage2 1B (224p, 4 frames)", "hub_id": "OpenGVLab/InternVideo2-Stage2_1B-224p-f4", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the hub tag; access gated by an automatic agreement on the hub.", "release_date": "2024-03", "paper_url": "https://arxiv.org/abs/2403.15377", "repo_url": "https://github.com/OpenGVLab/InternVideo" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Video-text retrieval and zero-shot action recognition on short clips; motion-aware where frame CLIP models are not.", "worst_at": "Only 4 frames at 224p per clip: long or slow scenes need splitting; research code base, not a packaged loader." }, "io": { "input": { "video": { "frames": 4, "resolution": "224x224", "sampling": "uniform over the clip" }, "text": { "max_tokens": 40 } }, "output": { "vector": { "dims": "projection dim per the repo config (not verified here)", "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "not_runnable", "status_reason": "Weights are a single .pt pickle and the model is defined only in the InternVideo GitHub repo (multi_modality), so it needs vendored model code plus a safetensors conversion before it can be a plugin; GPU in practice for 1B.", "serve_apps": [], "runtime": "InternVideo repo (multi_modality InternVideo2 Stage2)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 2821.0, "params_millions": 1000.0, "preprocessing": "Sample 4 frames uniformly, resize to 224, normalize ImageNet mean/std.", "postprocessing": "L2-normalize video and text projections.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 8.0, "num_cpus": 2.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://OpenGVLab/InternVideo2-Stage2_1B-224p-f4@4362e1f88a992e7edbfd7696f7f78b7f79426dfd" ], "formats": [ { "ref": "hf://OpenGVLab/InternVideo2-Stage2_1B-224p-f4@4362e1f88a992e7edbfd7696f7f78b7f79426dfd", "format": "pytorch_pt", "file": "InternVideo2-stage2_1b-224p-f4.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "InternVideo2 Stage2 1B (224p, 4 frames): embedding model", "description": "InternVideo2 Stage2 1B (224p, 4 frames) is a video and text embedding model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "InternVideo2 Stage2 1B (224p, 4 frames)", "internvideo2", "embedding", "OpenGVLab/InternVideo2-Stage2_1B-224p-f4", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is InternVideo2 Stage2 1B (224p, 4 frames) best at?", "a": "Video-text retrieval and zero-shot action recognition on short clips; motion-aware where frame CLIP models are not." }, { "q": "What are the limits of InternVideo2 Stage2 1B (224p, 4 frames)?", "a": "Only 4 frames at 224p per clip: long or slow scenes need splitting; research code base, not a packaged loader." }, { "q": "Can InternVideo2 Stage2 1B (224p, 4 frames) be used commercially?", "a": "Yes. Apache-2.0 per the hub tag; access gated by an automatic agreement on the hub." }, { "q": "How does InternVideo2 Stage2 1B (224p, 4 frames) run on Mixpeek?", "a": "Weights are a single .pt pickle and the model is defined only in the InternVideo GitHub repo (multi_modality), so it needs vendored model code plus a safetensors conversion before it can be a plugin; GPU in practice for 1B." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/OpenGVLab/InternVideo2-Stage2_1B-224p-f4", "https://arxiv.org/abs/2403.15377", "https://github.com/OpenGVLab/InternVideo" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "internvideo2", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2024-03", "params_millions": 1000.0, "size_on_disk_mb": 2821.0, "vector_dims": null, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 8.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "internvideo2-stage2-6b", "identity": { "family": "internvideo2", "display_name": "InternVideo2 Stage2 6B", "hub_id": "OpenGVLab/InternVideo2-Stage2_6B", "api_name": null, "revision": null, "variants": [ "OpenGVLab/InternVideo2-Stage2_6B-224p-f4 (Apache-2.0, .pt with audio encoder)" ], "license": "MIT", "commercial_use": true, "commercial_use_note": "MIT per the hub tag.", "release_date": "2025-02", "paper_url": "https://arxiv.org/abs/2403.15377", "repo_url": "https://github.com/OpenGVLab/InternVideo" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "State-of-the-art video-text retrieval and action recognition among open video encoders (per the paper).", "worst_at": "6.4B parameters: multi-GPU-class memory; far too large for scale-to-zero CPU." }, "io": { "input": { "video": { "frames": "4 to 8 per clip", "resolution": "224x224" }, "text": {} }, "output": { "vector": { "dims": "not verified here", "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "not_runnable", "status_reason": "Requires trust_remote_code (custom_code on the hub) and a large GPU. Would run as a GPU batch plugin only after vendoring the model code.", "serve_apps": [], "runtime": "transformers AutoModel with trust_remote_code=True", "weights_format": "safetensors", "trust_remote_code": true, "size_on_disk_mb": 25000.0, "params_millions": 6367.0, "preprocessing": "Sample frames, resize to 224, normalize.", "postprocessing": "L2-normalize projections.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 40.0, "num_cpus": 8.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://OpenGVLab/InternVideo2-Stage2_6B@fd8ef758daf132d4738325d39546dd76ce26911d" ], "formats": [ { "ref": "hf://OpenGVLab/InternVideo2-Stage2_6B@fd8ef758daf132d4738325d39546dd76ce26911d", "format": "safetensors", "file": "13 files, e.g. model-00001-of-00013.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "InternVideo2 Stage2 6B: embedding model", "description": "InternVideo2 Stage2 6B is a video and text embedding model (MIT). Not runnable on Mixpeek today.", "keywords": [ "InternVideo2 Stage2 6B", "internvideo2", "embedding", "OpenGVLab/InternVideo2-Stage2_6B", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is InternVideo2 Stage2 6B best at?", "a": "State-of-the-art video-text retrieval and action recognition among open video encoders (per the paper)." }, { "q": "What are the limits of InternVideo2 Stage2 6B?", "a": "6.4B parameters: multi-GPU-class memory; far too large for scale-to-zero CPU." }, { "q": "Can InternVideo2 Stage2 6B be used commercially?", "a": "Yes. MIT per the hub tag." }, { "q": "How does InternVideo2 Stage2 6B run on Mixpeek?", "a": "Requires trust_remote_code (custom_code on the hub) and a large GPU. Would run as a GPU batch plugin only after vendoring the model code." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/OpenGVLab/InternVideo2-Stage2_6B", "https://arxiv.org/abs/2403.15377", "https://github.com/OpenGVLab/InternVideo" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "internvideo2", "license": "MIT", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2025-02", "params_millions": 6367.0, "size_on_disk_mb": 25000.0, "vector_dims": null, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": true, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 40.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "mobileclip", "identity": { "family": "mobileclip", "display_name": "MobileCLIP (S0, S1, S2, B)", "hub_id": "apple/MobileCLIP-S2", "api_name": null, "revision": null, "variants": [ "apple/MobileCLIP-S0", "apple/MobileCLIP-S1", "apple/MobileCLIP-B", "apple/MobileCLIP-S2-OpenCLIP", "apple/MobileCLIP-B-OpenCLIP" ], "license": "apple-amlr", "commercial_use": false, "commercial_use_note": "Apple Machine Learning Research Model license: use only for Research Purposes, which exclude commercial exploitation, product development or use in any commercial product or service.", "release_date": "2024-03", "paper_url": "https://arxiv.org/abs/2311.17049", "repo_url": "https://github.com/apple/ml-mobileclip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Very fast image-text embedding on CPU and phones (S0 is the smallest) at CLIP-level zero-shot accuracy.", "worst_at": "Cannot be used commercially; lower ceiling than SigLIP so400m." }, "io": { "input": { "image": { "resolution": "256x256", "color": "RGB" }, "text": { "max_tokens": 77 }, "prompts": "CLIP-style captions" }, "output": { "vector": { "dims": 512, "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "CPU frame embedding at scale to zero, if the license allowed it" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by license (research only). Technically it would run as a CPU plugin through open_clip (the -OpenCLIP repos ship open_clip_model.safetensors); the mobileclip repos ship .pt pickles.", "serve_apps": [], "runtime": "open_clip (MobileCLIP-*-OpenCLIP) or the mobileclip package", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 398.0, "params_millions": null, "preprocessing": "Resize and center-crop to 256; no mean/std normalization in the reference transform.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 64, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://apple/MobileCLIP-S2@6da8f9f1c7fa5ae89385ef86bfdee297d105e6f6" ], "formats": [ { "ref": "hf://apple/MobileCLIP-S2@6da8f9f1c7fa5ae89385ef86bfdee297d105e6f6", "format": "pytorch_pt", "file": "mobileclip_s2.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "MobileCLIP (S0, S1, S2, B): embedding model", "description": "MobileCLIP (S0, S1, S2, B) is a image and text embedding model (apple-amlr). Not runnable on Mixpeek today.", "keywords": [ "MobileCLIP (S0, S1, S2, B)", "mobileclip", "embedding", "apple/MobileCLIP-S2", "visual-search", "product-search" ], "faqs": [ { "q": "What is MobileCLIP (S0, S1, S2, B) best at?", "a": "Very fast image-text embedding on CPU and phones (S0 is the smallest) at CLIP-level zero-shot accuracy." }, { "q": "What are the limits of MobileCLIP (S0, S1, S2, B)?", "a": "Cannot be used commercially; lower ceiling than SigLIP so400m." }, { "q": "Can MobileCLIP (S0, S1, S2, B) be used commercially?", "a": "No. Apple Machine Learning Research Model license: use only for Research Purposes, which exclude commercial exploitation, product development or use in any commercial product or service." }, { "q": "How does MobileCLIP (S0, S1, S2, B) run on Mixpeek?", "a": "Blocked by license (research only). Technically it would run as a CPU plugin through open_clip (the -OpenCLIP repos ship open_clip_model.safetensors); the mobileclip repos ship .pt pickles." }, { "q": "What does MobileCLIP (S0, S1, S2, B) output?", "a": "A 512-dimensional dense vector, compared by cosine similarity." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/apple/MobileCLIP-S2", "https://huggingface.co/apple/MobileCLIP-S2-OpenCLIP", "https://huggingface.co/apple/MobileCLIP2-S2/blob/main/LICENSE", "https://arxiv.org/abs/2311.17049", "https://github.com/apple/ml-mobileclip" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "mobileclip", "license": "apple-amlr", "commercial_use": false, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2024-03", "params_millions": null, "size_on_disk_mb": 398.0, "vector_dims": 512, "vector_distance": "cosine", "weights_format": "safetensors", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "mobileclip2", "identity": { "family": "mobileclip", "display_name": "MobileCLIP2 (S0, S2, S4, B, L-14)", "hub_id": "apple/MobileCLIP2-S2", "api_name": null, "revision": null, "variants": [ "apple/MobileCLIP2-S0", "apple/MobileCLIP2-S4", "apple/MobileCLIP2-B", "apple/MobileCLIP2-L-14" ], "license": "apple-amlr", "commercial_use": false, "commercial_use_note": "Apple Machine Learning Research Model license: use only for Research Purposes, which exclude commercial exploitation, product development or use in any commercial product or service.", "release_date": "2025-08", "paper_url": "https://arxiv.org/abs/2508.20691", "repo_url": "https://github.com/apple/ml-mobileclip" }, "task": { "category": "embedding", "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "best_at": "Second-generation MobileCLIP: better zero-shot accuracy at the same mobile latency.", "worst_at": "Cannot be used commercially; weights are .pt pickles." }, "io": { "input": { "image": { "resolution": "256x256 (S variants)", "color": "RGB" }, "text": { "max_tokens": 77 } }, "output": { "vector": { "dims": "per variant (not verified here)", "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [] }, "run": { "status": "not_runnable", "status_reason": "Blocked by license (research only) and pickle .pt weights. Would run as a CPU plugin through the mobileclip or open_clip loader.", "serve_apps": [], "runtime": "mobileclip package / open_clip", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 398.0, "params_millions": null, "preprocessing": "Resize and center-crop to the variant resolution.", "postprocessing": "L2-normalize; cosine similarity.", "serving_shape": "both", "batch_size": 64, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://apple/MobileCLIP2-S2@72424e7025436db18f15c3eff6ee8c7c15ad4481" ], "formats": [ { "ref": "hf://apple/MobileCLIP2-S2@72424e7025436db18f15c3eff6ee8c7c15ad4481", "format": "pytorch_pt", "file": "mobileclip2_s2.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "visual-search", "product-search" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "MobileCLIP2 (S0, S2, S4, B, L-14): embedding model", "description": "MobileCLIP2 (S0, S2, S4, B, L-14) is a image and text embedding model (apple-amlr). Not runnable on Mixpeek today.", "keywords": [ "MobileCLIP2 (S0, S2, S4, B, L-14)", "mobileclip", "embedding", "apple/MobileCLIP2-S2", "visual-search", "product-search" ], "faqs": [ { "q": "What is MobileCLIP2 (S0, S2, S4, B, L-14) best at?", "a": "Second-generation MobileCLIP: better zero-shot accuracy at the same mobile latency." }, { "q": "What are the limits of MobileCLIP2 (S0, S2, S4, B, L-14)?", "a": "Cannot be used commercially; weights are .pt pickles." }, { "q": "Can MobileCLIP2 (S0, S2, S4, B, L-14) be used commercially?", "a": "No. Apple Machine Learning Research Model license: use only for Research Purposes, which exclude commercial exploitation, product development or use in any commercial product or service." }, { "q": "How does MobileCLIP2 (S0, S2, S4, B, L-14) run on Mixpeek?", "a": "Blocked by license (research only) and pickle .pt weights. Would run as a CPU plugin through the mobileclip or open_clip loader." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/apple/MobileCLIP2-S2", "https://huggingface.co/apple/MobileCLIP2-S2/blob/main/LICENSE", "https://arxiv.org/abs/2508.20691" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "mobileclip", "license": "apple-amlr", "commercial_use": false, "modalities_in": [ "image", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "visual-search", "product-search" ], "recommended": false, "release_date": "2025-08", "params_millions": null, "size_on_disk_mb": 398.0, "vector_dims": null, "vector_distance": "cosine", "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "mobilesam", "identity": { "family": "mobilesam", "display_name": "MobileSAM", "hub_id": "dhkim2810/MobileSAM (mobile_sam.pt)", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the GitHub repo (the hub mirror is tagged MIT); no use restrictions either way.", "release_date": "2023-06", "paper_url": "https://arxiv.org/abs/2306.14289", "repo_url": "https://github.com/ChaoningZhang/MobileSAM" }, "task": { "category": "segmentation", "modalities_in": [ "image" ], "modalities_out": [ "masks", "scores" ], "best_at": "SAM with a 5M-parameter TinyViT encoder: box or point prompted masks on CPU in tens of milliseconds.", "worst_at": "Mask quality below SAM 2; images only." }, "io": { "input": { "image": { "resolution": "1024x1024" }, "prompts": "points and boxes" }, "output": { "masks": "binary masks with IoU predictions" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> MobileSAM masks -> masked crops -> SigLIP" ] }, "run": { "status": "not_runnable", "status_reason": "Weights are a .pt pickle and the model code is the repo's (segment_anything fork). The repo ships an ONNX export script; once exported to ONNX it runs as a CPU plugin with onnxruntime.", "serve_apps": [], "runtime": "mobile_sam package (sam_model_registry['vit_t']) or onnxruntime after export", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 41.0, "params_millions": 10.0, "preprocessing": "Resize longest side to 1024, pad, normalize.", "postprocessing": "Threshold mask logits, pick by predicted IoU.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://dhkim2810/MobileSAM@4e2e5245fe9dc63096d575d36e70eb39b03b3aba", "github://ChaoningZhang/MobileSAM/weights/mobile_sam.pt" ], "formats": [ { "ref": "hf://dhkim2810/MobileSAM@4e2e5245fe9dc63096d575d36e70eb39b03b3aba", "format": "pytorch_pt", "file": "mobile_sam.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false }, { "ref": "github://ChaoningZhang/MobileSAM/weights/mobile_sam.pt", "format": "pytorch_pt", "file": "mobile_sam.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "MobileSAM: segmentation model", "description": "MobileSAM is a image segmentation model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "MobileSAM", "mobilesam", "segmentation", "dhkim2810/MobileSAM (mobile_sam.pt)", "object-detection" ], "faqs": [ { "q": "What is MobileSAM best at?", "a": "SAM with a 5M-parameter TinyViT encoder: box or point prompted masks on CPU in tens of milliseconds." }, { "q": "What are the limits of MobileSAM?", "a": "Mask quality below SAM 2; images only." }, { "q": "Can MobileSAM be used commercially?", "a": "Yes. Apache-2.0 per the GitHub repo (the hub mirror is tagged MIT); no use restrictions either way." }, { "q": "How does MobileSAM run on Mixpeek?", "a": "Weights are a .pt pickle and the model code is the repo's (segment_anything fork). The repo ships an ONNX export script; once exported to ONNX it runs as a CPU plugin with onnxruntime." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/dhkim2810/MobileSAM", "https://github.com/ChaoningZhang/MobileSAM", "https://arxiv.org/abs/2306.14289" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "mobilesam", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "image" ], "modalities_out": [ "masks", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2023-06", "params_millions": 10.0, "size_on_disk_mb": 41.0, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "github", "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "qwen2-5-7b-instruct", "identity": { "family": "qwen2.5", "display_name": "Qwen2.5 7B Instruct", "hub_id": "Qwen/Qwen2.5-7B-Instruct", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; no use restrictions.", "release_date": "2024-09", "paper_url": "https://arxiv.org/abs/2412.15115", "repo_url": "https://github.com/QwenLM/Qwen2.5" }, "task": { "category": "generation", "modalities_in": [ "text" ], "modalities_out": [ "text", "structured" ], "best_at": "Local structured extraction from scraped pages with no provider API calls (web_scraper llm_provider=vllm).", "worst_at": "Runs nowhere today (vllm is not in the engine dependencies); needs half an L4 and 6 GB per actor when it does; text only." }, "io": { "input": { "text": { "max_tokens": "8192 (vllm_max_model_len)" } }, "output": { "text": "JSON matching response_shape" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "web_scraper@v1", "selected_by": "llm_provider=vllm with llm_model=Qwen/Qwen2.5-7B-Instruct (LOCAL_LLM_MODELS)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "web_scraper crawl -> LLMExtractorProcessor on vLLM -> extracted_data" ] }, "run": { "status": "not_runnable", "status_reason": "web_scraper's vLLM path imports vllm (shared/models/llm/vllm.py), but vllm is not in server/pyproject.toml or poetry.lock, so the import fails and the step cannot run.", "serve_apps": [], "runtime": "vllm.LLM (VLLMService, trust_remote_code false, gpu_memory_utilization 0.85)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 15231.0, "params_millions": 7616.0, "preprocessing": "Page content and title into the extraction prompt.", "postprocessing": "Parse JSON against response_shape.", "serving_shape": "batch", "batch_size": 12, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": 6.0, "num_cpus": 0.5, "num_gpus": 0.5, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "Never run in this codebase's image; nothing measured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Qwen/Qwen2.5-7B-Instruct@a09a35458c702b33eeacc393d103063234e8bc28" ], "formats": [ { "ref": "hf://Qwen/Qwen2.5-7B-Instruct@a09a35458c702b33eeacc393d103063234e8bc28", "format": "safetensors", "file": "4 files, e.g. model-00001-of-00004.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "tagging", "web-pages" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Qwen2.5 7B Instruct: language model", "description": "Qwen2.5 7B Instruct is a text language model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "Qwen2.5 7B Instruct", "qwen2.5", "language", "Qwen/Qwen2.5-7B-Instruct", "tagging", "web-pages" ], "faqs": [ { "q": "What is Qwen2.5 7B Instruct best at?", "a": "Local structured extraction from scraped pages with no provider API calls (web_scraper llm_provider=vllm)." }, { "q": "What are the limits of Qwen2.5 7B Instruct?", "a": "Runs nowhere today (vllm is not in the engine dependencies); needs half an L4 and 6 GB per actor when it does; text only." }, { "q": "Can Qwen2.5 7B Instruct be used commercially?", "a": "Yes. Apache-2.0 weights; no use restrictions." }, { "q": "How does Qwen2.5 7B Instruct run on Mixpeek?", "a": "web_scraper's vLLM path imports vllm (shared/models/llm/vllm.py), but vllm is not in server/pyproject.toml or poetry.lock, so the import fails and the step cannot run." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", "https://huggingface.co/api/models/Qwen/Qwen2.5-7B-Instruct", "https://arxiv.org/abs/2412.15115", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/web_scraper/v1/pipeline.py", "server/engine/extractors/web_scraper/v1/processors/llm_extractor.py", "server/shared/models/llm/vllm.py", "server/pyproject.toml" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "generation", "family": "qwen2.5", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "tagging", "web-pages" ], "recommended": false, "release_date": "2024-09", "params_millions": 7616.0, "size_on_disk_mb": 15231.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 0.5, "memory_gb": 6.0, "min_replicas": null, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "qwen2-5-vl-7b-instruct", "identity": { "family": "qwen2.5-vl", "display_name": "Qwen2.5-VL 7B Instruct", "hub_id": "Qwen/Qwen2.5-VL-7B-Instruct", "api_name": null, "revision": null, "variants": [], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 weights; no use restrictions.", "release_date": "2025-01", "paper_url": "https://arxiv.org/abs/2502.13923", "repo_url": "https://github.com/QwenLM/Qwen2.5-VL" }, "task": { "category": "generation", "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "best_at": "Local structured extraction from scraped pages with no provider API calls (web_scraper llm_provider=vllm).", "worst_at": "Runs nowhere today (vllm is not in the engine dependencies); needs half an L4 and 6 GB per actor when it does; the extractor passes page text only, so the vision side is unused." }, "io": { "input": { "text": { "max_tokens": "8192 (vllm_max_model_len)" } }, "output": { "text": "JSON matching response_shape" } }, "mixpeek_wiring": { "extractors": [ { "extractor": "web_scraper@v1", "selected_by": "llm_provider=vllm with llm_model=Qwen/Qwen2.5-VL-7B-Instruct (LOCAL_LLM_MODELS)" } ], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "web_scraper crawl -> LLMExtractorProcessor on vLLM -> extracted_data" ] }, "run": { "status": "not_runnable", "status_reason": "web_scraper's vLLM path imports vllm (shared/models/llm/vllm.py), but vllm is not in server/pyproject.toml or poetry.lock, so the import fails and the step cannot run.", "serve_apps": [], "runtime": "vllm.LLM (VLLMService, trust_remote_code false, gpu_memory_utilization 0.85)", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 16584.0, "params_millions": 8292.0, "preprocessing": "Page content and title into the extraction prompt.", "postprocessing": "Parse JSON against response_shape.", "serving_shape": "batch", "batch_size": 12, "serve_batch": null, "scale_to_zero": { "min_replicas": null, "max_replicas": null, "idle_timeout_s": null, "memory_gb": 6.0, "num_cpus": 0.5, "num_gpus": 0.5, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "Never run in this codebase's image; nothing measured." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Qwen/Qwen2.5-VL-7B-Instruct@cc594898137f460bfe9f0759e9844b3ce807cfb5" ], "formats": [ { "ref": "hf://Qwen/Qwen2.5-VL-7B-Instruct@cc594898137f460bfe9f0759e9844b3ce807cfb5", "format": "safetensors", "file": "5 files, e.g. model-00001-of-00005.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true } ], "use_cases": [ "captioning", "ocr", "tagging" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Qwen2.5-VL 7B Instruct: language model", "description": "Qwen2.5-VL 7B Instruct is a text and image language model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "Qwen2.5-VL 7B Instruct", "qwen2.5-vl", "language", "Qwen/Qwen2.5-VL-7B-Instruct", "captioning", "ocr", "tagging" ], "faqs": [ { "q": "What is Qwen2.5-VL 7B Instruct best at?", "a": "Local structured extraction from scraped pages with no provider API calls (web_scraper llm_provider=vllm)." }, { "q": "What are the limits of Qwen2.5-VL 7B Instruct?", "a": "Runs nowhere today (vllm is not in the engine dependencies); needs half an L4 and 6 GB per actor when it does; the extractor passes page text only, so the vision side is unused." }, { "q": "Can Qwen2.5-VL 7B Instruct be used commercially?", "a": "Yes. Apache-2.0 weights; no use restrictions." }, { "q": "How does Qwen2.5-VL 7B Instruct run on Mixpeek?", "a": "web_scraper's vLLM path imports vllm (shared/models/llm/vllm.py), but vllm is not in server/pyproject.toml or poetry.lock, so the import fails and the step cannot run." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct", "https://huggingface.co/api/models/Qwen/Qwen2.5-VL-7B-Instruct", "https://arxiv.org/abs/2502.13923", "server/engine/extractors/web_scraper/v1/definition.py", "server/engine/extractors/web_scraper/v1/pipeline.py", "server/engine/extractors/web_scraper/v1/processors/llm_extractor.py", "server/shared/models/llm/vllm.py", "server/pyproject.toml" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "generation", "family": "qwen2.5-vl", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "text", "image" ], "modalities_out": [ "text", "structured" ], "use_cases": [ "captioning", "ocr", "tagging" ], "recommended": false, "release_date": "2025-01", "params_millions": 8292.0, "size_on_disk_mb": 16584.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 0.5, "memory_gb": 6.0, "min_replicas": null, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "sam3", "identity": { "family": "sam3", "display_name": "SAM 3 (Segment Anything with Concepts)", "hub_id": "facebook/sam3", "api_name": null, "revision": null, "variants": [], "license": "SAM License (Meta, 2025-11-19)", "commercial_use": true, "commercial_use_note": "Custom Meta license: royalty-free commercial use allowed, with carve-outs (military, nuclear, espionage, sanctioned parties, ITAR); redistribution must carry the same terms; Meta may change terms. Hub access is gated by manual approval.", "release_date": "2025-11-19", "paper_url": "https://ai.meta.com/research/publications/sam-3-segment-anything-with-concepts/", "repo_url": "https://github.com/facebookresearch/sam3" }, "task": { "category": "segmentation", "modalities_in": [ "image", "video", "text" ], "modalities_out": [ "masks", "boxes", "scores", "tracks" ], "best_at": "Segments and tracks every instance of a concept named by a short noun phrase or an exemplar image ('yellow school bus'), across images and video.", "worst_at": "About 0.86B parameters: GPU in practice; long or compositional phrases; access gated and approvals reported as rejected." }, "io": { "input": { "image": { "resolution": "1008x1008" }, "video": { "frames": "streamed with a tracker memory" }, "prompts": "short noun phrase, image exemplars (boxes), or SAM 2 style points/boxes" }, "output": { "masks": "instance masks for every match", "boxes": "xyxy", "scores": "presence and instance scores", "tracks": "instance ids across frames" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "taxonomy labels -> SAM 3 concept prompts -> instance masks -> masked crops -> SigLIP", "SAM 3 instance counts per frame -> payload fields -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by gated hub access (manual approval, not guaranteed) and GPU-only practicality. Weights are safetensors with no remote code, so once access is granted it runs as a GPU plugin through transformers Sam3Model.", "serve_apps": [], "runtime": "transformers Sam3Model + Sam3Processor", "weights_format": "safetensors", "trust_remote_code": false, "size_on_disk_mb": 3440.0, "params_millions": 860.0, "preprocessing": "Resize to 1008, normalize; tokenize the concept phrase.", "postprocessing": "Score threshold on presence and instance scores, mask upsampling.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 12.0, "num_cpus": 4.0, "num_gpus": 1.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": false } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://facebook/sam3@3c879f39826c281e95690f02c7821c4de09afae7" ], "formats": [ { "ref": "hf://facebook/sam3@3c879f39826c281e95690f02c7821c4de09afae7", "format": "safetensors", "file": "model.safetensors", "loader": "transformers / open_clip from_pretrained; Mixpeek Hub import (api/models/huggingface.py)", "supported_today": true }, { "ref": "hf://facebook/sam3@3c879f39826c281e95690f02c7821c4de09afae7", "format": "pytorch_pt", "file": "sam3.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "tracking", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "SAM 3 (Segment Anything with Concepts): segmentation model", "description": "SAM 3 (Segment Anything with Concepts) is a image, video and text segmentation model (SAM License (Meta, 2025-11-19)). Not runnable on Mixpeek today.", "keywords": [ "SAM 3 (Segment Anything with Concepts)", "sam3", "segmentation", "facebook/sam3", "tracking", "object-detection" ], "faqs": [ { "q": "What is SAM 3 (Segment Anything with Concepts) best at?", "a": "Segments and tracks every instance of a concept named by a short noun phrase or an exemplar image ('yellow school bus'), across images and video." }, { "q": "What are the limits of SAM 3 (Segment Anything with Concepts)?", "a": "About 0.86B parameters: GPU in practice; long or compositional phrases; access gated and approvals reported as rejected." }, { "q": "Can SAM 3 (Segment Anything with Concepts) be used commercially?", "a": "Yes. Custom Meta license: royalty-free commercial use allowed, with carve-outs (military, nuclear, espionage, sanctioned parties, ITAR); redistribution must carry the same terms; Meta may change terms. Hub access is gated by manual approval." }, { "q": "How does SAM 3 (Segment Anything with Concepts) run on Mixpeek?", "a": "Blocked by gated hub access (manual approval, not guaranteed) and GPU-only practicality. Weights are safetensors with no remote code, so once access is granted it runs as a GPU plugin through transformers Sam3Model." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/facebook/sam3", "https://huggingface.co/facebook/sam3/blob/main/LICENSE", "https://github.com/facebookresearch/sam3", "https://roboflow.com/model-licenses/segment-anything-3", "https://scancode-licensedb.aboutcode.org/sam-2025-11-19.html" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "sam3", "license": "SAM License (Meta, 2025-11-19)", "commercial_use": true, "modalities_in": [ "image", "video", "text" ], "modalities_out": [ "masks", "boxes", "scores", "tracks" ], "use_cases": [ "segmentation", "tracking", "object-detection" ], "recommended": false, "release_date": "2025-11-19", "params_millions": 860.0, "size_on_disk_mb": 3440.0, "vector_dims": null, "vector_distance": null, "weights_format": "safetensors", "formats": [ "pytorch_pt", "safetensors" ], "loadable_today": true, "trust_remote_code": false, "cpu_fallback": false, "num_gpus": 1.0, "memory_gb": 12.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "videoprism-base-f16r288", "identity": { "family": "videoprism", "display_name": "VideoPrism Base (16 frames, 288px)", "hub_id": "google/videoprism-base-f16r288", "api_name": null, "revision": null, "variants": [ "google/videoprism-large-f8r288" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the hub tag and repo.", "release_date": "2025-06", "paper_url": "https://arxiv.org/abs/2402.13217", "repo_url": "https://github.com/google-deepmind/videoprism" }, "task": { "category": "embedding", "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "best_at": "General-purpose frozen video encoder: strong features for classification, localization and retrieval probes.", "worst_at": "JAX/Flax only; no text tower, so no text queries: video-to-video similarity or trained probes only." }, "io": { "input": { "video": { "frames": 16, "resolution": "288x288" } }, "output": { "vector": { "dims": "per model config (not verified here)", "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits -> VideoPrism clip vectors -> feature_search by example clip" ] }, "run": { "status": "not_runnable", "status_reason": "Weights are Flax .npz arrays loaded by the videoprism JAX package; Mixpeek's runtime is PyTorch, so it needs JAX in the image or a port. Weights-only arrays, Apache-2.0, so no license block.", "serve_apps": [], "runtime": "videoprism (JAX/Flax)", "weights_format": "jax", "trust_remote_code": false, "size_on_disk_mb": 458.0, "params_millions": null, "preprocessing": "Sample 16 frames, resize to 288, scale to 0..1.", "postprocessing": "Global pooled embedding; L2-normalize for retrieval.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/videoprism-base-f16r288@be719a406d563b66f0ac969e7c94bab8e997c81a" ], "formats": [ { "ref": "hf://google/videoprism-base-f16r288@be719a406d563b66f0ac969e7c94bab8e997c81a", "format": "jax", "file": "flax_base_f16r288_repeated.npz", "loader": "JAX/Flax; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "VideoPrism Base (16 frames, 288px): embedding model", "description": "VideoPrism Base (16 frames, 288px) is a video embedding model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "VideoPrism Base (16 frames, 288px)", "videoprism", "embedding", "google/videoprism-base-f16r288", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is VideoPrism Base (16 frames, 288px) best at?", "a": "General-purpose frozen video encoder: strong features for classification, localization and retrieval probes." }, { "q": "What are the limits of VideoPrism Base (16 frames, 288px)?", "a": "JAX/Flax only; no text tower, so no text queries: video-to-video similarity or trained probes only." }, { "q": "Can VideoPrism Base (16 frames, 288px) be used commercially?", "a": "Yes. Apache-2.0 per the hub tag and repo." }, { "q": "How does VideoPrism Base (16 frames, 288px) run on Mixpeek?", "a": "Weights are Flax .npz arrays loaded by the videoprism JAX package; Mixpeek's runtime is PyTorch, so it needs JAX in the image or a port. Weights-only arrays, Apache-2.0, so no license block." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/videoprism-base-f16r288", "https://arxiv.org/abs/2402.13217", "https://github.com/google-deepmind/videoprism" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "videoprism", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "video" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2025-06", "params_millions": null, "size_on_disk_mb": 458.0, "vector_dims": null, "vector_distance": "cosine", "weights_format": "jax", "formats": [ "jax" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "videoprism-lvt-base-f16r288", "identity": { "family": "videoprism", "display_name": "VideoPrism LvT Base (video-text, 16 frames, 288px)", "hub_id": "google/videoprism-lvt-base-f16r288", "api_name": null, "revision": null, "variants": [ "google/videoprism-lvt-large-f8r288" ], "license": "Apache-2.0", "commercial_use": true, "commercial_use_note": "Apache-2.0 per the hub tag and repo.", "release_date": "2025-07", "paper_url": "https://arxiv.org/abs/2402.13217", "repo_url": "https://github.com/google-deepmind/videoprism" }, "task": { "category": "embedding", "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "best_at": "Video-text retrieval and zero-shot classification with a paired text encoder.", "worst_at": "JAX/Flax only; text tower is English." }, "io": { "input": { "video": { "frames": 16, "resolution": "288x288" }, "text": {} }, "output": { "vector": { "dims": "per model config (not verified here)", "distance": "cosine", "type": "dense" } } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "scene splits (transcoding/multimodal extractor) -> clip-level vectors -> feature_search with a text query" ] }, "run": { "status": "not_runnable", "status_reason": "Weights are Flax .npz arrays loaded by the videoprism JAX package; Mixpeek's runtime is PyTorch, so it needs JAX in the image or a port. Weights-only arrays, Apache-2.0, so no license block.", "serve_apps": [], "runtime": "videoprism (JAX/Flax)", "weights_format": "jax", "trust_remote_code": false, "size_on_disk_mb": 991.0, "params_millions": null, "preprocessing": "Sample 16 frames, resize to 288, scale to 0..1.", "postprocessing": "Global pooled embedding; L2-normalize for retrieval.", "serving_shape": "batch", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 6.0, "num_cpus": 2.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://google/videoprism-lvt-base-f16r288@0119cca7d1fe61aa1d5718e1f71605703e1562e7" ], "formats": [ { "ref": "hf://google/videoprism-lvt-base-f16r288@0119cca7d1fe61aa1d5718e1f71605703e1562e7", "format": "jax", "file": "flax_lvt_base_f16r288_repeated.npz", "loader": "JAX/Flax; not in Mixpeek's runtime", "supported_today": false } ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "VideoPrism LvT Base (video-text, 16 frames, 288px): embedding model", "description": "VideoPrism LvT Base (video-text, 16 frames, 288px) is a video and text embedding model (Apache-2.0). Not runnable on Mixpeek today.", "keywords": [ "VideoPrism LvT Base (video-text, 16 frames, 288px)", "videoprism", "embedding", "google/videoprism-lvt-base-f16r288", "video-understanding", "action-recognition" ], "faqs": [ { "q": "What is VideoPrism LvT Base (video-text, 16 frames, 288px) best at?", "a": "Video-text retrieval and zero-shot classification with a paired text encoder." }, { "q": "What are the limits of VideoPrism LvT Base (video-text, 16 frames, 288px)?", "a": "JAX/Flax only; text tower is English." }, { "q": "Can VideoPrism LvT Base (video-text, 16 frames, 288px) be used commercially?", "a": "Yes. Apache-2.0 per the hub tag and repo." }, { "q": "How does VideoPrism LvT Base (video-text, 16 frames, 288px) run on Mixpeek?", "a": "Weights are Flax .npz arrays loaded by the videoprism JAX package; Mixpeek's runtime is PyTorch, so it needs JAX in the image or a port. Weights-only arrays, Apache-2.0, so no license block." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/google/videoprism-lvt-base-f16r288", "https://arxiv.org/abs/2402.13217", "https://github.com/google-deepmind/videoprism" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "embedding", "family": "videoprism", "license": "Apache-2.0", "commercial_use": true, "modalities_in": [ "video", "text" ], "modalities_out": [ "vector" ], "use_cases": [ "video-understanding", "action-recognition" ], "recommended": false, "release_date": "2025-07", "params_millions": null, "size_on_disk_mb": 991.0, "vector_dims": null, "vector_distance": "cosine", "weights_format": "jax", "formats": [ "jax" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 6.0, "min_replicas": 0.0, "serving_shape": "batch", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo-world-v2", "identity": { "family": "yolo-world", "display_name": "YOLO-World v2 (Ultralytics)", "hub_id": "yolov8s-worldv2.pt", "api_name": null, "revision": null, "variants": [ "yolov8m-worldv2.pt", "yolov8l-worldv2.pt", "yolov8x-worldv2.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license. The original Tencent AILab repo is GPL-3.0.", "release_date": "2024-01-30", "paper_url": "https://arxiv.org/abs/2401.17270", "repo_url": "https://github.com/AILab-CVC/YOLO-World" }, "task": { "category": "detection", "modalities_in": [ "image", "text" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Open-vocabulary detection from a list of text class names, in real time; set classes per namespace without training.", "worst_at": "Rare or compositional phrases; accuracy below closed-set YOLO on COCO classes; license blocks closed commercial use." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed" }, "prompts": "list of class names (CLIP text encoder), set once with set_classes" }, "output": { "boxes": "xyxy pixels", "labels": "open vocabulary (the given class list)", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "taxonomy labels -> YOLO-World class list -> boxes -> crops -> SigLIP", "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops. The class list is baked in at export.", "serve_apps": [], "runtime": "ultralytics YOLOWorld", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://ultralytics/assets/yolov8s-worldv2.pt" ], "formats": [ { "ref": "github://ultralytics/assets/yolov8s-worldv2.pt", "format": "pytorch_pt", "file": "yolov8s-worldv2.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "object-detection", "brand-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "YOLO-World v2 (Ultralytics): object detection model", "description": "YOLO-World v2 (Ultralytics) is a image and text object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "YOLO-World v2 (Ultralytics)", "yolo-world", "object detection", "yolov8s-worldv2.pt", "object-detection", "brand-detection" ], "faqs": [ { "q": "What is YOLO-World v2 (Ultralytics) best at?", "a": "Open-vocabulary detection from a list of text class names, in real time; set classes per namespace without training." }, { "q": "What are the limits of YOLO-World v2 (Ultralytics)?", "a": "Rare or compositional phrases; accuracy below closed-set YOLO on COCO classes; license blocks closed commercial use." }, { "q": "Can YOLO-World v2 (Ultralytics) be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license. The original Tencent AILab repo is GPL-3.0." }, { "q": "How does YOLO-World v2 (Ultralytics) run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops. The class list is baked in at export." } ] }, "last_verified": "2026-11-05", "sources": [ "https://docs.ultralytics.com/models/yolo-world/", "https://arxiv.org/abs/2401.17270", "https://github.com/AILab-CVC/YOLO-World" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolo-world", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image", "text" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection", "brand-detection" ], "recommended": false, "release_date": "2024-01-30", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo11-classify", "identity": { "family": "yolo11", "display_name": "Ultralytics YOLO11 classify", "hub_id": "Ultralytics/YOLO11 (yolo11n-cls.pt)", "api_name": null, "revision": null, "variants": [ "yolo11s-cls.pt", "yolo11m-cls.pt", "yolo11l-cls.pt", "yolo11x-cls.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2024-09-30", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "classification", "modalities_in": [ "image" ], "modalities_out": [ "labels", "scores" ], "best_at": "Fast classify for the fixed ImageNet label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "labels": "ImageNet-1k (1000 classes), top-5" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "frame labels -> payload field -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": 1.6, "preprocessing": "Resize and center-crop to 224.", "postprocessing": "Softmax over 1000 classes; top-k.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab" ], "formats": [ { "ref": "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab", "format": "pytorch_pt", "file": "15 files, e.g. yolo11l-pose.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "classification", "tagging" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO11 classify: classification model", "description": "Ultralytics YOLO11 classify is a image classification model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO11 classify", "yolo11", "classification", "Ultralytics/YOLO11 (yolo11n-cls.pt)", "tagging" ], "faqs": [ { "q": "What is Ultralytics YOLO11 classify best at?", "a": "Fast classify for the fixed ImageNet label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO11 classify?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO11 classify be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO11 classify run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO11", "https://docs.ultralytics.com/models/yolo11/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "classification", "family": "yolo11", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "labels", "scores" ], "use_cases": [ "classification", "tagging" ], "recommended": false, "release_date": "2024-09-30", "params_millions": 1.6, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo11-detect", "identity": { "family": "yolo11", "display_name": "Ultralytics YOLO11 detect", "hub_id": "Ultralytics/YOLO11 (yolo11n.pt)", "api_name": null, "revision": null, "variants": [ "yolo11s.pt", "yolo11m.pt", "yolo11l.pt", "yolo11x.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2024-09-30", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Fast detect for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 5.6, "params_millions": 2.6, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab" ], "formats": [ { "ref": "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab", "format": "pytorch_pt", "file": "15 files, e.g. yolo11l-pose.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO11 detect: object detection model", "description": "Ultralytics YOLO11 detect is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO11 detect", "yolo11", "object detection", "Ultralytics/YOLO11 (yolo11n.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO11 detect best at?", "a": "Fast detect for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO11 detect?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO11 detect be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO11 detect run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO11", "https://docs.ultralytics.com/models/yolo11/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolo11", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2024-09-30", "params_millions": 2.6, "size_on_disk_mb": 5.6, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo11-pose", "identity": { "family": "yolo11", "display_name": "Ultralytics YOLO11 pose", "hub_id": "Ultralytics/YOLO11 (yolo11n-pose.pt)", "api_name": null, "revision": null, "variants": [ "yolo11s-pose.pt", "yolo11m-pose.pt", "yolo11l-pose.pt", "yolo11x-pose.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2024-09-30", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "best_at": "Fast pose for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "person xyxy", "keypoints": "COCO 17 keypoints (x, y, visibility)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "person boxes + keypoints -> pose payload per frame -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 6.3, "params_millions": 2.9, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab" ], "formats": [ { "ref": "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab", "format": "pytorch_pt", "file": "15 files, e.g. yolo11l-pose.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "pose", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO11 pose: object detection model", "description": "Ultralytics YOLO11 pose is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO11 pose", "yolo11", "object detection", "Ultralytics/YOLO11 (yolo11n-pose.pt)", "pose", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO11 pose best at?", "a": "Fast pose for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO11 pose?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO11 pose be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO11 pose run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO11", "https://docs.ultralytics.com/models/yolo11/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolo11", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "use_cases": [ "pose", "object-detection" ], "recommended": false, "release_date": "2024-09-30", "params_millions": 2.9, "size_on_disk_mb": 6.3, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo11-segment", "identity": { "family": "yolo11", "display_name": "Ultralytics YOLO11 segment", "hub_id": "Ultralytics/YOLO11 (yolo11n-seg.pt)", "api_name": null, "revision": null, "variants": [ "yolo11s-seg.pt", "yolo11m-seg.pt", "yolo11l-seg.pt", "yolo11x-seg.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2024-09-30", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "segmentation", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "best_at": "Fast segment for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "masks": "per-instance bitmaps (prototype masks x coefficients) or polygons", "labels": "COCO 80 classes" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 6.2, "params_millions": 2.9, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS, mask decode from prototypes", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab" ], "formats": [ { "ref": "hf://Ultralytics/YOLO11@8b8ac7d1fae7468f85dbf89670dd66f41f485aab", "format": "pytorch_pt", "file": "15 files, e.g. yolo11l-pose.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO11 segment: segmentation model", "description": "Ultralytics YOLO11 segment is a image segmentation model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO11 segment", "yolo11", "segmentation", "Ultralytics/YOLO11 (yolo11n-seg.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO11 segment best at?", "a": "Fast segment for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO11 segment?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO11 segment be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO11 segment run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO11", "https://docs.ultralytics.com/models/yolo11/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "yolo11", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2024-09-30", "params_millions": 2.9, "size_on_disk_mb": 6.2, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo26-classify", "identity": { "family": "yolo26", "display_name": "Ultralytics YOLO26 classify", "hub_id": "Ultralytics/YOLO26 (yolo26n-cls.pt)", "api_name": null, "revision": null, "variants": [ "yolo26s-cls.pt", "yolo26m-cls.pt", "yolo26l-cls.pt", "yolo26x-cls.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2026-01-14", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "classification", "modalities_in": [ "image" ], "modalities_out": [ "labels", "scores" ], "best_at": "Fast classify for the fixed ImageNet label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "224x224", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "labels": "ImageNet-1k (1000 classes), top-5" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "frame labels -> payload field -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 5.8, "params_millions": null, "preprocessing": "Resize and center-crop to 224.", "postprocessing": "Softmax over 1000 classes; top-k.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d" ], "formats": [ { "ref": "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d", "format": "pytorch_pt", "file": "25 files, e.g. yolo26l-cls.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "classification", "tagging" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO26 classify: classification model", "description": "Ultralytics YOLO26 classify is a image classification model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO26 classify", "yolo26", "classification", "Ultralytics/YOLO26 (yolo26n-cls.pt)", "tagging" ], "faqs": [ { "q": "What is Ultralytics YOLO26 classify best at?", "a": "Fast classify for the fixed ImageNet label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO26 classify?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO26 classify be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO26 classify run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO26", "https://docs.ultralytics.com/models/yolo26/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "classification", "family": "yolo26", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "labels", "scores" ], "use_cases": [ "classification", "tagging" ], "recommended": false, "release_date": "2026-01-14", "params_millions": null, "size_on_disk_mb": 5.8, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo26-detect", "identity": { "family": "yolo26", "display_name": "Ultralytics YOLO26 detect", "hub_id": "Ultralytics/YOLO26 (yolo26n.pt)", "api_name": null, "revision": null, "variants": [ "yolo26s.pt", "yolo26m.pt", "yolo26l.pt", "yolo26x.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2026-01-14", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Fast detect for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 5.5, "params_millions": null, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then end-to-end, NMS-free", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d" ], "formats": [ { "ref": "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d", "format": "pytorch_pt", "file": "25 files, e.g. yolo26l-cls.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO26 detect: object detection model", "description": "Ultralytics YOLO26 detect is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO26 detect", "yolo26", "object detection", "Ultralytics/YOLO26 (yolo26n.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO26 detect best at?", "a": "Fast detect for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO26 detect?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO26 detect be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO26 detect run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO26", "https://docs.ultralytics.com/models/yolo26/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolo26", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2026-01-14", "params_millions": null, "size_on_disk_mb": 5.5, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo26-pose", "identity": { "family": "yolo26", "display_name": "Ultralytics YOLO26 pose", "hub_id": "Ultralytics/YOLO26 (yolo26n-pose.pt)", "api_name": null, "revision": null, "variants": [ "yolo26s-pose.pt", "yolo26m-pose.pt", "yolo26l-pose.pt", "yolo26x-pose.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2026-01-14", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "best_at": "Fast pose for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "person xyxy", "keypoints": "COCO 17 keypoints (x, y, visibility)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "person boxes + keypoints -> pose payload per frame -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 7.9, "params_millions": null, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then end-to-end, NMS-free", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d" ], "formats": [ { "ref": "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d", "format": "pytorch_pt", "file": "25 files, e.g. yolo26l-cls.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "pose", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO26 pose: object detection model", "description": "Ultralytics YOLO26 pose is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO26 pose", "yolo26", "object detection", "Ultralytics/YOLO26 (yolo26n-pose.pt)", "pose", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO26 pose best at?", "a": "Fast pose for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO26 pose?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO26 pose be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO26 pose run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO26", "https://docs.ultralytics.com/models/yolo26/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolo26", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "use_cases": [ "pose", "object-detection" ], "recommended": false, "release_date": "2026-01-14", "params_millions": null, "size_on_disk_mb": 7.9, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolo26-segment", "identity": { "family": "yolo26", "display_name": "Ultralytics YOLO26 segment", "hub_id": "Ultralytics/YOLO26 (yolo26n-seg.pt)", "api_name": null, "revision": null, "variants": [ "yolo26s-seg.pt", "yolo26m-seg.pt", "yolo26l-seg.pt", "yolo26x-seg.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2026-01-14", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "segmentation", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "best_at": "Fast segment for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "masks": "per-instance bitmaps (prototype masks x coefficients) or polygons", "labels": "COCO 80 classes" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 6.7, "params_millions": null, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then end-to-end, NMS-free, mask decode from prototypes", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d" ], "formats": [ { "ref": "hf://Ultralytics/YOLO26@070ac3c51435984d992ba92eb3f234daf4e4500d", "format": "pytorch_pt", "file": "25 files, e.g. yolo26l-cls.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLO26 segment: segmentation model", "description": "Ultralytics YOLO26 segment is a image segmentation model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLO26 segment", "yolo26", "segmentation", "Ultralytics/YOLO26 (yolo26n-seg.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLO26 segment best at?", "a": "Fast segment for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLO26 segment?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLO26 segment be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLO26 segment run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLO26", "https://docs.ultralytics.com/models/yolo26/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "yolo26", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2026-01-14", "params_millions": null, "size_on_disk_mb": 6.7, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yoloe-11-segment", "identity": { "family": "yoloe", "display_name": "YOLOE-11 (text, visual and prompt-free)", "hub_id": "yoloe-11s-seg.pt", "api_name": null, "revision": null, "variants": [ "yoloe-11m-seg.pt", "yoloe-11l-seg.pt", "yoloe-11s-seg-pf.pt (prompt-free)", "yoloe-v8s-seg.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2025-03", "paper_url": "https://arxiv.org/abs/2503.07465", "repo_url": "https://github.com/THU-MIG/yoloe" }, "task": { "category": "segmentation", "modalities_in": [ "image", "text" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "best_at": "Open-vocabulary detection and segmentation from text, an example image, or no prompt (prompt-free variant names objects from a built-in vocabulary).", "worst_at": "License blocks closed commercial use; masks coarser than SAM." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed" }, "prompts": "text class names, visual prompt (boxes on a reference image), or none (-pf)" }, "output": { "boxes": "xyxy pixels", "masks": "per-instance bitmaps", "labels": "open vocabulary", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "masks -> masked crops -> SigLIP vectors", "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLOE", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": null, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold, NMS, mask decode.", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 2.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "github://ultralytics/assets/yoloe-11s-seg.pt" ], "formats": [ { "ref": "github://ultralytics/assets/yoloe-11s-seg.pt", "format": "pytorch_pt", "file": "yoloe-11s-seg.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "YOLOE-11 (text, visual and prompt-free): segmentation model", "description": "YOLOE-11 (text, visual and prompt-free) is a image and text segmentation model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "YOLOE-11 (text, visual and prompt-free)", "yoloe", "segmentation", "yoloe-11s-seg.pt", "object-detection" ], "faqs": [ { "q": "What is YOLOE-11 (text, visual and prompt-free) best at?", "a": "Open-vocabulary detection and segmentation from text, an example image, or no prompt (prompt-free variant names objects from a built-in vocabulary)." }, { "q": "What are the limits of YOLOE-11 (text, visual and prompt-free)?", "a": "License blocks closed commercial use; masks coarser than SAM." }, { "q": "Can YOLOE-11 (text, visual and prompt-free) be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does YOLOE-11 (text, visual and prompt-free) run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://docs.ultralytics.com/models/yoloe/", "https://arxiv.org/abs/2503.07465", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "yoloe", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image", "text" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2025-03", "params_millions": null, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 2.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "github" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolov8-detect", "identity": { "family": "yolov8", "display_name": "Ultralytics YOLOv8 detect", "hub_id": "Ultralytics/YOLOv8 (yolov8n.pt)", "api_name": null, "revision": null, "variants": [ "yolov8s.pt", "yolov8m.pt", "yolov8l.pt", "yolov8x.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2023-01-10", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "best_at": "Fast detect for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "labels": "COCO 80 classes", "scores": "0..1" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": 6.5, "params_millions": 3.2, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d" ], "formats": [ { "ref": "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d", "format": "pytorch_pt", "file": "5 files, e.g. yolov8l.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLOv8 detect: object detection model", "description": "Ultralytics YOLOv8 detect is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLOv8 detect", "yolov8", "object detection", "Ultralytics/YOLOv8 (yolov8n.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLOv8 detect best at?", "a": "Fast detect for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLOv8 detect?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLOv8 detect be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLOv8 detect run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLOv8", "https://docs.ultralytics.com/models/yolov8/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolov8", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "labels", "scores" ], "use_cases": [ "object-detection" ], "recommended": false, "release_date": "2023-01-10", "params_millions": 3.2, "size_on_disk_mb": 6.5, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolov8-pose", "identity": { "family": "yolov8", "display_name": "Ultralytics YOLOv8 pose", "hub_id": "Ultralytics/YOLOv8 (yolov8n-pose.pt)", "api_name": null, "revision": null, "variants": [ "yolov8s-pose.pt", "yolov8m-pose.pt", "yolov8l-pose.pt", "yolov8x-pose.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2023-01-10", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "detection", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "best_at": "Fast pose for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "person xyxy", "keypoints": "COCO 17 keypoints (x, y, visibility)" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "person boxes + keypoints -> pose payload per frame -> attribute filter" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": 3.3, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d" ], "formats": [ { "ref": "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d", "format": "pytorch_pt", "file": "5 files, e.g. yolov8l.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "pose", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLOv8 pose: object detection model", "description": "Ultralytics YOLOv8 pose is a image object detection model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLOv8 pose", "yolov8", "object detection", "Ultralytics/YOLOv8 (yolov8n-pose.pt)", "pose", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLOv8 pose best at?", "a": "Fast pose for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLOv8 pose?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLOv8 pose be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLOv8 pose run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLOv8", "https://docs.ultralytics.com/models/yolov8/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "detection", "family": "yolov8", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "keypoints", "scores" ], "use_cases": [ "pose", "object-detection" ], "recommended": false, "release_date": "2023-01-10", "params_millions": 3.3, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 }, { "id": "yolov8-segment", "identity": { "family": "yolov8", "display_name": "Ultralytics YOLOv8 segment", "hub_id": "Ultralytics/YOLOv8 (yolov8n-seg.pt)", "api_name": null, "revision": null, "variants": [ "yolov8s-seg.pt", "yolov8m-seg.pt", "yolov8l-seg.pt", "yolov8x-seg.pt" ], "license": "AGPL-3.0", "commercial_use": false, "commercial_use_note": "AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license.", "release_date": "2023-01-10", "paper_url": null, "repo_url": "https://github.com/ultralytics/ultralytics" }, "task": { "category": "segmentation", "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "best_at": "Fast segment for the fixed COCO label set; n size runs in real time on CPU.", "worst_at": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, "io": { "input": { "image": { "resolution": "640x640 letterboxed", "color": "RGB" }, "prompts": "none (fixed classes)" }, "output": { "boxes": "xyxy pixels", "masks": "per-instance bitmaps (prototype masks x coefficients) or polygons", "labels": "COCO 80 classes" } }, "mixpeek_wiring": { "extractors": [], "feature_uris": [], "vector_indexes": [], "retriever_stages": [], "composes_with": [ "detector boxes -> crops -> SigLIP vectors per object (one document per object, or a multi-vector per frame)", "labels -> payload field -> attribute filter in a retriever", "scene splits -> keyframes -> detector -> object documents with frame timestamps" ] }, "run": { "status": "not_runnable", "status_reason": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops.", "serve_apps": [], "runtime": "ultralytics YOLO (or onnxruntime after export)", "weights_format": "pickle", "trust_remote_code": false, "size_on_disk_mb": null, "params_millions": 3.4, "preprocessing": "Letterbox to 640x640 (stride-aligned padding), RGB, scale to 0..1.", "postprocessing": "Confidence threshold then NMS, mask decode from prototypes", "serving_shape": "both", "batch_size": null, "serve_batch": null, "scale_to_zero": { "min_replicas": 0, "max_replicas": 1, "idle_timeout_s": 600.0, "memory_gb": 1.0, "num_cpus": 1.0, "num_gpus": 0.0, "cold_start_s": { "warm_node_s": null, "new_node_s": null, "source": null }, "cpu_fallback": true } }, "cost": { "unmeasured": true, "measured": null, "note": "No committed measurement; plan row 3 measures it in the standalone image." }, "evidence": { "unmeasured": true, "items": [] }, "refs": [ "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d" ], "formats": [ { "ref": "hf://Ultralytics/YOLOv8@27f858f2dc238503a66ce10248cf7ec71a3ded5d", "format": "pytorch_pt", "file": "5 files, e.g. yolov8l.pt", "loader": "torch.load (pickle); Mixpeek refuses pickle weights (shared/security/torch_pickle.py)", "supported_today": false } ], "use_cases": [ "segmentation", "object-detection" ], "recommended": { "value": false, "for": [], "reason": null }, "demos": [], "seo": { "title": "Ultralytics YOLOv8 segment: segmentation model", "description": "Ultralytics YOLOv8 segment is a image segmentation model (AGPL-3.0). Not runnable on Mixpeek today.", "keywords": [ "Ultralytics YOLOv8 segment", "yolov8", "segmentation", "Ultralytics/YOLOv8 (yolov8n-seg.pt)", "object-detection" ], "faqs": [ { "q": "What is Ultralytics YOLOv8 segment best at?", "a": "Fast segment for the fixed COCO label set; n size runs in real time on CPU." }, { "q": "What are the limits of Ultralytics YOLOv8 segment?", "a": "Closed label set (needs fine-tuning for new classes); license blocks closed commercial use; small or crowded objects at 640 input." }, { "q": "Can Ultralytics YOLOv8 segment be used commercially?", "a": "No. AGPL-3.0: offering it over a network obliges publishing the whole application's source under AGPL; closed commercial use needs an Ultralytics Enterprise license." }, { "q": "How does Ultralytics YOLOv8 segment run on Mixpeek?", "a": "Blocked by AGPL-3.0 and pickle .pt weights. Would run as a custom extractor plugin that exports the checkpoint to ONNX once (ultralytics export format=onnx) and serves it with onnxruntime on CPU, emitting boxes, labels and crops." } ] }, "last_verified": "2026-11-05", "sources": [ "https://huggingface.co/Ultralytics/YOLOv8", "https://docs.ultralytics.com/models/yolov8/", "https://www.ultralytics.com/license", "https://roboflow.com/model-licenses/yoloe" ], "registry": null, "templates": [], "usage": { "namespaces": null, "collections": null, "uncounted": true, "templates": 0, "counted_at": null, "source": null }, "facets": { "status": "not_runnable", "category": "segmentation", "family": "yolov8", "license": "AGPL-3.0", "commercial_use": false, "modalities_in": [ "image" ], "modalities_out": [ "boxes", "masks", "labels", "scores" ], "use_cases": [ "segmentation", "object-detection" ], "recommended": false, "release_date": "2023-01-10", "params_millions": 3.4, "size_on_disk_mb": null, "vector_dims": null, "vector_distance": null, "weights_format": "pickle", "formats": [ "pytorch_pt" ], "loadable_today": false, "trust_remote_code": false, "cpu_fallback": true, "num_gpus": 0.0, "memory_gb": 1.0, "min_replicas": 0.0, "serving_shape": "both", "served": false, "ref_schemes": [ "hf" ], "has_evidence": false, "cost_measured": false, "template_count": 0, "usage_namespaces": null, "usage_collections": null, "usage_uncounted": true, "has_demo": false, "rank_score": 0.0 }, "rank_score": 0.0 } ] }