Contextual video match

# contextual-video-match: one manifest spins up the namespace.
# Platform manifest schema (GET /v1/discovery/schema). Validate with POST /v1/manifest/validate,
# apply with POST /v1/manifest/apply or the Deploy button. Wiring comes from the flow diagram:
# edges are bucket -> collection sources, collection -> retriever scope, retriever -> view.
# Applying a SECOND time, over a namespace this template already created: use
# POST /v1/manifest/apply?mode=create_missing, which creates what is missing and leaves
# what exists alone. The default, create_only, fails the WHOLE apply and rolls it back if
# any resource already exists, so an upgrade looks like a dead end without this. Use
# mode=upsert to also patch resources that exist but have drifted from this file.
version: '1.0'
metadata:
  name: contextual-video-match
  description: "Namespace template contextual-video-match. Generated from the flow diagram on mixpeek.com/templates/contextual-video-match."
namespaces:
  - name: contextual-video-match
    description: "Everything below lives in this namespace."
    feature_extractors:
      - name: multimodal_extractor
        version: v1
      - name: text_extractor
        version: v1
    # Daily spend budget, in dollars of what you are charged (1,000 credits a dollar).
    # New batches pause once the namespace has spent this much today; queries are not capped.
    # Raise it later on the namespace: PATCH /v1/namespaces/<id> {"spend_budget": {"daily_usd": N}}.
    budget:
      daily_usd: 0.1

# Data sources. A storage connection carries credentials, so it is created in Studio
# (or POST /v1/organizations/storage-connections) and synced into the bucket named here.
#   video-corpus: s3, continuous, the video library you want matched against, in your account -> bucket videos
#   article-feed: manual, continuous, one object per page: the article text, its URL, its publish time and the tenant it belongs to -> bucket articles
buckets:
  - name: videos
    namespace: contextual-video-match
    description: "Meant to be fed by video-corpus (s3, continuous). No source is connected yet: applying this manifest creates the bucket only. Connect the source to this bucket in Studio (Syncs) to start the feed."
    schema:
      properties:
        content:
          type: video
  - name: articles
    namespace: contextual-video-match
    description: "Meant to be fed by article-feed (manual, continuous): upload files here or write them through the API. Nothing arrives until you do."
    schema:
      properties:
        content:
          type: text

# Connect your own storage. Create the two secrets, uncomment, and apply again;
# apply live-tests the connection, so it must have real credentials to succeed.
# Until then the buckets above accept direct uploads.
# storage_connections:
#   - name: video-corpus-connection
#     provider: s3
#     description: "Read-only access to the s3 location holding the video library you want matched against, in your account."
#     config:
#       region: us-east-1
#       credentials:
#         type: access_key
#         access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
#         secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

# bucket_syncs:
#   - name: video-corpus-sync
#     bucket: videos
#     connection: video-corpus-connection
#     source_path: "videos/"
#     sync_mode: continuous
#     polling_interval_seconds: 300
#     skip_duplicates: true
#     file_filters:
#       include_patterns: ["*.mp4", "*.mov", "*.webm"]
#     schema_mapping:
#       mappings:
#         content:
#           target_type: blob
#           source: {type: file}
#           blob_type: video

collections:
  - name: video-scenes
    namespace: contextual-video-match
    description: "multimodal_extractor@v1 over bucket videos. Feeds scene-transcripts, contextual-match."
    source:
      type: bucket
      bucket: videos
    feature_extractor:
      name: multimodal_extractor
      version: v1
      parameters:
        run_transcription: true
        run_video_description: true
        run_ocr: true
        description_prompt: "Identify the named entities and topical keywords present in this video segment, from what is spoken, shown on screen, and visible in the frame. Transcribe any text burned into the frame, such as captions, lower thirds, tickers and signage, verbatim."
        response_shape:
          type: "object"
          properties:
            entities:
              type: "array"
              items:
                type: "string"
              description: "Named entities (people, organizations, places) in canonical English form, deduplicated. 3-10 items."
            keywords:
              type: "array"
              items:
                type: "string"
              description: "Topical keywords or short phrases describing what the segment is about. 3-8 items."
            iab_categories:
              type: "array"
              items:
                type: "string"
              description: "IAB Content Taxonomy tier-1 category names that describe this segment, written exactly as IAB names them, for example \"News & Politics\", \"Arts & Entertainment\", \"Sports\", \"Automotive\". 1-3 items."
            ocr_text:
              type: "string"
              description: "Text burned into the frame, transcribed verbatim: captions, lower thirds, tickers, signage. Empty string when the segment shows no text."
      input_mappings:
        video: content
      field_passthrough:
        - source_path: file_location
        - source_path: segment_id
        - source_path: start_time
        - source_path: end_time
        - source_path: tenant_id
    enabled: true
  - name: article-multimodal
    namespace: contextual-video-match
    description: "multimodal_extractor@v1 over bucket articles. Feeds contextual-match."
    source:
      type: bucket
      bucket: articles
    feature_extractor:
      name: multimodal_extractor
      version: v1
      input_mappings:
        text: content
      field_passthrough:
        - source_path: article_url
        - source_path: published_at
        - source_path: tenant_id
    enabled: true
  - name: scene-transcripts
    namespace: contextual-video-match
    description: "text_extractor@v1 over collection video-scenes. Feeds contextual-match."
    source:
      type: collection
      collection: video-scenes
    feature_extractor:
      name: text_extractor
      version: v1
      input_mappings:
        text: transcription
      field_passthrough:
        - source_path: segment_id
        - source_path: video_segment_url
        - source_path: start_time
        - source_path: end_time
    enabled: true
  - name: article-text
    namespace: contextual-video-match
    description: "text_extractor@v1 over bucket articles. Feeds contextual-match."
    source:
      type: bucket
      bucket: articles
    feature_extractor:
      name: text_extractor
      version: v1
      input_mappings:
        text: content
      field_passthrough:
        - source_path: article_url
        - source_path: published_at
        - source_path: tenant_id
    enabled: true
retrievers:
  - name: contextual-match
    namespace: contextual-video-match
    description: "Matches scene-transcripts and video-scenes against a document from article-text and article-multimodal, with no query-time embedding."
    collections:
      - scene-transcripts
      - video-scenes
      - article-text
      - article-multimodal
    input_schema:
      article_text_doc_id:
        type: string
        required: true
        description: "Document id of the article in article-text, whose stored text vector the scene transcripts are matched against"
      article_multimodal_doc_id:
        type: string
        required: true
        description: "Document id of the same article in article-multimodal, whose stored multimodal vector the video scenes are matched against"
      article_entities:
        type: array
        required: false
        description: "Entities already extracted from the article; drives the 0.25 overlap term"
      article_keywords:
        type: array
        required: false
        description: "Keywords already extracted from the article; drives the 0.15 overlap term"
      article_iab_categories:
        type: array
        required: false
        description: "IAB categories for the article; drives the 0.10 taxonomy term and the Entertainment-against-News penalty"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://text_extractor@v1/multilingual_e5_large_instruct_v1"
              collection_identifiers:
                - scene-transcripts
              query:
                input_mode: document
                document_ref:
                  collection_id: article-text
                  document_id: "{{INPUT.article_text_doc_id}}"
              top_k: 30
              min_score: 0.15
            - feature_uri: "mixpeek://multimodal_extractor@v1/vertex_multimodal_embedding"
              collection_identifiers:
                - video-scenes
              query:
                input_mode: document
                document_ref:
                  collection_id: article-multimodal
                  document_id: "{{INPUT.article_multimodal_doc_id}}"
              top_k: 30
          fusion: rrf
          final_top_k: 30
      - stage_name: deduplicate
        stage_id: deduplicate
        parameters:
          strategy: field
          fields:
            - source_video_url
            - start_time
          keep: first
      - stage_name: score
        stage_id: code_execution
        parameters:
          language: python
          output_field: scoring
          timeout_ms: 5000
          code: |
            _ae = {{ INPUT.article_entities }}
            _ak = {{ INPUT.article_keywords }}
            _ac = {{ INPUT.article_iab_categories }}
            art_ents = set(str(e).lower() for e in (_ae or []))
            art_kws = set(str(k).lower() for k in (_ak or []))
            art_cats = set(str(c).lower() for c in (_ac or []))
            result = []
            for doc in docs:
                ve = set(str(e).lower() for e in doc.get('entities', []))
                vk = set(str(k).lower() for k in doc.get('keywords', []))
                vc = set(str(c).lower() for c in doc.get('iab_categories', []))
                vt = ((doc.get('transcription') or '') + ' ' + (doc.get('ocr_text') or '')).lower()
                te = set(e for e in art_ents if e in vt)
                all_ents = (art_ents & ve) | te
                eo = len(all_ents) / max(len(art_ents), 1) if art_ents else 0
                ko = len(art_kws & vk) / max(len(art_kws), 1) if art_kws else 0
                to = len(art_cats & vc) / max(len(art_cats), 1) if art_cats else 0
                s = doc.get('scores', {}).get('search', doc.get('score', 0))
                cs = 0.50 * s + 0.25 * min(eo * 1.5, 1.0) + 0.15 * min(ko * 1.5, 1.0) + 0.10 * to
                tp = 0.7 if vc and vc & {'entertainment', 'arts & entertainment'} and art_cats & {'news & politics', 'law government & politics'} else 1.0
                cs *= tp
                result.append({'contextual_score': round(cs, 4), 'entity_overlap': round(eo, 4), 'keyword_overlap': round(ko, 4), 'taxonomy_overlap': round(to, 4), 'entities_matched': list(all_ents)[:5], 'keywords_matched': list(art_kws & vk)[:5], 'iab_matched': list(art_cats & vc)[:3], 'taxonomy_penalty': tp})
    tags:
      - template:contextual-video-match