Video Moderation Extraction Foundation

# video-moderation: one manifest spins up the namespace.
# Platform manifest schema (GET /v1/discovery/schema). Validate with POST /v1/manifest/validate,
# apply with POST /v1/manifest/apply or the Deploy button. Wiring comes from the flow diagram:
# edges are bucket -> collection sources, collection -> retriever scope, retriever -> view.
# Applying a SECOND time, over a namespace this template already created: use
# POST /v1/manifest/apply?mode=create_missing, which creates what is missing and leaves
# what exists alone. The default, create_only, fails the WHOLE apply and rolls it back if
# any resource already exists, so an upgrade looks like a dead end without this. Use
# mode=upsert to also patch resources that exist but have drifted from this file.
version: '1.0'
metadata:
  name: video-moderation
  description: "Namespace template video-moderation. Generated from the flow diagram on mixpeek.com/templates/video-moderation."
namespaces:
  - name: video-moderation
    description: "Everything below lives in this namespace."
    feature_extractors:
      - name: multimodal_extractor
        version: v2
      - name: image_extractor
        version: v1
      - name: passthrough_extractor
        version: v1
    # Daily spend budget, in dollars of what you are charged (1,000 credits a dollar).
    # New batches pause once the namespace has spent this much today; queries are not capped.
    # Raise it later on the namespace: PATCH /v1/namespaces/<id> {"spend_budget": {"daily_usd": N}}.
    budget:
      daily_usd: 1

# Data sources. A storage connection carries credentials, so it is created in Studio
# (or POST /v1/organizations/storage-connections) and synced into the bucket named here.
#   source-video: s3, continuous, your footage, in your account -> bucket source-video
#   brand-marks: manual, one-shot, customer-supplied logo and mark variants -> bucket brand-mark-refs
#   person-references: manual, one-shot, customer-supplied public-figure reference imagery -> bucket person-refs
buckets:
  - name: source-video
    namespace: video-moderation
    description: "Meant to be fed by source-video (s3, continuous). No source is connected yet: applying this manifest creates the bucket only. Connect the source to this bucket in Studio (Syncs) to start the feed."
    schema:
      properties:
        content:
          type: video
  - name: brand-mark-refs
    namespace: video-moderation
    description: "Meant to be fed by brand-marks (manual, one-shot): upload files here or write them through the API. Nothing arrives until you do."
    schema:
      properties:
        content:
          type: image
    # Starts empty and is yours to fill. Until it has files, screen against the hosted
    # set instead: 189,830 brand marks from LogoDet-3K, OpenLogo, OSLD, Logos-in-the-Wild and the ESPN team sets
    #   curl -X POST https://api.mixpeek.com/v1/marketplace/catalog/brand-logo-match/execute \
    #     -H 'Content-Type: application/json' \
    #     -d '{"inputs":{"query":"<image url>"},"pagination":{"method":"offset","page_number":1,"page_size":10}}'
    # No key and no subscription. https://mixpeek.com/marketplace/brand-logo-match
  - name: person-refs
    namespace: video-moderation
    description: "Meant to be fed by person-references (manual, one-shot): upload files here or write them through the API. Nothing arrives until you do."
    schema:
      properties:
        content:
          type: image
    # Starts empty and is yours to fill. Until it has files, screen against the hosted
    # set instead: 32,667 public-figure faces from LFW, FaceScrub, Wikipedia and the 2025 NFL, NBA, MLB, NHL, NCAA and Premier League rosters
    #   curl -X POST https://api.mixpeek.com/v1/marketplace/catalog/celebrity-face-match/execute \
    #     -H 'Content-Type: application/json' \
    #     -d '{"inputs":{"query":"<image url>"},"pagination":{"method":"offset","page_number":1,"page_size":10}}'
    # No key and no subscription. https://mixpeek.com/marketplace/celebrity-face-match

# Connect your own storage. Create the two secrets, uncomment, and apply again;
# apply live-tests the connection, so it must have real credentials to succeed.
# Until then the buckets above accept direct uploads.
# storage_connections:
#   - name: source-video-connection
#     provider: s3
#     description: "Read-only access to the s3 location holding your footage, in your account."
#     config:
#       region: us-east-1
#       credentials:
#         type: access_key
#         access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
#         secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

# bucket_syncs:
#   - name: source-video-sync
#     bucket: source-video
#     connection: source-video-connection
#     source_path: "footage/"
#     sync_mode: continuous
#     polling_interval_seconds: 300
#     skip_duplicates: true
#     file_filters:
#       include_patterns: ["*.mp4", "*.mov", "*.mkv", "*.webm"]
#     schema_mapping:
#       mappings:
#         content:
#           target_type: blob
#           source: {type: file}
#           blob_type: video

collections:
  - name: video-segments
    namespace: video-moderation
    description: "multimodal_extractor@v2 over bucket source-video. Feeds flagged-segment-monitor, flagged-segment, faces, keyframes, onscreen-text, segment-rollup, roll-up-by-segment, segment-search."
    source:
      type: bucket
      bucket: source-video
    feature_extractor:
      name: multimodal_extractor
      version: v2
      parameters:
        run_transcription: true
        run_transcription_embedding: true
        run_ocr: true
        run_ocr_embedding: true
      input_mappings:
        video: content
      field_passthrough:
        - source_path: file_location
        - source_path: segment_id
        - source_path: start_time
        - source_path: end_time
    alert_applications:
      - alert: flagged-segment
        input_mappings:
          - input_key: query
            source: {source_type: constant, value: "nudity, violence or a weapon"}
          - input_key: document_id
            source: {source_type: document_field, path: _internal.document_id}
    enabled: true
  - name: brand-mark-index
    namespace: video-moderation
    description: "image_extractor@v1 over bucket brand-mark-refs. Feeds brand-mark-match."
    source:
      type: bucket
      bucket: brand-mark-refs
    feature_extractor:
      name: image_extractor
      version: v1
      input_mappings:
        image: content
      field_passthrough:
        - source_path: mark_name
        - source_path: rule
    enabled: true
  - name: keyframes
    namespace: video-moderation
    description: "image_extractor@v1 over collection video-segments. Feeds violence-context."
    source:
      type: collection
      collection: video-segments
    feature_extractor:
      name: image_extractor
      version: v1
      input_mappings:
        image: thumbnail_url
      field_passthrough:
        - source_path: thumbnail_url
        - source_path: segment_id
        - source_path: frame_ts
    enabled: true

# NOT APPLIED. Every collection person-verdicts derives from (faces, age-estimation) is commented out in this
# manifest, so it has no source to be created from. It applies once one of them does.
#   - name: person-verdicts
#     namespace: video-moderation
#     description: "Cluster person-rollup is the intended producer and is emitted commented out below, so nothing writes here yet. One document per person_id. Writes back n_looks. Declared as a passthrough collection so the retrievers that name it still resolve; read what they return as source rows, not as this view's output. age_median and aged wait on the age-estimation block above."
#     source:
#       type: none
#     feature_extractor:
#       name: passthrough_extractor
#       version: v1
#       field_passthrough: []
#     enabled: true
  - name: segment-verdicts
    namespace: video-moderation
    description: "Cluster segment-rollup is the intended producer and is emitted commented out below. Until it applies, this collection holds a plain copy of video-segments's rows, with none of the fields this view describes. One document per segment_id. Writes back n_person. Declared as a passthrough collection so the retrievers that name it still resolve; read what they return as source rows, not as this view's output. decision, reason, n_minor and n_unaged wait on the age-estimation block above, n_borderline on violence-context."
    source:
      type: collection
      collection: video-segments
    feature_extractor:
      name: passthrough_extractor
      version: v1
      field_passthrough: []
    enabled: true

# NOT APPLIED. Face recognition is turned off on Mixpeek Cloud, so this face step does not deploy. It runs on a dedicated or customer-hosted deployment, where face_identity_extractor is served.
# The block below is correct and is emitted commented out so the rest of this
# manifest applies; an apply is all-or-nothing and would otherwise roll back.
#   - name: person-ref-index
#     namespace: video-moderation
#     description: "face_identity_extractor@v1 over bucket person-refs. Feeds public-figure-match."
#     source:
#       type: bucket
#       bucket: person-refs
#     feature_extractor:
#       name: face_identity_extractor
#       version: v1
#       input_mappings:
#         image: content
#       field_passthrough:
#         - source_path: person_name
#         - source_path: rights_note
#     enabled: true

# NOT APPLIED. Face recognition is turned off on Mixpeek Cloud, so this face step does not deploy. It runs on a dedicated or customer-hosted deployment, where face_identity_extractor is served.
# The block below is correct and is emitted commented out so the rest of this
# manifest applies; an apply is all-or-nothing and would otherwise roll back.
#   - name: faces
#     namespace: video-moderation
#     description: "face_identity_extractor@v1 over collection video-segments. Feeds age-estimation, person-rollup, segment-rollup."
#     source:
#       type: collection
#       collection: video-segments
#     feature_extractor:
#       name: face_identity_extractor
#       version: v1
#       input_mappings:
#         video: video_segment_url
#       field_passthrough:
#         - source_path: segment_id
#         - source_path: bbox
#         - source_path: detection_score
#         - source_path: quality_score
#     enabled: true

# NOT APPLIED. scrolling_text_extractor@v1 declares expected_input_types {video: video}, and a collection-to-collection chain hands it video_segment_url, which the platform types as STRING. POST /v1/manifest/apply rejects the pair and rolls the whole manifest back. face_identity_extractor@v1 takes the same mapping and applies, because it declares no expected_input_types. Tracked with backend; remove this line when the chain accepts a URL string for a typed video input.
# The block below is correct and is emitted commented out so the rest of this
# manifest applies; an apply is all-or-nothing and would otherwise roll back.
#   - name: onscreen-text
#     namespace: video-moderation
#     description: "scrolling_text_extractor@v1 over collection video-segments. Feeds roll-up-by-segment."
#     source:
#       type: collection
#       collection: video-segments
#     feature_extractor:
#       name: scrolling_text_extractor
#       version: v1
#       input_mappings:
#         video: video_segment_url
#       field_passthrough:
#         - source_path: segment_id
#     enabled: true

# NOT APPLIED. No age_estimator extractor is published to install yet, and it reads the face crops that face recognition would produce, which is off on Mixpeek Cloud.
# The block below is correct and is emitted commented out so the rest of this
# manifest applies; an apply is all-or-nothing and would otherwise roll back.
#   - name: age-estimation
#     namespace: video-moderation
#     description: "age_estimator@v1 over collection faces. Feeds person-rollup. age_estimator is a template-provided extractor (a plugin that ships with this template, not a hosted one): install it, then apply this block."
#     source:
#       type: collection
#       collection: faces
#     feature_extractor:
#       name: age_estimator
#       version: v1
#       input_mappings:
#         face: face_crop_url
#       field_passthrough:
#         - source_path: person_id
#         - source_path: age_median
#         - source_path: aged
#         - source_path: n_looks
#     enabled: true

# NOT APPLIED. No context_scorer extractor is published to install yet, so this step has nothing to run.
# The block below is correct and is emitted commented out so the rest of this
# manifest applies; an apply is all-or-nothing and would otherwise roll back.
#   - name: violence-context
#     namespace: video-moderation
#     description: "context_scorer@v1 over collection keyframes. Feeds roll-up-by-segment. context_scorer is a template-provided extractor (a plugin that ships with this template, not a hosted one): install it, then apply this block."
#     source:
#       type: collection
#       collection: keyframes
#     feature_extractor:
#       name: context_scorer
#       version: v1
#       input_mappings:
#         image: thumbnail_url
#       field_passthrough:
#         - source_path: segment_id
#         - source_path: staged_score
#         - source_path: real_score
#         - source_path: margin
#     enabled: true
retrievers:
  - name: brand-mark-match
    namespace: video-moderation
    description: "Searches brand-mark-index across 1 feature index."
    collections:
      - brand-mark-index
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://image_extractor@v1/google_siglip_base_v1"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 5
          fusion: rrf
          final_top_k: 5
    tags:
      - template:video-moderation
  # NOT APPLIED. Face recognition is turned off on Mixpeek Cloud, so this face step does not deploy. It runs on a dedicated or customer-hosted deployment, where face_identity_extractor is served.
  # - name: public-figure-match
  #   namespace: video-moderation
  #   description: "Searches person-ref-index across 1 feature index."
  #   collections:
  #     - person-ref-index
  #   input_schema:
  #     query:
  #       type: text
  #       required: true
  #       description: "URL of the image or frame to match; encoded by the index below and compared against it"
  #   stages:
  #     - stage_name: search
  #       stage_id: feature_search
  #       parameters:
  #         searches:
  #           - feature_uri: "mixpeek://face_identity_extractor@v1/insightface__arcface"
  #             query:
  #               input_mode: content
  #               value: "{{INPUT.query}}"
  #             top_k: 5
  #         fusion: rrf
  #         final_top_k: 5
  #   tags:
  #     - template:video-moderation
  - name: roll-up-by-segment
    namespace: video-moderation
    description: "Pipeline retriever over video-segments. Groups documents by segment_id."
    collections:
      - video-segments
      # - person-verdicts  (add once its commented-out block above applies)
      # - violence-context  (add once context_scorer is installed and its block above is applied)
      # - onscreen-text  (add once its commented-out block above applies)
    input_schema:
      limit:
        type: text
        required: false
        description: "Optional cap on documents grouped per run"
    stages:
      - stage_name: source
        stage_id: attribute_filter
        parameters:
          field: segment_id
          operator: exists
          value: true
      - stage_name: group
        stage_id: group_by
        parameters:
          group_by_field: segment_id
    tags:
      - template:video-moderation
  - name: segment-search
    namespace: video-moderation
    description: "Plain-text search over every segment of your footage, the foundation's own entry point: describe what is on screen and get the segments back with their source."
    collections:
      - video-segments
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://multimodal_extractor@v2/gemini-embedding-2"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 20
          fusion: rrf
          final_top_k: 20
    tags:
      - template:video-moderation
  - name: review-queue
    namespace: video-moderation
    description: "Searches segment-verdicts across 1 feature index."
    collections:
      - segment-verdicts
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://multimodal_extractor@v2/gemini-embedding-2"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 100
          fusion: rrf
          final_top_k: 100
    tags:
      - template:video-moderation
  - name: moderation-search
    namespace: video-moderation
    description: "Searches segment-verdicts across 3 feature indexes with learned fusion."
    collections:
      - segment-verdicts
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
      session_id:
        type: text
        required: false
        description: "Session identifier; the learned fusion uses it as personal context"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://multimodal_extractor@v2/gemini-embedding-2"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 25
            - feature_uri: "mixpeek://multimodal_extractor@v2/multilingual_e5_large_instruct_v1"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 25
            - feature_uri: "mixpeek://multimodal_extractor@v2/multilingual_e5_large_instruct_ocr_v1"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 25
          fusion: learned
          final_top_k: 25
          # Shadow first: weights are learned and logged, static fusion is served, until the
          # signals justify a rollout. Flip rollout_pct through the retriever update path.
          learning_config:
            shadow_mode: true
            rollout_pct: 0.0
            context_features:
              - INPUT.session_id
    tags:
      - template:video-moderation
  - name: cleared-set-export
    namespace: video-moderation
    description: "Searches segment-verdicts across 1 feature index."
    collections:
      - segment-verdicts
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
    stages:
      - stage_name: search
        stage_id: feature_search
        parameters:
          searches:
            - feature_uri: "mixpeek://multimodal_extractor@v2/gemini-embedding-2"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 500
          fusion: rrf
          final_top_k: 500
    tags:
      - template:video-moderation
  - name: flagged-segment-monitor
    namespace: video-moderation
    description: "The alert's check: does the segment just written show nudity, violence or a weapon. The search is scoped to that one segment, so earlier footage is never re-reported, and an llm_filter watches the segment and keeps it only when it does. Materialises into flagged-segment."
    collections:
      - video-segments
    input_schema:
      query:
        type: text
        required: true
        description: "What to look for; searched across every index below"
      document_id:
        type: text
        required: true
        description: "The written document this run is scoped to, supplied by the alert attachment"
    stages:
      - stage_name: search
        stage_id: feature_search
        pre_filters:
          AND:
            - field: _internal.document_id
              operator: eq
              value: "{{INPUT.document_id}}"
        parameters:
          searches:
            - feature_uri: "mixpeek://multimodal_extractor@v2/gemini-embedding-2"
              query:
                input_mode: text
                value: "{{INPUT.query}}"
              top_k: 5
          fusion: rrf
          final_top_k: 5
      - stage_name: judge
        stage_id: llm_filter
        parameters:
          criteria: "Keep this video segment only if it shows {{INPUT.query}}. Judge what is on screen and in the audio, not the file name or metadata. Discard it otherwise."
          include_media: true
    tags:
      - template:video-moderation
# clusters:

# NOT APPLIED. A cluster trigger's output_collections redirects a run into a collection this manifest also declares, and the reference graph validates. The destination never receives the cluster output vector index: that index is installed by the cluster CREATE path (api/clusters/services.py::_create_output_collection), and api/triggers/executor.py dispatches the run straight to the queue without it, so the output points are dropped at upsert and the collection stays empty. Tracked with backend; remove this line when a redirected destination gets the index.
#   - name: person-rollup
#     namespace: video-moderation
#     description: "Groups the faces collection into one cluster per person, on the ArcFace embedding face_identity_extractor writes. Feeds person-verdicts."
#     # faces also feeds this cluster and is NOT named below: it is emitted
#     # commented out above, so naming it here would fail the whole apply. Add it
#     # to source_collections once that block is applied.
#     # age-estimation also feeds this cluster and is NOT named below: it is emitted
#     # commented out above, so naming it here would fail the whole apply. Add it
#     # to source_collections once that block is applied.
#     source_collections:
#       []  # a composite run reads centroids from the executions named below
#     cluster_type: vector
#     vector_config:
#       feature_uris:
#         - mixpeek://face_identity_extractor@v1/insightface__arcface
#       clustering_method: hdbscan
#       face_cluster_merge:
#         enabled: true
#         centroid_cosine_threshold: 0.55
#         bbox_field: bbox
#         frame_field: frame_number
#         scene_field: segment_id
#     llm_labeling:
#       enabled: true
#       provider: google
#       model_name: gemini-2.5-flash
#     enrich_source_collection: true
#     source_enrichment_config:
#       field_mappings:
#         - source_field: cluster_id
#           target_field: person_id
#         - source_field: cluster_label
#           target_field: person_label

# NOT APPLIED. A cluster trigger's output_collections redirects a run into a collection this manifest also declares, and the reference graph validates. The destination never receives the cluster output vector index: that index is installed by the cluster CREATE path (api/clusters/services.py::_create_output_collection), and api/triggers/executor.py dispatches the run straight to the queue without it, so the output points are dropped at upsert and the collection stays empty. Tracked with backend; remove this line when a redirected destination gets the index.
#   - name: segment-rollup
#     namespace: video-moderation
#     description: "Groups faces and video-segments by segment_id, one cluster per segment. Feeds segment-verdicts. Runs after person-rollup, which is what puts person_id on a face."
#     # faces also feeds this cluster and is NOT named below: it is emitted
#     # commented out above, so naming it here would fail the whole apply. Add it
#     # to source_collections once that block is applied.
#     source_collections:
#       - video-segments
#     cluster_type: attribute
#     attribute_config:
#       attributes:
#         - segment_id
#     llm_labeling:
#       enabled: true
#       provider: google
#       model_name: gemini-2.5-flash
alerts:
  - name: flagged-segment
    namespace: video-moderation
    description: "Emails the org notifications address (Settings -> Notifications; unset means opted out) when a newly written segment looks like nudity, violence or a weapon, so it reaches review without anyone searching for it."
    source: retriever
    retriever: flagged-segment-monitor
    trigger_on: results
    notification_config:
      channels:
        - channel_type: email
      include_matches: true
      include_scores: true
    enabled: false
# triggers:

# NOT APPLIED. A cluster trigger's output_collections redirects a run into a collection this manifest also declares, and the reference graph validates. The destination never receives the cluster output vector index: that index is installed by the cluster CREATE path (api/clusters/services.py::_create_output_collection), and api/triggers/executor.py dispatches the run straight to the queue without it, so the output points are dropped at upsert and the collection stays empty. Tracked with backend; remove this line when a redirected destination gets the index.
#   - name: person-rollup-nightly
#     namespace: video-moderation
#     description: "Runs person-rollup nightly and lands the result in person-verdicts."
#     action_type: cluster
#     trigger_type: cron
#     schedule_config:
#       cron_expression: "0 2 * * *"
#       timezone: UTC
#     action_config:
#       cluster: person-rollup
#       output_collections:
#         - person-verdicts
#     status: active

# NOT APPLIED. A cluster trigger's output_collections redirects a run into a collection this manifest also declares, and the reference graph validates. The destination never receives the cluster output vector index: that index is installed by the cluster CREATE path (api/clusters/services.py::_create_output_collection), and api/triggers/executor.py dispatches the run straight to the queue without it, so the output points are dropped at upsert and the collection stays empty. Tracked with backend; remove this line when a redirected destination gets the index.
#   - name: segment-rollup-nightly
#     namespace: video-moderation
#     description: "Runs segment-rollup nightly and lands the result in segment-verdicts. An hour after person-rollup, since it reads what that run writes."
#     action_type: cluster
#     trigger_type: cron
#     schedule_config:
#       cron_expression: "0 3 * * *"
#       timezone: UTC
#     action_config:
#       cluster: segment-rollup
#       output_collections:
#         - segment-verdicts
#     status: active

# Plan 11.14: two public-domain films to moderate and two CC0 photographs as
# brand-mark references, ingested after apply, and the use case they must
# satisfy.
seed:
  - bucket: source-video
    max_objects: 2
    objects:
      - blobs: {content: samples/starter/videos/norelco-speed.mp4}
      - blobs: {content: samples/starter/videos/mexican-hat-dance.mp4}
  - bucket: brand-mark-refs
    max_objects: 2
    objects:
      - blobs: {content: samples/starter/images/mustang-san-francisco.jpg}
      - blobs: {content: samples/starter/images/thessaloniki-port-tankers.jpg}
acceptance:
  max_cost_usd: 0.5
  steps:
    - name: video-segmented
      kind: documents
      collection: video-segments
      expect: {min_documents: 2}
    - name: brand-marks-indexed
      kind: documents
      collection: brand-mark-index
      expect: {min_documents: 2}
    - name: segment-search-answers
      kind: query
      retriever: segment-search
      inputs: {query: people dancing}
      expect: {min_results: 1}
    - name: brand-mark-match-answers
      kind: query
      retriever: brand-mark-match
      inputs: {query: a car on a city street}
      expect: {min_results: 1}