Scalable Multimodal Processing Engine

from mixpeek import Mixpeek

client = Mixpeek(api_key="your-api-key")

# Create sharded collection for scale
collection = client.collections.create(
    collection_name="production_media",
    feature_extractor={
        "feature_extractor_name": "multimodal_extractor",
        "version": "v1"
    },
    sharding={
        "enabled": True,
        "strategy": "hash",
        "num_shards": 32
    }
)

# Batch ingest from manifest
manifest = client.manifests.create(
    manifest_name="q4_uploads",
    source_uri="s3://bucket/manifest.json"
)

# Process at scale with retries
job = client.jobs.create(
    collection_id=collection.id,
    manifest_id=manifest.id,
    batch_size=500,
    max_retries=3,
    idempotency_key="q4_uploads_v1"
)

# Monitor progress
status = client.jobs.get_status(job_id=job.id)
print(f"Processed: {status['completed']}/{status['total']}")