Frame-Level Video Content Search and Discovery

import { Mixpeek } from 'mixpeek';

const client = new Mixpeek({ apiKey: process.env.MIXPEEK_API_KEY });

// Connect to video archive
await client.buckets.connect({
  collection_id: 'video-archive',
  bucket_uri: 's3://media-company-archive/',
  extractors: [
    'scene-detection',
    'speech-to-text',
    'object-detection',
    'face-detection',
    'video-embedding'
  ],
  settings: {
    frame_interval: 1, // Analyze every second
    audio_transcription: true,
    scene_detection: true
  }
});

// Monitor indexing progress
const status = await client.collections.status('video-archive');

console.log(`Processed: ${status.processed_hours} hours`);
console.log(`Remaining: ${status.remaining_hours} hours`);
console.log(`Estimated completion: ${status.estimated_completion}`);

// Processing rate: ~5-10x realtime
// 100,000 hours processes in ~10,000-20,000 hours

async function searchVideo(query: string, filters?: object) {
  const results = await client.retrieve({
    collection_id: 'video-archive',
    query: {
      type: 'text',
      text: query // e.g., "sunset over ocean with birds flying"
    },
    filters: filters,
    limit: 50,
    return_frames: true // Return thumbnail frames
  });

  return results.map(r => ({
    video_id: r.object_id,
    title: r.metadata.title,
    timestamp: r.timestamp,
    duration: r.segment_duration,
    thumbnail: r.frame_url,
    transcript: r.transcript_segment,
    relevance: r.score
  }));
}

// Search with preview clips
async function searchWithPreview(query: string) {
  const results = await searchVideo(query);

  // Generate preview clips for top results
  const previews = await Promise.all(
    results.slice(0, 10).map(async (r) => ({
      ...r,
      preview_url: await generatePreviewClip(
        r.video_id,
        r.timestamp,
        10 // 10 second preview
      )
    }))
  );

  return previews;
}