Automated Content Moderation for UGC Platforms

import { Mixpeek } from 'mixpeek';

const client = new Mixpeek({ apiKey: process.env.MIXPEEK_API_KEY });

// Define moderation policies
const policies = await client.moderation.createPolicy({
  policy_id: 'community-guidelines-v2',
  rules: [
    {
      category: 'violence',
      action: 'block',
      threshold: 0.85,
      exceptions: ['news', 'documentary', 'educational']
    },
    {
      category: 'nudity',
      action: 'block',
      threshold: 0.90,
      exceptions: ['art', 'educational', 'medical']
    },
    {
      category: 'hate_speech',
      action: 'block',
      threshold: 0.80,
      review_required: true
    },
    {
      category: 'self_harm',
      action: 'block_and_alert',
      threshold: 0.70,
      alert_team: 'trust_safety'
    }
  ],
  age_gating: {
    suggestive: 18,
    alcohol: 21,
    gambling: 21
  }
});

// Real-time moderation on upload
async function moderateUpload(uploadId: string, contentUrl: string, contentType: 'video' | 'image') {
  const moderation = await client.moderation.analyze({
    url: contentUrl,
    content_type: contentType,
    policy_id: 'community-guidelines-v2',
    extractors: [
      'violence-detection',
      'nudity-detection',
      'hate-speech-detection',
      'self-harm-detection',
      'audio-analysis'  // For video audio tracks
    ]
  });

  return {
    upload_id: uploadId,
    decision: moderation.action, // 'approve', 'block', 'review'
    violations: moderation.violations,
    confidence: moderation.confidence,
    processing_time_ms: moderation.latency,
    review_required: moderation.review_required
  };
}

// Escalation workflow for edge cases
async function handleModerationResult(result: ModerationResult) {
  switch (result.decision) {
    case 'approve':
      await publishContent(result.upload_id);
      break;

    case 'block':
      await notifyUploader(result.upload_id, result.violations);
      await logViolation(result);
      break;

    case 'review':
      // Queue for human review with AI context
      await queueForReview({
        upload_id: result.upload_id,
        ai_analysis: result.violations,
        confidence: result.confidence,
        priority: calculatePriority(result),
        suggested_action: result.suggested_action
      });
      break;
  }

  // Special handling for self-harm content
  if (result.violations.some(v => v.category === 'self_harm')) {
    await alertTrustSafety(result);
  }
}

// Continuous accuracy monitoring
async function getModerationMetrics(dateRange: { start: string; end: string }) {
  const metrics = await client.moderation.getMetrics({
    policy_id: 'community-guidelines-v2',
    date_range: dateRange
  });

  return {
    total_processed: metrics.total,
    auto_approved: metrics.approved_count,
    auto_blocked: metrics.blocked_count,
    sent_to_review: metrics.review_count,

    // Accuracy metrics (requires human review sampling)
    true_positive_rate: metrics.tpr,
    false_positive_rate: metrics.fpr,

    // Latency
    avg_processing_ms: metrics.avg_latency,
    p99_processing_ms: metrics.p99_latency,

    // By category
    violations_by_category: metrics.category_breakdown
  };
}