Visual Inventory Reconciliation for Warehouses

import { Mixpeek } from 'mixpeek';

const client = new Mixpeek({ apiKey: process.env.MIXPEEK_API_KEY });

// Index product catalog with multiple views
await client.buckets.connect({
  collection_id: 'product-catalog',
  bucket_uri: 's3://catalog/product-images/',
  extractors: [
    'image-embedding',     // Visual similarity
    'barcode-detection',   // UPC/EAN codes
    'text-extraction',     // Product labels
    'color-analysis',      // Color variants
    'dimension-estimation' // Size estimation
  ],
  settings: {
    sku_field: 'sku',
    images_per_product: 4,  // Front, back, side, detail
    variant_grouping: true
  }
});

// Identify products in warehouse photos
async function identifyInventory(imageUrl: string, location: string) {
  const detected = await client.extract({
    url: imageUrl,
    extractors: ['object-detection', 'barcode-detection']
  });

  // Match detected items against catalog
  const identified = await Promise.all(
    detected.objects.map(async (obj) => {
      const match = await client.retrieve({
        collection_id: 'product-catalog',
        query: {
          type: 'image',
          embedding: obj.embedding
        },
        limit: 1
      });

      return {
        bounding_box: obj.bbox,
        sku: match[0]?.sku,
        confidence: match[0]?.score,
        quantity: obj.count || 1,
        location: location
      };
    })
  );

  return identified.filter(i => i.confidence > 0.85);
}

// Reconcile visual vs system inventory
async function reconcileInventory(
  visualCount: InventoryItem[],
  systemCount: InventoryItem[],
  location: string
) {
  const discrepancies = [];

  for (const item of systemCount) {
    const visualItem = visualCount.find(v => v.sku === item.sku);
    const visualQty = visualItem?.quantity || 0;

    if (Math.abs(visualQty - item.quantity) > 0) {
      discrepancies.push({
        sku: item.sku,
        location: location,
        system_qty: item.quantity,
        visual_qty: visualQty,
        variance: visualQty - item.quantity,
        action_required: Math.abs(visualQty - item.quantity) > item.threshold
      });
    }
  }

  // Items seen but not in system (potential putaway errors)
  const unexpected = visualCount.filter(
    v => !systemCount.find(s => s.sku === v.sku)
  );

  return { discrepancies, unexpected };
}

// Automated inventory monitoring
async function setupContinuousMonitoring(warehouseId: string) {
  // Configure camera feeds for monitoring
  await client.streams.connect({
    stream_id: `warehouse-${warehouseId}`,
    sources: [
      { camera_id: 'aisle-1', interval: '1h' },
      { camera_id: 'aisle-2', interval: '1h' },
      // ... more cameras
    ],
    on_frame: async (frame) => {
      const inventory = await identifyInventory(frame.url, frame.location);
      const system = await getSystemInventory(frame.location);
      const { discrepancies } = await reconcileInventory(inventory, system, frame.location);

      if (discrepancies.some(d => d.action_required)) {
        await alertInventoryTeam(discrepancies);
      }
    }
  });
}