LFM3-Omni-2B

# Describe a video segment from its frames and soundtrack (usage from the model card).
# pip install "transformers>=5.20.2" torch torchvision torchcodec accelerate pillow librosa soundfile
import torch
from transformers import AutoModelForMultimodalLM, AutoProcessor

model_id = "LiquidAI/LFM3-Omni-2B"
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModelForMultimodalLM.from_pretrained(
    model_id, dtype=torch.bfloat16 if device == "cuda" else torch.float32,
    device_map=device, trust_remote_code=True).eval()
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)

conversation = [
    {"role": "system", "content": "Answer questions about the video using what you see and hear."},
    {"role": "user", "content": [
        {"type": "video", "video": "segment_0184.mp4"},
        {"type": "text", "text": "Describe this clip in two sentences for a search index: people, objects, on-screen text, and what is said."},
    ]},
]
inputs = processor.apply_chat_template(
    conversation, add_generation_prompt=True, tokenize=True, return_dict=True,
    return_tensors="pt", load_audio_from_video=True,
    processor_kwargs={"use_audio_in_video": True}).to(device)
with torch.no_grad():
    out = model.generate(**inputs, max_new_tokens=256, generation_mode="text")
description = processor.decode(
    out.sequences[0, inputs["input_ids"].shape[1]:out.sequence_lengths[0]], skip_special_tokens=True)
# Embed the description with a text embedding model and upsert it with the file and timestamp.