Video Understanding

import openai
import cv2
import base64
from typing import List

def extract_frames(video_path: str, fps: float = 1.0) -> List[str]:
    """Extract frames from video at specified FPS, return as base64."""
    video = cv2.VideoCapture(video_path)
    original_fps = video.get(cv2.CAP_PROP_FPS)
    frame_interval = int(original_fps / fps)

    frames = []
    frame_count = 0

    while True:
        success, frame = video.read()
        if not success:
            break

        if frame_count % frame_interval == 0:
            # Resize for efficiency
            frame = cv2.resize(frame, (512, 512))
            _, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85])
            frames.append(base64.b64encode(buffer).decode('utf-8'))

        frame_count += 1

    video.release()
    return frames

def describe_video(video_path: str, max_frames: int = 20) -> str:
    """Generate description of video using GPT-4V with frame sampling."""
    client = openai.OpenAI()

    # Extract frames (1 FPS)
    frames = extract_frames(video_path, fps=1.0)

    # Limit frames to avoid token limits
    if len(frames) > max_frames:
        # Uniform sampling
        indices = [int(i * len(frames) / max_frames) for i in range(max_frames)]
        frames = [frames[i] for i in indices]

    # Build message with frames
    content = [{"type": "text", "text": """Analyze this video represented as frames.
Provide:
1. A brief summary of what happens
2. Key actions or events in chronological order
3. Notable objects, people, or settings"""}]

    for i, frame in enumerate(frames):
        content.append({
            "type": "image_url",
            "image_url": {
                "url": f"data:image/jpeg;base64,{frame}",
                "detail": "low"  # Use 'high' for more detail
            }
        })

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": content}],
        max_tokens=1000
    )

    return response.choices[0].message.content

# Usage
description = describe_video("cooking_video.mp4")
print(description)