Caption & Tag

{
  "topic": "The story of a multi-generational family bakery, its history, its destruction by fire, and the determination to rebuild and adapt the business for a new era.",
  "summary": "The video chronicles the history of the Jenny Lee Bakery, a beloved institution in McKees Rocks, Pennsylvania, run by the Baker family for generations. It details the bakery's founding in 1941, its role in the community, and the passion for baking passed down through generations. The story takes a tragic turn with a devastating fire and a recession, leading to the closure and demolition of the bakery. However, the narrative concludes with the current generation, Scott Baker, deciding to rebuild the business with a modern, wholesale-focused approach.",
  "chapters": [
    {
      "start_time": "00:00:00",
      "end_time": "00:00:15",
      "description": "Scott Baker introduces himself and his family's deep-rooted connection to the McKees Rocks community through the Jenny Lee Bakery, which his grandfather opened in 1941."
    },
    {
      "start_time": "00:00:15",
      "end_time": "00:00:31",
      "description": "A long-time employee and customer, Donna, shares fond memories of visiting the bakery for treats after church and later working there herself."
    }
  ]
}

from vlmrun.client import VLMRun

# Initialize the VLMRun client
client = VLMRun(api_key="<VLMRUN_API_KEY>")

# Caption the video
response = client.agent.completions.create(
    model="vlmrun-orion-1:auto",
    messages=[
        {
          "role": "user",
          "content": [
            {"type": "text", "text": "Parse this video"},
            {"type": "video_url", "video_url": {"url": "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/bakery.mp4"}}
          ]
        }
    ],
)

print(response.choices[0].message.content)

from vlmrun.client import VLMRun
from pydantic import BaseModel, Field

# Define the response schema
class ParsedVideoChapter(BaseModel):
    start_time: str = Field(..., description="Start time in HH:MM:SS format")
    end_time: str = Field(..., description="End time in HH:MM:SS format")
    description: str = Field(..., description="Description of the chapter")

class ParsedVideoResponse(BaseModel):
    topic: str = Field(..., description="Topic of the video")
    summary: str = Field(..., description="Summary of the video content")
    chapters: list[ParsedVideoChapter] = Field(..., description="Video chapters")

# Initialize the VLMRun client
client = VLMRun(api_key="<VLMRUN_API_KEY>")

# Parse the video with structured output
response = client.agent.completions.create(
    model="vlmrun-orion-1:auto",
    messages=[
        {
          "role": "user",
          "content": [
            {"type": "text", "text": "Parse this video"},
            {"type": "video_url", "video_url": {"url": "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/bakery.mp4"}}
          ]
        }
    ],
    response_format={"type": "json_schema", "schema": ParsedVideoResponse.model_json_schema()}
)

# Validate the response
result = ParsedVideoResponse.model_validate_json(response.choices[0].message.content)
# >>> ParsedVideoResponse(topic="...", summary="...", chapters=[...])

import { VlmRun } from "vlmrun";

const client = new VlmRun({
  apiKey: "<VLMRUN_API_KEY>",
  baseURL: "https://api.vlm.run/v1"
});

const response = await client.agent.completions.create({
  model: "vlmrun-orion-1:auto",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "Parse this video" },
        { type: "video_url", video_url: { url: "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/bakery.mp4" } }
      ]
    }
  ]
});

console.log(response.choices[0].message.content);

import { VlmRun } from "vlmrun";
import { z } from "zod";
import { zodToJsonSchema } from "zod-to-json-schema";

// Define the response schema with Zod
const ParsedVideoResponseSchema = z.object({
  topic: z.string().describe("Topic of the video"),
  summary: z.string().describe("Summary of the video content"),
  chapters: z.array(z.object({
    start_time: z.string().describe("Start time in HH:MM:SS format"),
    end_time: z.string().describe("End time in HH:MM:SS format"),
    description: z.string().describe("Description of the chapter")
  })).describe("Video chapters")
});

// Initialize the VLMRun client
const client = new VlmRun({
  apiKey: "<VLMRUN_API_KEY>",
  baseURL: "https://api.vlm.run/v1"
});

// Parse the video with structured output
const response = await client.agent.completions.create({
  model: "vlmrun-orion-1:auto",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "Parse this video" },
        { type: "video_url", video_url: { url: "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/bakery.mp4" } }
      ]
    }
  ],
  response_format: {
    type: "json_schema",
    schema: zodToJsonSchema(ParsedVideoResponseSchema)
  }
});

const result = ParsedVideoResponseSchema.parse(JSON.parse(response.choices[0].message.content));