Skip to main content
The client.video object allows you to process video files and extract structured data.

Generate Predictions

import { VlmRun } from "vlmrun";
import type { PredictionResponse } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

// Process a video file with a predefined schema using file ID
const response: PredictionResponse = await client.video.generate({
  fileId: "file_abc123",
  domain: "video.transcription",
});

// Process a video from URL
const responseFromUrl: PredictionResponse = await client.video.generate({
  url: "https://example.com/video.mp4",
  domain: "video.transcription",
});

console.log(`Prediction ID: ${response.id}`);
console.log(`Status: ${response.status}`);

Upload and Process Video

import { VlmRun } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

// First upload the video file
const file = await client.files.upload({
  filePath: "path/to/video.mp4",
});

// Then process the uploaded video
const response = await client.video.generate({
  fileId: file.id,
  domain: "video.transcription",
});

// Wait for completion if needed
if (response.status !== "completed") {
  const completed = await client.predictions.wait(response.id, 300);
  console.log(`Result: ${JSON.stringify(completed.response)}`);
}

Batch Processing

For longer videos or when you want asynchronous processing:
import { VlmRun } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

// Process video in batch mode
const response = await client.video.generate({
  fileId: "file_abc123",
  domain: "video.transcription",
  batch: true,
});

// Poll for completion
const completed = await client.predictions.wait(response.id, 600);
console.log(`Transcription: ${JSON.stringify(completed.response)}`);

Custom Configuration

import { VlmRun, GenerationConfig } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

// Process with custom configuration
const response = await client.video.generate({
  fileId: "file_abc123",
  domain: "video.dashcam-analytics",
  config: {
    detail: "hi",
    grounding: true,
  },
  metadata: {
    environment: "prod",
    sessionId: "session_123",
  },
});

Auto-Generate Schema

Generate a schema automatically from a video file:
import { VlmRun } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

// Auto-generate schema from video
const schemaResponse = await client.video.schema({
  fileId: "file_abc123",
});

console.log(`Generated schema: ${JSON.stringify(schemaResponse.response)}`);

Get Usage

import { VlmRun } from "vlmrun";
import type { CreditUsage } from "vlmrun";

const client = new VlmRun({
  apiKey: "your-api-key",
});

const response = await client.video.generate({
  fileId: "file_abc123",
  domain: "video.transcription",
});

// Access usage information
const usage: CreditUsage | undefined = response.usage;
if (usage) {
  console.log(`Elements processed: ${usage.elements_processed}`);
  console.log(`Element type: ${usage.element_type}`);
  console.log(`Credits used: ${usage.credits_used}`);
}

TypeScript Interfaces

interface FilePredictionParams {
  fileId?: string;
  url?: string;
  model?: string;
  domain: string;
  batch?: boolean;
  config?: GenerationConfigParams;
  metadata?: RequestMetadataParams;
  callbackUrl?: string;
}

interface GenerationConfigParams {
  detail?: "auto" | "hi" | "lo";
  responseModel?: ZodType;
  jsonSchema?: Record<string, any> | null;
  confidence?: boolean;
  grounding?: boolean;
  gqlStmt?: string | null;
}

interface RequestMetadataParams {
  environment?: "dev" | "staging" | "prod";
  sessionId?: string | null;
  allowTraining?: boolean;
}

interface PredictionResponse {
  id: string;
  created_at: string;
  completed_at?: string;
  response?: any;
  status: JobStatus;
  message?: string;
  usage?: CreditUsage;
}

interface CreditUsage {
  elements_processed?: number;
  element_type?: "image" | "page" | "video" | "audio";
  credits_used?: number;
}

Error Handling

import { VlmRun, ApiError } from "vlmrun";

try {
  const response = await client.video.generate({
    fileId: "invalid_file_id",
    domain: "video.transcription",
  });
} catch (error) {
  if (error instanceof ApiError) {
    console.error("API Error:", error.message);
    console.error("Status:", error.http_status);
  } else {
    console.error("Unknown Error:", error);
  }
}