Visual Grounding

from vlmrun.client import VLMRun

# Initialize the VLMRun client
client = VLMRun(api_key="<VLMRUN_API_KEY>")

# Perform visual grounding
response = client.agent.completions.create(
    model="vlmrun-orion-1:auto",
    messages=[
        {
          "role": "user",
          "content": [
            {"type": "text", "text": "Localize all the speaker names in the TV news broadcast text and visualize them on the image. Only provide one bounding box for each speaker name."},
            {"type": "image_url", "image_url": {"url": "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/media.tv-news/finance_bb_3_speakers.jpg", "detail": "auto"}}
          ]
        }
    ]
)

print(response.choices[0].message.content)
# >>> {"elements": [{"content": "HAIDI STROUD-WATTS", "xywh": [0.428, 0.217, 0.128, 0.286]}, ...]}

from vlmrun.client import VLMRun
from pydantic import BaseModel, Field

# Define the response schema
class GroundingWithText(BaseModel):
  content: str = Field(..., description="The text content")
  xywh: tuple[float, float, float, float] = Field(..., description="Bounding box (x, y, w, h)")

class GroundingResponse(BaseModel):
  elements: list[GroundingWithText] = Field(..., description="Text to visual mappings")

# Initialize the VLMRun client
client = VLMRun(api_key="<VLMRUN_API_KEY>")

# Perform visual grounding with structured output
response = client.agent.completions.create(
    model="vlmrun-orion-1:auto",
    messages=[
        {
          "role": "user",
          "content": [
            {"type": "text", "text": "Localize all the speaker names in the TV news broadcast text and visualize them on the image. Only provide one bounding box for each speaker name."},
            {"type": "image_url", "image_url": {"url": "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/media.tv-news/finance_bb_3_speakers.jpg", "detail": "auto"}}
          ]
        }
    ],
    response_format={"type": "json_schema", "schema": GroundingResponse.model_json_schema()},
)

# Validate the response
result = GroundingResponse.model_validate_json(response.choices[0].message.content)
# >>> GroundingResponse(elements=[GroundingWithText(content="...", xywh=(...)), ...])

import { VlmRun } from "vlmrun";

const client = new VlmRun({
  apiKey: "<VLMRUN_API_KEY>",
  baseURL: "https://api.vlm.run/v1"
});

const response = await client.agent.completions.create({
  model: "vlmrun-orion-1:auto",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "Localize all the speaker names in the TV news broadcast text and visualize them on the image. Only provide one bounding box for each speaker name." },
        { type: "image_url", image_url: { url: "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/media.tv-news/finance_bb_3_speakers.jpg", detail: "auto" } }
      ]
    }
  ]
});

console.log(response.choices[0].message.content);

import { VlmRun } from "vlmrun";
import { z } from "zod";
import { zodToJsonSchema } from "zod-to-json-schema";

// Define the response schema with Zod
const GroundingResponseSchema = z.object({
  elements: z.array(z.object({
    content: z.string().describe("The text content"),
    xywh: z.array(z.number()).describe("Bounding box (x, y, w, h)")
  })).describe("Text to visual mappings")
});

// Initialize the VLMRun client
const client = new VlmRun({
  apiKey: "<VLMRUN_API_KEY>",
  baseURL: "https://api.vlm.run/v1"
});

// Perform visual grounding with structured output
const response = await client.agent.completions.create({
  model: "vlmrun-orion-1:auto",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "Localize all the speaker names in the TV news broadcast text and visualize them on the image. Only provide one bounding box for each speaker name." },
        { type: "image_url", image_url: { url: "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/media.tv-news/finance_bb_3_speakers.jpg", detail: "auto" } }
      ]
    }
  ],
  response_format: {
    type: "json_schema",
    schema: zodToJsonSchema(GroundingResponseSchema)
  }
});

const result = GroundingResponseSchema.parse(JSON.parse(response.choices[0].message.content));