Image → JSON

from PIL import Image
from vlmrun.client import VLMRun

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>"
)

from PIL import Image
from vlmrun.client import VLMRun
from vlmrun.client.types import GenerationConfig, AgentSkill

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>",
    config=GenerationConfig(
        skills=[AgentSkill(skill_name="<skill-name>")]
    )
)

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    domain: "<domain>",
});

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    config: {
        skills: [{ skillName: "<skill-name>" }],
    },
});

{
  "usage": {
    "elements_processed": 123,
    "element_type": "image",
    "credits_used": 123,
    "steps": 123,
    "message": "<string>",
    "duration_seconds": 0,
    "service_tier": "<string>",
    "mode_multiplier": 123,
    "standard_cost_dollars": 123,
    "cost_dollars": 123,
    "savings_dollars": 123
  },
  "id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "completed_at": "2023-11-07T05:31:56Z",
  "response": "<unknown>",
  "status": "pending",
  "domain": "<string>",
  "fallback": false,
  "error": "<string>"
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

POST

image

generate

from PIL import Image
from vlmrun.client import VLMRun

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>"
)

from PIL import Image
from vlmrun.client import VLMRun
from vlmrun.client.types import GenerationConfig, AgentSkill

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>",
    config=GenerationConfig(
        skills=[AgentSkill(skill_name="<skill-name>")]
    )
)

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    domain: "<domain>",
});

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    config: {
        skills: [{ skillName: "<skill-name>" }],
    },
});

{
  "usage": {
    "elements_processed": 123,
    "element_type": "image",
    "credits_used": 123,
    "steps": 123,
    "message": "<string>",
    "duration_seconds": 0,
    "service_tier": "<string>",
    "mode_multiplier": 123,
    "standard_cost_dollars": 123,
    "cost_dollars": 123,
    "savings_dollars": 123
  },
  "id": "<string>",
  "created_at": "2023-11-07T05:31:56Z",
  "completed_at": "2023-11-07T05:31:56Z",
  "response": "<unknown>",
  "status": "pending",
  "domain": "<string>",
  "fallback": false,
  "error": "<string>"
}

{
  "detail": [
    {
      "loc": [
        "<string>"
      ],
      "msg": "<string>",
      "type": "<string>",
      "input": "<unknown>",
      "ctx": {}
    }
  ]
}

For all supported image domains, see the Hub Catalog.

from PIL import Image
from vlmrun.client import VLMRun

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>"
)

from PIL import Image
from vlmrun.client import VLMRun
from vlmrun.client.types import GenerationConfig, AgentSkill

client = VLMRun(api_key="<VLMRUN_API_KEY>")
response = client.image.generate(
    images=[Image.open("<path>.jpg")],
    domain="<domain>",
    config=GenerationConfig(
        skills=[AgentSkill(skill_name="<skill-name>")]
    )
)

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    domain: "<domain>",
});

import { VlmRun } from "vlmrun";

const client = new VlmRun({apiKey: "<VLMRUN_API_KEY>"});
const fileResponse = await client.files.upload(
    filePath: "<path>.jpg"
);
const response = await client.image.generate({
    fileId: fileResponse.id,
    config: {
        skills: [{ skillName: "<skill-name>" }],
    },
});

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

Request to the VLM API (i.e. structured prediction).

images

string[]

required

List of base64 encoded images or URLs to the images.

metadata

RequestMetadata · object

Optional metadata to pass to the model.

Hide child attributes

metadata.environment

enum<string>

default:dev

The environment where the request was made.

Available options:

dev,

staging,

prod

metadata.session_id

string | null

The session ID of the request

metadata.allow_logging

boolean

default:true

Whether to enable logs for this request.

metadata.allow_training

boolean

default:true

Whether the file can be used for training

metadata.allow_retention

boolean

default:true

Whether to allow retention of the data

metadata.extra

Extra · object

Extra metadata for the request (e.g. dataset_id, subset_id).

config

GenerationConfig · object

The VLM generation config to be used for //generate.

Hide child attributes

config.prompt

string | null

Additional user instructions appended to the application or skill prompt for this request.

config.detail

enum<string>

default:auto

The detail level to use for processing multimodal data.

Available options:

auto,

hi,

lo

config.json_schema

Json Schema · object | null

The overridden JSON schema to use for the model. To be used instead of the response model.

config.skills

AgentSkill · object[] | null

List of agent skills to enable for this generation request.

Hide child attributes

config.skills.type

string

default:skill_reference

The type of the skill. Use 'skill_reference' for DB-stored skills referenced by id/name. Use 'inline' to provide the skill as a base64-encoded zip bundle.

config.skills.skill_id

string | null

The unique identifier of the skill — a UUID or a name string (e.g., 'pillow', 'batch-processing').

config.skills.skill_name

string | null

Human-readable skill name for lookup (e.g., 'invoice-extraction'). Alternative to skill_id. Deprecated in favour of skill_id.

config.skills.skill_version

default:latest

The version of the skill — an integer (e.g. 2) or 'latest'.

config.skills.version

DEPRECATED: Use 'skill_version' instead. The version of the skill.

config.skills.name

string | null

Human-readable name for the inline skill (used for discovery and logging).

config.skills.description

string | null

Short description of what the inline skill does.

config.skills.source

InlineSkillSource · object | null

Source payload for inline skills. Contains the base64-encoded zip bundle with type, media_type, and data fields.

Hide child attributes

config.skills.source.data

string

required

Base64-encoded zip bundle containing the skill files.

config.skills.source.type

string

default:base64

Encoding type for the inline skill data. Currently only 'base64' is supported.

Allowed value: "base64"

config.skills.source.media_type

string

default:application/zip

MIME type of the skill bundle. Must be 'application/zip'.

config.skills.bundle

string | null

DEPRECATED: Use 'source.data' instead. Base64-encoded zip bundle containing the skill files (inline skills only).

config.gql_stmt

string | null

The GraphQL statement to use for the application. If provided, the response model will be generated from the GraphQL statement.

config.max_retries

integer

default:1

The maximum number of retries to use for the application.

config.max_tokens

integer

default:65535

The maximum number of tokens to use for the application.

config.temperature

number

default:0

The temperature to use for the application.

config.confidence

boolean

default:false

Include confidence scores in the response (included in the _metadata field).

config.grounding

boolean

default:false

Include grounding in the response (included in the _metadata field).

config.keyframes

boolean

default:false

Include keyframes in the video transcription response.

config.video_segment_duration

number | null

Duration in seconds for each video segment when chunking a video for transcription. Defaults to 150.0s.

Required range: x >= 1

config.video_frames_per_segment

integer | null

Number of frames to sample per video segment for captioning. Defaults to 8.

Required range: x >= 1

config.video_model

string | null

Model ID to use for video segment captioning (e.g. 'vlmrun-orion-1:fast'). When omitted, the server default is used.

config.video_input_mode

enum<string> | null

How to pass video to the captioning model: 'frames' extracts N JPEG frames per segment, 'native_video' sends the mp4 clip directly via video_url for models with native video understanding. Defaults to 'native_video' for Qwen deployment models, 'frames' for others.

Available options:

frames,

native_video

config.video_transcribe_audio

boolean

default:false

When True, transcribe the audio track to align segment boundaries. When False (default), skip ASR and use fixed-duration video segments only (visual-only captioning).

config.chat_context

string | null

Plain-text chat transcript (prior turns + current request) used to ground video captioning / transcription on what the user wants extracted.

config.page_indices

integer[] | null

0-indexed page indices to process for document files. If None, all pages are processed.

config.use_context_cache

boolean

default:true

Reuse cached representations of document/video content across calls. When True (default), the file is cached after the first call so repeated queries against the same file skip re-transmitting its contents. Set to False to always send the full content.

config.service_tier

enum<string> | null

Delivery tier for the request. 'standard'/'default' uses baseline rates, 'flex' applies a 50% discount with higher latency, 'priority' applies a 1.8x premium. When omitted (or 'auto'), the server default ('standard') applies. The chosen tier drives both billing and the latency/availability SLO.

Available options:

auto,

default,

standard,

flex,

priority

string

Unique identifier of the request.

created_at

string<date-time>

Date and time when the request was created (in UTC timezone)

callback_url

string<uri> | null

The URL to call when the request is completed.

Minimum string length: 1

model

default:vlm-1

The model to use for generating the response.

Available options:

vlm-1,

vlm-1:auto,

vlm-1:fast,

vlm-1:pro

domain

The domain identifier (e.g. image.caption). Optional when a skill is provided via config.skills.

Available options:

aerospace.remote-sensing,

document.invoice,

document.markdown,

document.receipt,

document.resume,

image.caption,

image.object-detection,

image.person-detection,

retail.product-catalog

batch

boolean

default:false

Whether to process the image in batch mode (async).

Response

Successful Response

Base prediction response for all API responses.

usage

CreditUsageResponse · object

The usage metrics for the request.

Hide child attributes

usage.elements_processed

integer | null

Number of elements processed.

usage.element_type

enum<string> | null

The type of element processed (e.g. image, page, video, audio).

Available options:

image,

page,

video,

audio

usage.credits_used

integer | null

Amount of total credits used.

usage.steps

integer | null

Number of steps processed, in case of agentic execution.

usage.message

string | null

The message from the credit usage job.

usage.duration_seconds

integer

default:0

Duration of the request in seconds.

usage.service_tier

string | null

Delivery tier (standard, priority, flex).

usage.mode_multiplier

number | null

Pricing multiplier applied to standard cost.

usage.standard_cost_dollars

number | null

Pre-multiplier customer cost in USD, derived from tokens and unit rates.

usage.cost_dollars

number | null

Effective customer cost in USD after the service-tier multiplier.

usage.savings_dollars

number | null

Discount in USD when using flex (standard_cost_dollars - cost_dollars).

string

Unique identifier of the response.

created_at

string<date-time>

Date and time when the request was created (in UTC timezone)

completed_at

string<date-time> | null

Date and time when the response was completed (in UTC timezone)

response

any | null

The response from the model. May be an empty dict/list when the model found no extractable content (valid for status=completed).

status

enum<string>

default:pending

The status of the job.

Available options:

pending,

enqueued,

running,

completed,

failed,

paused

domain

string | null

The domain of the prediction (e.g. document.invoice, image.caption).

fallback

boolean

default:false

True when the JSON was produced by a fallback model. False when the primary model succeeded.

error

string | null

Why the job failed. Set only when status=failed.

Getting started

Chat Completions

Generate

Agents

Files

Skills

Feedback

Authorizations

Body

Response