Multi-modal Artifacts

from pathlib import Path
from pydantic import BaseModel, Field
from PIL import Image
from vlmrun.client import VLMRun
from vlmrun.types import ImageRef

client = VLMRun(api_key="<VLMRUN_API_KEY>")

# Define a response model with an ImageRef field
class BlurredImageResponse(BaseModel):
    image: ImageRef = Field(..., description="The blurred image")

# Make a chat completion request
response = client.agent.completions.create(
    model="vlmrun-orion-1:auto",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "Blur all the faces in this image"},
                {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}}
            ]
        }
    ],
    response_format={
        "type": "json_schema",
        "schema": BlurredImageResponse.model_json_schema()
    }
)

# Parse the response
result = BlurredImageResponse.model_validate_json(response.choices[0].message.content)

# Retrieve the artifact using session_id and object_id
blurred_image: Image.Image = client.artifacts.get(
    session_id=response.session_id,
    object_id=result.image.id
)

# Display or save the image
blurred_image.save("blurred_output.jpg")