Visual Grounding

from pathlib import Path
from vlmrun.client import VLMRun
from vlmrun.client.types import GenerationConfig, PredictionResponse

client = VLMRun(api_key="...")
prediction: PredictionResponse = client.image.generate(
    file=Path("path/to/license.jpg"),
    domain="document.us-drivers-license",
    config=GenerationConfig(grounding=True),
)
print(prediction.response.model_dump_json(indent=2))

{
  "license_number": "1234567",
  "license_number_metadata": {
    "confidence": "hi",
    "bboxes": [
      {
        "xywh": [0.2, 0.15, 0.15, 0.05],
        "page": 0
      }
    ]
  },
  "license_expiration_date": "2014-01-05",
  "license_expiration_date_metadata": {
    "confidence": "hi",
    "bboxes": [
      {
        "xywh": [0.8, 0.2, 0.15, 0.05],
        "page": 0
      }
    ]
  }
  ...
}

{
  "invoice_number": "INV-2023-0042",
  "invoice_number_metadata": {
    "confidence": "hi",
    "bboxes": [
      {
        "xywh": [0.7, 0.1, 0.2, 0.05],
        "page": 1
      },
      {
        "xywh": [0.7, 0.1, 0.2, 0.05],
        "page": 2 // invoice_number can typically be found on each page of the document
      },
      ...
    ]
  },
  "total_amount": "1000",
  "total_amount_metadata": {
    "confidence": "hi",
    "bboxes": [
      {
        "xywh": [0.2, 0.2, 0.3, 0.05],
        "page": 0 // `total_amount` can typically be found on the first and last page of the document
      },
      {
        "xywh": [0.6, 0.8, 0.3, 0.05],
        "page": 6 // `total_amount` can typically be found on the first and last page of the document
      }
    ]
  }
}