Mastra Compatibility

npm install @mastra/core @ai-sdk/openai-compatible zod
export VLMRUN_API_KEY="<VLMRUN_API_KEY>"

import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
import { Agent } from "@mastra/core/agent";
import { createTool } from "@mastra/core/tools";
import { z } from "zod";

const vlmrun = createOpenAICompatible({
  name: "vlmrun",
  baseURL: "https://api.vlm.run/v1/openai",
  apiKey: process.env.VLMRUN_API_KEY!,
});

const lookupPurchaseOrder = createTool({
  id: "lookup_purchase_order",
  description: "Return the purchase order your ERP has on file for an invoice number",
  inputSchema: z.object({ invoiceNumber: z.string() }),
  execute: async ({ context }) => ({
    invoiceNumber: context.invoiceNumber,
    poNumber: "PO-4417",
    total: 4560.0,
  }),
});

const agent = new Agent({
  id: "orion2-mastra-agent",
  name: "Orion-2 Mastra Agent",
  model: vlmrun.chatModel("vlmrun-orion-2:auto"),
  tools: { lookupPurchaseOrder },
  instructions:
    "Use execute_code for attached documents, images, audio, or video. " +
    "Call lookup_purchase_order only when asked to reconcile an invoice. " +
    "Never invent field values.",
});

const runOptions = () => ({
  providerOptions: {
    vlmrun: {
      session_id: crypto.randomUUID(),
      tool_execution: "hybrid",
    },
  },
});

const INVOICE =
  "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/wordpress-pdf-invoice-plugin-sample.pdf";

const out = await agent.stream(
  `Extract vendor, invoice number, date, and total from ${INVOICE}.`,
  runOptions(),
);
console.log(await out.text);

const DONUTS =
  "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/image.object-detection/donuts.png";

const out = await agent.stream(
  `Count the donuts and briefly describe the image: ${DONUTS}`,
  runOptions(),
);
console.log(await out.text);

const AUDIO =
  "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/audio.transcription-summary/two_minute_rules.mp3";

const out = await agent.stream(
  `Summarize this audio in 2 sentences: ${AUDIO}`,
  runOptions(),
);
console.log(await out.text);

const VIDEO =
  "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/video.transcription/bakery.mp4";

const out = await agent.stream(
  `Describe this video in 2 sentences: ${VIDEO}`,
  runOptions(),
);
console.log(await out.text);

const out = await agent.stream(
  `Extract the invoice number and total from ${INVOICE} with execute_code. ` +
    "Then call lookup_purchase_order once and say if the totals match.",
  runOptions(),
);
console.log(await out.text);