Vision-Language Models

from cactus import cactus_init, cactus_complete, cactus_destroy
import json

model = cactus_init("weights/lfm2-vl-450m", None, False)

messages = json.dumps([{
    "role": "user",
    "content": "Describe this image in detail",
    "images": ["photo.jpg"]
}])

result = json.loads(cactus_complete(model, messages, None, None, None))
print(result["response"])

cactus_destroy(model)

import Foundation

let model = try cactusInit("/path/to/weights/lfm2-vl-1.6b", nil, false)
defer { cactusDestroy(model) }

let messages = #"
[
    {
        "role": "user",
        "content": "What objects are in this image?",
        "images": ["/path/to/image.jpg"]
    }
]
"#

let resultJson = try cactusComplete(model, messages, nil, nil, nil)
if let data = resultJson.data(using: .utf8),
   let result = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
    print(result["response"] as? String ?? "")
}

messages = json.dumps([{
    "role": "user",
    "content": "Compare these two images. What are the differences?",
    "images": ["image1.jpg", "image2.jpg"]
}])

result = json.loads(cactus_complete(model, messages, None, None, None))
print(result["response"])

conversation = []

# First turn: Show image
conversation.append({
    "role": "user",
    "content": "What's in this image?",
    "images": ["scene.jpg"]
})

messages = json.dumps(conversation)
result = json.loads(cactus_complete(model, messages, None, None, None))
conversation.append({"role": "assistant", "content": result["response"]})

# Second turn: Follow-up question (no new image)
conversation.append({
    "role": "user",
    "content": "What color is the car?"
})

messages = json.dumps(conversation)
result = json.loads(cactus_complete(model, messages, None, None, None))
print(result["response"])

tasks = [
    "How many people are in this photo?",
    "What is the dominant color?",
    "Is this photo taken indoors or outdoors?",
    "What time of day is it?"
]

for question in tasks:
    messages = json.dumps([{
        "role": "user",
        "content": question,
        "images": ["photo.jpg"]
    }])
    
    result = json.loads(cactus_complete(model, messages, None, None, None))
    print(f"Q: {question}")
    print(f"A: {result['response']}\n")

def caption_image(image_path, detail_level="medium"):
    prompts = {
        "brief": "Caption this image in one short sentence.",
        "medium": "Describe this image.",
        "detailed": "Provide a detailed description of everything visible in this image."
    }
    
    messages = json.dumps([{
        "role": "user",
        "content": prompts[detail_level],
        "images": [image_path]
    }])
    
    result = json.loads(cactus_complete(model, messages, None, None, None))
    return result["response"]

brief = caption_image("photo.jpg", "brief")
detailed = caption_image("photo.jpg", "detailed")
print(f"Brief: {brief}")
print(f"Detailed: {detailed}")

messages = json.dumps([{
    "role": "user",
    "content": "List all objects visible in this image with their approximate locations.",
    "images": ["scene.jpg"]
}])

result = json.loads(cactus_complete(model, messages, None, None, None))
print(result["response"])

messages = json.dumps([{
    "role": "user",
    "content": "Extract all text visible in this image.",
    "images": ["document.jpg"]
}])

result = json.loads(cactus_complete(model, messages, None, None, None))
print(result["response"])

from cactus import cactus_image_embed
import numpy as np

# Generate image embedding
embedding = cactus_image_embed(model, "image.jpg")
print(f"Embedding dimension: {len(embedding)}")

# Compute similarity between images
img1_emb = cactus_image_embed(model, "img1.jpg")
img2_emb = cactus_image_embed(model, "img2.jpg")

similarity = np.dot(img1_emb, img2_emb)
print(f"Similarity: {similarity:.3f}")

try:
    result = json.loads(cactus_complete(model, messages, None, None, None))
    if not result["success"]:
        print(f"Error: {result['error']}")
except RuntimeError as e:
    print(f"Failed to process image: {e}")