Custom Benchmarks

def evaluate(program_path: str) -> dict:
    """
    Score a program.

    Args:
        program_path: Path to .py file (code tasks) or .txt file (prompt tasks)

    Returns:
        Dictionary with at least 'combined_score' key
    """
    # Your scoring logic here
    return {
        "combined_score": 0.73,  # Required: 0.0 to 1.0 (or higher)
        # Optional: add more metrics
        "accuracy": 0.85,
        "latency_ms": 120,
    }

import time
import numpy as np
import sys
import os
from importlib import __import__
import itertools

BENCHMARK = 0.036529889880030156
TOL = 1e-6
NUM_POINTS = 11

def check_inside_triangle_wtol(points: np.ndarray, tol: float = 1e-6):
    """Check that all points are inside the equilateral triangle."""
    for x, y in points:
        cond1 = y >= -tol
        cond2 = np.sqrt(3) * x <= np.sqrt(3) - y + tol
        cond3 = y <= np.sqrt(3) * x + tol

        if not (cond1 and cond2 and cond3):
            raise ValueError(
                f"Point ({x}, {y}) is outside the triangle (tolerance: {tol})."
            )

def triangle_area(a: np.array, b: np.array, c: np.array) -> float:
    return np.abs(a[0] * (b[1] - c[1]) + b[0] * (c[1] - a[1]) + c[0] * (a[1] - b[1])) / 2

def evaluate(program_path: str):
    try:
        # 1. Import the program dynamically
        abs_program_path = os.path.abspath(program_path)
        program_dir = os.path.dirname(abs_program_path)
        module_name = os.path.splitext(os.path.basename(program_path))[0]

        points = None
        try:
            sys.path.insert(0, program_dir)
            program = __import__(module_name)

            # 2. Run the program and time it
            start_time = time.time()
            points = program.heilbronn_triangle11()
            end_time = time.time()
            eval_time = end_time - start_time
        finally:
            if program_dir in sys.path:
                sys.path.remove(program_dir)

        # 3. Validate output
        if not isinstance(points, np.ndarray):
            points = np.array(points)

        if points.shape != (NUM_POINTS, 2):
            raise ValueError(f"Invalid shape: {points.shape}, expected {(NUM_POINTS, 2)}")

        check_inside_triangle_wtol(points, TOL)

        # 4. Compute metric
        a = np.array([0, 0])
        b = np.array([1, 0])
        c = np.array([0.5, np.sqrt(3) / 2])
        min_triangle_area = min(
            [triangle_area(p1, p2, p3) for p1, p2, p3 in itertools.combinations(points, 3)]
        )
        min_area_normalized = min_triangle_area / triangle_area(a, b, c)

        # 5. Return metrics
        return {
            "min_area_normalized": float(min_area_normalized),
            "combined_score": float(min_area_normalized / BENCHMARK),
            "eval_time": float(eval_time),
        }
    except Exception as e:
        # 6. Always return on failure
        return {"combined_score": 0.0, "error": str(e)}

import numpy as np

# EVOLVE-BLOCK-START
def heilbronn_triangle11():
    """Generate 11 points in an equilateral triangle."""
    # Simple uniform random placement (LLM will improve this)
    points = []
    for i in range(11):
        x = np.random.uniform(0, 1)
        y = np.random.uniform(0, np.sqrt(3)/2)
        points.append([x, y])
    return np.array(points)
# EVOLVE-BLOCK-END

# Helper functions outside the evolve block are preserved
def helper_function():
    pass

Answer the question by reasoning step by step.

language: text
diff_based_generation: false

prompt:
  system_message: |-
    You are optimizing a prompt for question answering.
    Generate a prompt that improves accuracy on HotPotQA.

# System prompt tells the LLM what to optimize
prompt:
  system_message: |-
    You are an expert at geometric optimization.
    Your goal is to place 11 points in an equilateral triangle
    to maximize the minimum triangle area.

# Search algorithm
search:
  type: topk
  num_context_programs: 4

# Evaluation settings
evaluator:
  timeout: 10  # seconds per evaluation

# LLM settings
llm:
  models:
    - name: gpt-4
      temperature: 0.7