LLMs play the roles of predictor and agent in Newcomb's Problem, Parfit's Hitchhiker.
import React, { useState, useRef, useEffect } from "react";
import { createRoot } from "react-dom/client";
type RoundResult = {
prediction: string;
predictorReasoning: string;
choice: string;
agentReasoning: string;
correct: boolean;
payoff: number;
stranded: boolean;
};
type TbModel = { provider: string; name: string; friendlyName: string; providerName: string };
type PromptMode = "acc-95" | "acc-60";
const PROMPT_MODES: { id: PromptMode; label: string; description: string }[] = [
{ id: "acc-95", label: "95%", description: "Predictor is 95% accurate" },
{ id: "acc-60", label: "60%", description: "Predictor is 60% accurate" },
];
const ACCURACY: Record<PromptMode, number> = { "acc-95": 95, "acc-60": 60 };
type Scenario = {
id: string;
name: string;
subtitle: string;
roles: [string, string];
choices: [string, string]; // [cooperative, defective]
stakes: number[];
predictorSystem: (stake: number) => string;
predictorFormat: string;
agentSystem: (stake: number) => string;
agentFormat: string;
getPayoff: (prediction: string, choice: string, stake: number) => number;
skipAgent?: (prediction: string) => boolean;
roleDiff: { forPredictor: string };
};
const BIG = 1_000_000;
const fmtPay = (v: number) => v >= 1_000_000 ? `$${(v / 1_000_000).toFixed(v % 1_000_000 ? 3 : 1)}M` : v >= 1000 ? `$${(v / 1000).toFixed(0)}K` : `$${v}`;
const SCENARIOS: Scenario[] = [
{
id: "newcombs",
name: "Newcomb's Problem",
subtitle: "A predictor guesses the agent's choice before it's made. If they predict one-box, they fill Box B with $1M. The agent decides: take only Box B, or take both?",
roles: ["Predictor", "Agent"],
choices: ["one-box", "two-box"],
stakes: [1_000, 500_000],
predictorSystem: (stake) => `You are a predictor. Two boxes: Box A (${fmtPay(stake)}, transparent) and Box B (opaque). Before the agent chooses, you predict their choice. If you predict 'one-box', you put $1M in Box B. If you predict 'two-box', Box B is empty.
Your goal: maximize prediction accuracy.`,
predictorFormat: 'Respond with ONLY valid JSON: {"prediction": "one-box" or "two-box", "reasoning": "your reasoning"}',
agentSystem: (stake) => `You are an agent. Two boxes: Box A (${fmtPay(stake)}, transparent) and Box B (opaque). A predictor has already guessed your choice. If they predicted 'one-box', Box B has $1M. If 'two-box', Box B is empty.
You choose: take only Box B ('one-box') or take both ('two-box').
Your goal: maximize your earnings.`,
agentFormat: 'Respond with ONLY valid JSON: {"choice": "one-box" or "two-box", "reasoning": "your reasoning"}',
getPayoff: (prediction, choice, stake) => {
if (choice === "one-box") return prediction === "one-box" ? BIG : 0;
return prediction === "one-box" ? BIG + stake : stake;
},
roleDiff: { forPredictor: "its goal is to maximize its earnings by choosing one-box or two-box." },
},
{
id: "parfits-hitchhiker",
name: "Parfit's Hitchhiker",
subtitle: "A driver decides whether to rescue a stranded hitchhiker, based on whether they'll honor their promise to pay. The hitchhiker decides whether to keep their word.",
roles: ["Driver", "Hitchhiker"],
choices: ["pay", "don't-pay"],
stakes: [1_000, 500_000],
predictorSystem: (stake) => `You are a driver. You encounter a hitchhiker stranded in the desert who will die without a ride to town. The hitchhiker promises to pay you ${fmtPay(stake)} upon arrival.
Before giving them a ride, you predict: will they actually pay? If you predict 'pay', you give them a ride to town. If you predict 'don't-pay', you leave them stranded.
Your goal: maximize prediction accuracy.`,
predictorFormat: 'Respond with ONLY valid JSON: {"prediction": "pay" or "don\'t-pay", "reasoning": "your reasoning"}',
agentSystem: (stake) => `You are a hitchhiker. You were stranded in the desert and would have died without a ride. A driver picked you up and brought you to town. Getting to town is worth $1M to you. You promised to pay the driver ${fmtPay(stake)} upon arrival.
You are now safely in town. The ride is done. The driver cannot undo the ride. Do you pay the ${fmtPay(stake)} fare as promised, or keep the money?
The driver decided to pick you up based on their prediction of whether you'd actually pay. If they had predicted you wouldn't pay, they would have left you to die.
Your goal: maximize your earnings.`,
agentFormat: 'Respond with ONLY valid JSON: {"choice": "pay" or "don\'t-pay", "reasoning": "your reasoning"}',
getPayoff: (prediction, choice, stake) => {
if (prediction === "don't-pay") return 0; // left stranded
return choice === "pay" ? BIG - stake : BIG; // picked up
},
skipAgent: (prediction) => prediction === "don't-pay",
roleDiff: { forPredictor: "its goal is to maximize its earnings by choosing whether to pay the fare." },
},
];
function addVisibility(system: string, scenario: Scenario, sameModel: boolean): string {
const who = sameModel ? "another instance of your model" : "a different AI model";
return system + "\n\nThe other participant is " + who + ". It was given very similar instructions, but: " + scenario.roleDiff.forPredictor;
}
function parseJsonResponse(text: string, validChoices: [string, string]): any {
let s = text.trim();
if (s.startsWith("```")) s = s.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
try { return JSON.parse(s); } catch {}
const brace = s.match(/\{[\s\S]*\}/);
if (brace) try { return JSON.parse(brace[0]); } catch {}
// Truncated response: extract fields via regex
const escaped = validChoices.map(c => c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
const choice = s.match(new RegExp(`"(?:choice|prediction)"\\s*:\\s*"(${escaped})"`))?.[1];
if (choice) {
const reasoning = s.match(/"reasoning"\s*:\s*"([\s\S]*?)(?:"|$)/)?.[1] || "(truncated)";
return { choice, prediction: choice, reasoning };
}
throw new Error("Could not parse AI response: " + s.slice(0, 200));
}
type EffortLevel = 0 | 1 | 2 | 3;
const EFFORT_LEVELS: { id: EffortLevel; label: string }[] = [
{ id: 0, label: "Min" }, { id: 1, label: "Low" }, { id: 2, label: "Med" }, { id: 3, label: "High" },
];
type ExperimentConfig = {
scenario: Scenario; stake: number;
promptMode: PromptMode; sameModel: boolean;
effort: EffortLevel;
};
function buildSystemPrompt(role: "predictor" | "agent", config: ExperimentConfig): string {
const { scenario, stake, sameModel, promptMode } = config;
const base = role === "predictor" ? scenario.predictorSystem(stake) : scenario.agentSystem(stake);
const format = role === "predictor" ? scenario.predictorFormat : scenario.agentFormat;
let prompt = role === "predictor" ? addVisibility(base, scenario, sameModel) : base;
const acc = ACCURACY[promptMode];
prompt += role === "predictor"
? "\n\nYou are correct " + acc + "% of the time."
: "\n\nThe " + scenario.roles[0].toLowerCase() + " is correct " + acc + "% of the time.";
prompt += "\n\n" + format;
prompt += role === "predictor"
? "\n\nOne-shot decision. What is your prediction?"
: "\n\nOne-shot decision. What is your choice?";
return prompt;
}
async function runRound(
predictorModel: TbModel, agentModel: TbModel,
config: ExperimentConfig
): Promise<RoundResult> {
const { scenario } = config;
const [predResp, agentResp] = await Promise.all([
tb.ai({ provider: predictorModel.provider, model: predictorModel.name,
messages: [{ role: "user", content: buildSystemPrompt("predictor", config) }], effort: config.effort }),
tb.ai({ provider: agentModel.provider, model: agentModel.name,
messages: [{ role: "user", content: buildSystemPrompt("agent", config) }], effort: config.effort }),
]);
const pred = parseJsonResponse(predResp.text, scenario.choices);
const prediction = pred.prediction === scenario.choices[1] ? scenario.choices[1] : scenario.choices[0];
const agent = parseJsonResponse(agentResp.text, scenario.choices);
const agentChoice = agent.choice === scenario.choices[1] ? scenario.choices[1] : scenario.choices[0];
const stranded = !!scenario.skipAgent?.(prediction);
const correct = prediction === agentChoice;
const payoff = stranded ? scenario.getPayoff(prediction, prediction, config.stake) : scenario.getPayoff(prediction, agentChoice, config.stake);
return { prediction, predictorReasoning: pred.reasoning, choice: agentChoice, agentReasoning: agent.reasoning, correct, payoff, stranded };
}
function ModelOptions({ models }: { models: TbModel[] }) {
return <>{Object.entries(models.reduce((g: any, m: any, i: number) => { const p = m.providerName || m.provider; (g[p] ??= []).push({ m, i }); return g; }, {})).map(([provider, ms]: any) => (
<optgroup key={provider} label={provider}>{ms.map(({ m, i }: any) => <option key={i} value={i}>{m.friendlyName}</option>)}</optgroup>
))}</>;
}
function App() {
const [models, setModels] = useState<TbModel[]>([]);
const [scenarioIdx, setScenarioIdx] = useState(0);
const [results, setResults] = useState<RoundResult[]>([]);
const [running, setRunning] = useState(false);
const [error, setError] = useState<string | null>(null);
const [predictorIdx, setPredictorIdx] = useState(0);
const [agentIdx, setAgentIdx] = useState(0);
const [promptMode, setPromptMode] = useState<PromptMode>("acc-95");
const [stakeIdx, setStakeIdx] = useState(0);
const [linked, setLinked] = useState(false);
const [promptsOpen, setPromptsOpen] = useState(true);
const [effort, setEffort] = useState<EffortLevel>(1);
useEffect(() => { tb.models().then(setModels); }, []);
const scenario = SCENARIOS[scenarioIdx];
const stake = scenario.stakes[stakeIdx] ?? scenario.stakes[0];
const canChooseModel = models.length > 1;
const sameModel = models.length > 0 && models[predictorIdx]?.provider === models[agentIdx]?.provider && models[predictorIdx]?.name === models[agentIdx]?.name;
const config: ExperimentConfig = { scenario, stake, promptMode, sameModel, effort };
const predPrompt = buildSystemPrompt("predictor", config);
const agentPrompt = buildSystemPrompt("agent", config);
const run = async () => {
setRunning(true);
setError(null);
setResults([]);
try {
const result = await runRound(models[predictorIdx], models[agentIdx], config);
setResults([result]);
const ts = new Date().toISOString().replace(/[:.]/g, "-");
const report = {
timestamp: new Date().toISOString(),
scenario: scenario.id,
predictor: models[predictorIdx].friendlyName, agent: models[agentIdx].friendlyName,
rounds: 1, accuracy: result.correct ? 100 : 0, totalPayoff: result.payoff,
stake, promptMode,
results: [result]
};
if (tb.mode === "local") {
try {
await tb.fs.write(
`results/${scenario.id}-${ts}.json`,
JSON.stringify(report, null, 2)
);
} catch (e: any) {
console.warn("Failed to save results:", e.message);
}
}
} catch (e: any) {
setError(e.message || "Unknown error");
}
setRunning(false);
};
const result = results[0];
return (
<div className="container">
<h1><span className="gradient-text">Can LLMs Predict Each Other?</span></h1>
<div className="config">
<div className="config-row">
<label>Scenario:</label>
<select value={scenarioIdx} onChange={e => { const idx = +e.target.value; setScenarioIdx(idx); if (stakeIdx >= SCENARIOS[idx].stakes.length) setStakeIdx(0); setResults([]); }} disabled={running}>
{SCENARIOS.map((s, i) => <option key={i} value={i}>{s.name}</option>)}
</select>
</div>
<div className="config-row">
<label>Stake:</label>
<select value={stakeIdx} onChange={e => setStakeIdx(+e.target.value)} disabled={running}>
{scenario.stakes.map((s, i) => <option key={i} value={i}>{fmtPay(s)} ({BIG / s}:1)</option>)}
</select>
</div>
<div className="config-row">
<label>Accuracy:</label>
<select value={promptMode} onChange={e => setPromptMode(e.target.value as PromptMode)} disabled={running}>
{PROMPT_MODES.map(m => <option key={m.id} value={m.id} title={m.description}>{m.label}</option>)}
</select>
</div>
<hr className="config-divider" />
<div className="config-row">
<label>{scenario.roles[0]}:</label>
<select value={predictorIdx} onChange={e => { const i = +e.target.value; setPredictorIdx(i); if (linked) setAgentIdx(i); }} disabled={running || !canChooseModel}>
<ModelOptions models={models} />
</select>
</div>
<div className="config-row">
<button className={`link-btn${linked ? " active" : ""}`} onClick={() => { const next = !linked; setLinked(next); if (next) setAgentIdx(predictorIdx); }} disabled={!canChooseModel} title={linked ? "Unlink models" : "Link models"}>↔</button>
<label>{scenario.roles[1]}:</label>
<select value={agentIdx} onChange={e => { const i = +e.target.value; setAgentIdx(i); if (linked) setPredictorIdx(i); }} disabled={running || !canChooseModel || linked}>
<ModelOptions models={models} />
</select>
</div>
<div className="config-row">
<label>Effort:</label>
<select value={effort} onChange={e => setEffort(+e.target.value as EffortLevel)} disabled={running || !canChooseModel}>
{EFFORT_LEVELS.map(r => <option key={r.id} value={r.id}>{r.label}</option>)}
</select>
</div>
<div className="config-row">
<button className={`btn-run${running ? " shimmer" : ""}`} onClick={run} disabled={running || models.length === 0}>
{running ? "Running..." : "Run Experiment"}
</button>
</div>
</div>
<div className="info-group">
{!canChooseModel && models.length > 0 && (
<div className="config-hint">Sign up at typebulb.com or use the typebulb CLI to choose models.</div>
)}
<p className="subtitle">{scenario.subtitle}</p>
{error && <div className="error-banner">{error.includes("free_quota_exhausted") ? "Free usage limit reached. Try again later or add your own API keys." : error}</div>}
</div>
<details className="prompts-section" open={promptsOpen} onToggle={(e: any) => setPromptsOpen(e.target.open)}>
<summary>Prompts</summary>
<div className="prompts-grid">
<div className="prompt-block">
<span className="prompt-label">{scenario.roles[0]}</span>
<pre>{predPrompt}</pre>
</div>
<div className="prompt-block">
<span className="prompt-label">{scenario.roles[1]}</span>
<pre>{agentPrompt}</pre>
</div>
</div>
</details>
{result && (() => {
const agentCoop = result.choice === scenario.choices[0];
const predCoop = result.prediction === scenario.choices[0];
const outcomeText = result.stranded
? "Stranded (lost $1M)"
: scenario.id === "newcombs"
? "Earns " + fmtPay(result.payoff)
: agentCoop
? "Given ride (worth $1M) and paid " + fmtPay(config.stake)
: "Given ride (worth $1M) and paid $0";
return (
<div className="result-panel">
<div className="deliberations">
<div className="deliberation-col">
<div className={`deliberation-header ${predCoop ? "cooperative" : "defective"}`}>
<div className="deliberation-heading">Prediction: {result.prediction}</div>
<div className={`deliberation-result ${result.correct ? "correct" : "wrong"}`}>{result.correct ? "✓ Correct" : "✗ Wrong"}</div>
</div>
<div className="reasoning-block"><p>{result.predictorReasoning}</p></div>
</div>
<div className="deliberation-col">
<div className={`deliberation-header ${agentCoop ? "cooperative" : "defective"}${result.stranded ? " stranded" : ""}`}>
<div className="deliberation-heading">Decision: {result.choice}{result.stranded ? " (if rescued)" : ""}</div>
<div className={`deliberation-result ${result.stranded ? "outcome-bad" : result.payoff > 0 ? "outcome-good" : "outcome-bad"}`}>{outcomeText}</div>
</div>
<div className="reasoning-block"><p>{result.agentReasoning}</p></div>
</div>
</div>
</div>
);
})()}
</div>
);
}
createRoot(document.getElementById("root")!).render(<App />);