See how OpenAI's o200k_base tokenizer splits text into the numbered chunks a model reads, and which ids repeat.
---
format: typebulb/v1
name: Tokenizer
---
**code.tsx**
```tsx
import React, { useMemo, useState } from "react";
import { createRoot } from "react-dom/client";
import { encode, decode } from "gpt-tokenizer";
type Token = { id: number; text: string };
type Stat = { id: number; text: string; count: number; share: number };
// Golden angle: consecutive ids land far apart on the wheel, so adjacent tokens contrast.
const hueOf = (id: number) => String(Math.round((id * 137.508) % 360));
const GLYPHS: Record<string, string> = { " ": "␣", "\n": "⏎", "\r": "⏎", "\t": "⇥" };
function tokenize(text: string): Token[] {
return encode(text).map((id) => ({ id, text: decode([id]) }));
}
// Whitespace renders as a dimmed glyph: a pill holding a real newline or tab is two lines tall.
function Marked({ text }: { text: string }) {
return (
<>
{text.split(/(\s)/).map((part, i) =>
part === "" ? null : /^\s$/.test(part) ? (
<i key={i} className="ws">{GLYPHS[part] ?? "·"}</i>
) : (
<React.Fragment key={i}>{part}</React.Fragment>
)
)}
</>
);
}
function App() {
const [text, setText] = useState(
"Count them: strawberry is one token, STRAWBERRY is four, and antidisestablishmentarianism is six."
);
const [active, setActive] = useState<number | null>(null);
const tokens = useMemo(() => tokenize(text), [text]);
const stats = useMemo<Stat[]>(() => {
const byId = new Map<number, Stat>();
for (const t of tokens) {
const seen = byId.get(t.id);
if (seen) seen.count += 1;
else byId.set(t.id, { id: t.id, text: t.text, count: 1, share: 0 });
}
const items = [...byId.values()];
for (const it of items) it.share = (it.count / tokens.length) * 100;
return items.sort((a, b) => b.count - a.count || a.id - b.id);
}, [tokens]);
const maxCount = stats.length ? stats[0].count : 1;
const focus = active === null ? null : stats.find((s) => s.id === active) ?? null;
const figures: [string, string | number][] = [
["tokens", tokens.length],
["unique ids", stats.length],
["characters", text.length],
["chars / token", tokens.length ? (text.length / tokens.length).toFixed(2) : "—"],
];
return (
<div className="app">
<header>
<h1>Tokenizer</h1>
<p className="subtitle">
Models don't read words, they read numbered chunks called tokens. Here's how your
text breaks up under <code>o200k_base</code>, the tokenizer behind GPT-5.x.
</p>
</header>
<textarea
className="input"
value={text}
onChange={(e) => setText(e.target.value)}
placeholder="Type or paste text here…"
spellCheck={false}
rows={4}
/>
<p className="summary">
{figures.map(([k, v], i) => (
<React.Fragment key={k}>
{i > 0 && <span className="sep"> · </span>}
<b>{v}</b> {k}
</React.Fragment>
))}
</p>
<div className="strip" onMouseLeave={() => setActive(null)}>
{tokens.length === 0 ? (
<span className="empty">no tokens yet</span>
) : (
tokens.map((t, i) => (
<span
key={i}
className={"tok" + (active === t.id ? " on" : active === null ? "" : " dim")}
style={{ "--h": hueOf(t.id) } as React.CSSProperties}
onMouseEnter={() => setActive(t.id)}
>
<Marked text={t.text} />
</span>
))
)}
</div>
<div className="inspect">
{focus ? (
<>
<span className="chip" style={{ "--h": hueOf(focus.id) } as React.CSSProperties}>
#{focus.id}
</span>
<span className="mono"><Marked text={focus.text} /></span>
<span className="muted">
appears {focus.count}× · {focus.share.toFixed(1)}% of this text
</span>
</>
) : (
<span className="muted">Hover a token to trace every place its id appears.</span>
)}
</div>
{stats.length > 0 && (
<section className="panel">
<h2>Most frequent ids</h2>
<div className="rows">
{stats.map((s) => (
<div
key={s.id}
className={"row" + (active === s.id ? " on" : "")}
style={{ "--h": hueOf(s.id) } as React.CSSProperties}
onMouseEnter={() => setActive(s.id)}
onMouseLeave={() => setActive(null)}
>
<span className="chip">#{s.id}</span>
<span className="label"><Marked text={s.text} /></span>
<span className="track">
<span className="fill" style={{ width: `${(s.count / maxCount) * 100}%` }} />
</span>
<span className="count">
{s.count}<span className="muted"> · {s.share.toFixed(0)}%</span>
</span>
</div>
))}
</div>
</section>
)}
</div>
);
}
createRoot(document.getElementById("root")!).render(<App />);
```
**styles.css**
```css
/* Token colour is composed at the point of use, never in a variable declared on html: a
var(--h) inside such a variable substitutes against html, which has no hue, so every pill
would take the fallback and come out identical. Themes set the OKLCH lightness/chroma
knobs; each pill supplies its own hue. OKLCH because equal lightness across hues reads as
equal brightness, which is what keeps a 360° sweep looking like one pastel (or one somber)
family rather than a highlighter set. */
:root {
--bg: #f5f5f7;
--surface: #ffffff;
--sunken: #fafafa;
--text: #1b1b1f;
--muted: #70707a;
--border: #e5e5ea;
--hairline: #ededf1;
--tok-l: 0.945; --tok-c: 0.05;
--tok-bd-l: 0.87; --tok-bd-c: 0.08;
--tok-fg-l: 0.44; --tok-fg-c: 0.10;
--bar-l: 0.82; --bar-c: 0.10;
}
html[data-theme="dark"] {
--bg: #0f0f11;
--surface: #171719;
--sunken: #131316;
--text: #eaeaef;
--muted: #8b8b95;
--border: #2a2a30;
--hairline: #222227;
--tok-l: 0.30; --tok-c: 0.042;
--tok-bd-l: 0.40; --tok-bd-c: 0.055;
--tok-fg-l: 0.87; --tok-fg-c: 0.045;
--bar-l: 0.50; --bar-c: 0.07;
}
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
background: var(--bg);
color: var(--text);
font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
}
.app {
max-width: 840px;
margin: 0 auto;
padding: 28px 16px 36px;
display: flex;
flex-direction: column;
gap: 14px;
}
h1 { font-size: 20px; font-weight: 650; letter-spacing: -0.01em; }
.subtitle { color: var(--muted); margin-top: 2px; }
.subtitle code { font-family: ui-monospace, "SF Mono", "Consolas", monospace; font-size: 0.94em; }
.muted { color: var(--muted); }
/* ── Summary line ────────────────────────────────────────────────────── */
.summary { text-align: center; color: var(--muted); }
.summary b { color: var(--text); font-weight: 620; font-variant-numeric: tabular-nums; }
.sep { opacity: 0.45; }
/* ── Input ───────────────────────────────────────────────────────────── */
.input {
width: 100%;
padding: 10px 12px;
font: 14px/1.55 ui-monospace, "SF Mono", "Consolas", monospace;
background: var(--surface);
color: var(--text);
border: 1px solid var(--border);
border-radius: 10px;
resize: vertical;
transition: border-color 0.15s;
}
.input:focus { outline: none; border-color: var(--muted); }
/* ── Token strip ─────────────────────────────────────────────────────── */
.strip {
display: flex;
flex-wrap: wrap;
align-items: flex-start;
gap: 3px;
padding: 9px 10px;
min-height: 62px;
max-height: 280px;
overflow-y: auto;
background: var(--sunken);
border: 1px solid var(--border);
border-radius: 10px;
}
/* A pill must not shrink and must not wrap inside itself: flex items default to
flex-shrink: 1, and a shrunk pill with wrappable text breaks after its leading space,
which is what made "␣." render two lines tall. Fixed height + the text-box trim keeps
every pill one size with the glyphs optically centred (see the mirror's word-pill rule). */
.tok {
flex: 0 0 auto;
display: inline-block;
align-content: center;
text-align: center;
text-box: trim-both cap alphabetic;
height: 26px;
padding: 0 6px;
border-radius: 5px;
white-space: pre;
font: 14px/1 ui-monospace, "SF Mono", "Consolas", monospace;
background: oklch(var(--tok-l) var(--tok-c) var(--h, 250));
color: oklch(var(--tok-fg-l) var(--tok-fg-c) var(--h, 250));
border: 1px solid oklch(var(--tok-bd-l) var(--tok-bd-c) var(--h, 250));
cursor: default;
transition: opacity 0.12s, transform 0.12s, box-shadow 0.12s;
}
.tok.on {
transform: translateY(-1px);
box-shadow: 0 0 0 2px var(--sunken),
0 0 0 3px oklch(var(--tok-fg-l) var(--tok-fg-c) var(--h, 250));
}
.tok.dim { opacity: 0.32; }
.ws { font-style: normal; opacity: 0.45; }
.empty { color: var(--muted); font-style: italic; }
/* ── Inspector line ──────────────────────────────────────────────────── */
.inspect {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 8px;
min-height: 24px;
}
.mono { font-family: ui-monospace, "SF Mono", "Consolas", monospace; white-space: pre; }
.chip {
display: inline-block;
align-content: center;
text-align: center;
text-box: trim-both cap alphabetic;
height: 22px;
padding: 0 7px;
border-radius: 5px;
font: 13px/1 ui-monospace, "SF Mono", "Consolas", monospace;
font-variant-numeric: tabular-nums;
background: oklch(var(--tok-l) var(--tok-c) var(--h, 250));
color: oklch(var(--tok-fg-l) var(--tok-fg-c) var(--h, 250));
border: 1px solid oklch(var(--tok-bd-l) var(--tok-bd-c) var(--h, 250));
}
/* ── Frequency panel ─────────────────────────────────────────────────── */
.panel {
padding: 14px 14px 12px;
background: var(--surface);
border: 1px solid var(--border);
border-radius: 10px;
}
.panel h2 { font-size: 15px; font-weight: 620; margin-bottom: 10px; }
.rows {
display: flex;
flex-direction: column;
gap: 2px;
max-height: 340px;
overflow-y: auto;
}
.row {
display: grid;
grid-template-columns: 74px minmax(0, 120px) 1fr 78px;
align-items: center;
gap: 10px;
padding: 3px 6px;
border-radius: 7px;
transition: background 0.12s;
}
.row.on { background: var(--sunken); }
.label {
overflow: hidden;
text-overflow: ellipsis;
white-space: pre;
font-family: ui-monospace, "SF Mono", "Consolas", monospace;
}
.track {
height: 10px;
border-radius: 5px;
background: var(--hairline);
overflow: hidden;
}
.fill {
display: block;
height: 100%;
min-width: 3px;
border-radius: 5px;
background: oklch(var(--bar-l) var(--bar-c) var(--h, 250));
transition: width 0.25s ease;
}
.count { text-align: right; font-variant-numeric: tabular-nums; }
@media (max-width: 520px) {
.row { grid-template-columns: 66px minmax(0, 74px) 1fr 64px; gap: 8px; }
}
```
**index.html**
```html
<div id="root"></div>
```
**config.json**
```json
{
"description": "See how OpenAI's o200k_base tokenizer splits text into the numbered chunks a model reads, and which ids repeat.",
"dependencies": {
"react": "^19.2.8",
"react-dom": "^19.2.8",
"gpt-tokenizer": "^3.4.0"
}
}
```