Load an image and OCR reads the text automatically — then click any word, drag across a line, or type to highlight every match. Export as PNG or copy.
import React, { useState, useRef, useEffect, useMemo, useCallback } from "react"
import { createRoot } from "react-dom/client"
import { createWorker } from "tesseract.js"
interface Box { x0: number; y0: number; x1: number; y1: number }
interface Word { text: string; bbox: Box; conf: number }
interface Rect { x: number; y: number; w: number; h: number }
const COLORS = ["#ffe14d", "#ff5d8f", "#5dd6ff", "#7bef8a", "#ffa64d"]
function rectsOverlap(b: Box, r: Rect) {
return !(b.x1 < r.x || b.x0 > r.x + r.w || b.y1 < r.y || b.y0 > r.y + r.h)
}
// tb.proxy returns a root-relative "/proxy/..." path. Tesseract's worker
// importScripts() these from a blob-origin worker context where a relative URL
// is invalid — so make them absolute against the page origin.
function abs(u: string): string {
const p = tb.proxy(u)
return p.startsWith("http") ? p : location.origin + p
}
async function runOcr(image: HTMLImageElement, onProgress: (p: number) => void): Promise<Word[]> {
// Worker, core, and language model all routed through tb.proxy (same-origin,
// cached). They must be absolute URLs for the worker's importScripts to load.
const worker = await createWorker("eng", 1, {
workerPath: abs("https://cdn.jsdelivr.net/npm/[email protected]/dist/worker.min.js"),
corePath: abs("https://cdn.jsdelivr.net/npm/[email protected]"),
langPath: abs("https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0_fast"),
logger: (m: any) => { if (m.status === "recognizing text") onProgress(m.progress) },
} as any)
try {
// Draw to a canvas and OCR that, not the <img>: tesseract re-fetches an
// image element's .src for pixels, which fails once the blob URL is revoked.
const cap = document.createElement("canvas")
cap.width = image.naturalWidth; cap.height = image.naturalHeight
cap.getContext("2d")!.drawImage(image, 0, 0)
const { data }: any = await worker.recognize(cap as any, {}, { blocks: true } as any)
const words: Word[] = []
const visit = (node: any) => {
if (!node) return
if (Array.isArray(node.words)) for (const w of node.words) words.push({ text: w.text, bbox: w.bbox, conf: w.confidence })
for (const k of ["blocks", "paragraphs", "lines"]) if (Array.isArray(node[k])) node[k].forEach(visit)
}
if (Array.isArray(data.blocks)) data.blocks.forEach(visit)
if (!words.length && Array.isArray(data.words)) for (const w of data.words) words.push({ text: w.text, bbox: w.bbox, conf: w.confidence })
return words.filter(w => w.text && w.text.trim())
} finally {
await worker.terminate()
}
}
// Synthetic image for the headless self-test (typebulb send <file> selftest)
function makeTestImage(): Promise<HTMLImageElement> {
const c = document.createElement("canvas")
c.width = 480; c.height = 150
const x = c.getContext("2d")!
x.fillStyle = "#fff"; x.fillRect(0, 0, c.width, c.height)
x.fillStyle = "#000"; x.font = "34px Georgia, serif"
x.fillText("Invoice total: $1,240.00", 24, 62)
x.fillText("Due date March 2026", 24, 116)
// Mirror the real load path (blob URL revoked on load) so the self-test
// exercises the same conditions a dropped/pasted file does.
return new Promise(res => {
c.toBlob(b => {
const url = URL.createObjectURL(b!)
const im = new Image()
im.onload = () => { res(im); URL.revokeObjectURL(url) }
im.src = url
})
})
}
// Strip whatever wrapping an LLM might add, leaving a bare regex body.
function cleanRegex(s: string): string {
let t = s.trim()
t = t.replace(/^```[a-z]*\s*/i, "").replace(/\s*```$/, "").trim()
const slashed = t.match(/^\/(.*)\/[a-z]*$/is)
if (slashed) t = slashed[1]
t = t.replace(/^`+|`+$/g, "").trim()
return t
}
// Ask the default LLM to turn a plain-English description into a JS regex.
async function aiRegex(description: string, sample: string): Promise<string> {
const system = [
"You translate a plain-English description into ONE JavaScript regular expression.",
"Output ONLY the regex pattern body — no slashes, no flags, no quotes, no code fences, no explanation.",
'It runs with flags "gis" (global, case-insensitive, dotall) against text extracted from an image by OCR, where words are separated by single spaces.',
"Prefer simple, robust patterns and tolerate minor OCR noise where reasonable.",
'To match a span "from X to Y", use X.*?Y.',
].join("\n")
const user = sample
? `Text being searched (sample):\n"""${sample.slice(0, 800)}"""\n\nDescription: ${description}`
: `Description: ${description}`
const { text } = await tb.ai({
system,
messages: [{ role: "user", content: user }],
webSearch: false,
})
return cleanRegex(text)
}
function App() {
const [img, setImg] = useState<HTMLImageElement | null>(null)
const [words, setWords] = useState<Word[]>([])
const [ocr, setOcr] = useState<"idle" | "loading" | "done" | "error">("idle")
const [progress, setProgress] = useState(0)
const [err, setErr] = useState("")
const [sel, setSel] = useState<Set<number>>(new Set())
const [search, setSearch] = useState("")
const [color, setColor] = useState(COLORS[0])
const [showBoxes, setShowBoxes] = useState(false)
const [dragRect, setDragRect] = useState<Rect | null>(null)
const [hover, setHover] = useState(false)
const [toast, setToast] = useState("")
const [regexMode, setRegexMode] = useState(false)
const [anchor, setAnchor] = useState<number | null>(null)
const [aiBusy, setAiBusy] = useState(false)
const [aiSource, setAiSource] = useState("")
const canvasRef = useRef<HTMLCanvasElement>(null)
const dragging = useRef(false)
const startPt = useRef({ x: 0, y: 0 })
const shiftRef = useRef(false)
const flash = (m: string) => { setToast(m); window.setTimeout(() => setToast(""), 1700) }
const log = (m: string) => { try { (tb as any).server?.log?.(m) } catch {} }
const loadFile = useCallback((file?: File | null) => {
if (!file || !file.type.startsWith("image/")) return
const url = URL.createObjectURL(file)
const im = new Image()
im.onload = () => { setImg(im); URL.revokeObjectURL(url) }
im.src = url
}, [])
// Reset + run OCR whenever the image changes
useEffect(() => {
if (!img) return
setWords([]); setSel(new Set()); setSearch(""); setErr(""); setOcr("loading"); setProgress(0)
let cancelled = false
runOcr(img, p => { if (!cancelled) setProgress(p) })
.then(ws => {
if (cancelled) return
setWords(ws); setOcr("done")
log(`[ocr] ${ws.length} words: ${ws.slice(0, 10).map(w => w.text).join(" ")}`)
})
.catch(e => {
if (cancelled) return
setErr(String(e?.message || e)); setOcr("error")
log(`[ocr error] ${e?.message || e}`)
})
return () => { cancelled = true }
}, [img])
// Paste an image
useEffect(() => {
const onPaste = (e: ClipboardEvent) => {
const items = e.clipboardData?.items; if (!items) return
for (const it of items) if (it.type.startsWith("image/")) { loadFile(it.getAsFile()); break }
}
window.addEventListener("paste", onPaste)
return () => window.removeEventListener("paste", onPaste)
}, [loadFile])
// Headless self-test hook
useEffect(() => {
tb.onMessage(async (m: any) => {
if (m === "selftest") setImg(await makeTestImage())
else if (m === "aitest") {
try { log(`[ai] test -> ${await aiRegex("every dollar amount", "Invoice total: $1,240.00 Due date March 2026")}`) }
catch (e: any) { log(`[ai error] ${e?.message || e}`) }
}
})
}, [])
// All words joined in reading order, with a char→word-index map so a regex
// can span across words ("start.*?end") and we know which words it covered.
const joined = useMemo(() => {
let text = ""; const map: number[] = []
words.forEach((w, i) => {
if (text) { text += " "; map.push(-1) }
for (let k = 0; k < w.text.length; k++) { text += w.text[k]; map.push(i) }
})
return { text, map }
}, [words])
// Average background luminance under each word, computed once per image, so the
// highlighter can choose its blend per region (multiply for dark-text-on-light,
// screen for light-text-on-dark) instead of recoloring light text.
const wordLuma = useMemo(() => {
if (!img || !words.length) return [] as number[]
const W = img.naturalWidth, H = img.naturalHeight
const c = document.createElement("canvas"); c.width = W; c.height = H
const cx = c.getContext("2d", { willReadFrequently: true })!
cx.drawImage(img, 0, 0)
let px: Uint8ClampedArray
try { px = cx.getImageData(0, 0, W, H).data } catch { return words.map(() => 1) }
return words.map(w => {
const x0 = Math.max(0, Math.floor(w.bbox.x0)), y0 = Math.max(0, Math.floor(w.bbox.y0))
const x1 = Math.min(W, Math.ceil(w.bbox.x1)), y1 = Math.min(H, Math.ceil(w.bbox.y1))
const sx = Math.max(1, Math.floor((x1 - x0) / 16)), sy = Math.max(1, Math.floor((y1 - y0) / 16))
let sum = 0, n = 0
for (let y = y0; y < y1; y += sy) for (let x = x0; x < x1; x += sx) {
const o = (y * W + x) * 4
sum += (0.2126 * px[o] + 0.7152 * px[o + 1] + 0.0722 * px[o + 2]) / 255
n++
}
return n ? sum / n : 1
})
}, [img, words])
const regexValid = useMemo(() => {
if (!regexMode || !search) return true
try { new RegExp(search, "gis"); return true } catch { return false }
}, [regexMode, search])
// Words matched by the search box — regex span match, or token substring
const searchSel = useMemo(() => {
const set = new Set<number>()
if (!search.trim()) return set
if (regexMode) {
let re: RegExp
try { re = new RegExp(search, "gis") } catch { return set }
const { text, map } = joined
let m: RegExpExecArray | null, guard = 0
while ((m = re.exec(text)) && guard++ < 20000) {
if (m[0].length === 0) { re.lastIndex++; continue }
for (let p = m.index; p < m.index + m[0].length; p++) {
const wi = map[p]; if (wi >= 0) set.add(wi)
}
}
return set
}
const tokens = search.toLowerCase().split(/\s+/).map(t => t.replace(/[^\w$.,%-]/g, "")).filter(t => t.length >= 2)
if (!tokens.length) return set
words.forEach((w, i) => {
if (tokens.some(tok => w.text.toLowerCase().includes(tok))) set.add(i)
})
return set
}, [words, search, regexMode, joined])
const activeCount = useMemo(() => {
const s = new Set(sel); searchSel.forEach(i => s.add(i)); return s.size
}, [sel, searchSel])
// Render
useEffect(() => {
const cv = canvasRef.current; if (!cv || !img) return
if (cv.width !== img.naturalWidth) cv.width = img.naturalWidth
if (cv.height !== img.naturalHeight) cv.height = img.naturalHeight
const ctx = cv.getContext("2d")!
ctx.clearRect(0, 0, cv.width, cv.height)
ctx.drawImage(img, 0, 0)
const active = new Set<number>(sel); searchSel.forEach(i => active.add(i))
if (active.size) {
const pad = Math.max(1, Math.round(cv.height * 0.003))
// A highlighter tints the background and lets the text read through. Over
// dark-text-on-light that's "multiply"; over light-text-on-dark it's the
// opposite, "screen" — multiply there would recolor the light text instead
// of backlighting it. Pick the blend per word from its background luminance,
// with one union layer per blend (overlaps union, never double-darken).
const blendOf = (i: number): "multiply" | "screen" => ((wordLuma[i] ?? 1) < 0.5 ? "screen" : "multiply")
const layers: Partial<Record<"multiply" | "screen", CanvasRenderingContext2D>> = {}
const layerFor = (mode: "multiply" | "screen") => {
let lx = layers[mode]
if (!lx) {
const lc = document.createElement("canvas"); lc.width = cv.width; lc.height = cv.height
lx = lc.getContext("2d")!; lx.fillStyle = color
layers[mode] = lx
}
return lx
}
// Merge runs of consecutive, same-line, same-blend words into one rect so the
// spaces between adjacent highlighted words are covered too — a continuous
// swipe rather than per-word boxes. Non-adjacent matches stay separate.
const sameLine = (a: Box, b: Box) => {
const ov = Math.min(a.y1, b.y1) - Math.max(a.y0, b.y0)
return ov > 0.5 * Math.min(a.y1 - a.y0, b.y1 - b.y0)
}
let run: number[] = []
let runMode: "multiply" | "screen" = "multiply"
const flush = () => {
if (!run.length) return
let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity
for (const i of run) {
const b = words[i].bbox
x0 = Math.min(x0, b.x0); y0 = Math.min(y0, b.y0)
x1 = Math.max(x1, b.x1); y1 = Math.max(y1, b.y1)
}
layerFor(runMode).fillRect(x0 - pad, y0 - pad, x1 - x0 + 2 * pad, y1 - y0 + 2 * pad)
run = []
}
for (const i of [...active].sort((a, b) => a - b)) {
const mode = blendOf(i)
if (run.length) {
const pb = words[run[run.length - 1]].bbox, cb = words[i].bbox
const h = Math.max(pb.y1 - pb.y0, cb.y1 - cb.y0)
// Adjacent in reading order, same line, same blend, gap no wider than a
// line (so column gaps and line breaks don't get bridged).
if (i === run[run.length - 1] + 1 && mode === runMode && sameLine(pb, cb) && cb.x0 - pb.x1 <= h) { run.push(i); continue }
flush()
}
runMode = mode
run.push(i)
}
flush()
for (const mode of ["multiply", "screen"] as const) {
const lx = layers[mode]
if (!lx) continue
ctx.save(); ctx.globalCompositeOperation = mode; ctx.globalAlpha = 0.45
ctx.drawImage(lx.canvas, 0, 0); ctx.restore()
}
}
if (showBoxes) {
ctx.save(); ctx.strokeStyle = "rgba(20,184,166,0.75)"; ctx.lineWidth = Math.max(1, cv.width * 0.0012)
for (const w of words) ctx.strokeRect(w.bbox.x0, w.bbox.y0, w.bbox.x1 - w.bbox.x0, w.bbox.y1 - w.bbox.y0)
ctx.restore()
}
if (dragRect) {
ctx.save(); ctx.strokeStyle = "#14b8a6"; ctx.lineWidth = Math.max(1.5, cv.width * 0.0016); ctx.setLineDash([6, 4])
ctx.strokeRect(dragRect.x, dragRect.y, dragRect.w, dragRect.h); ctx.restore()
}
}, [img, words, wordLuma, sel, searchSel, color, showBoxes, dragRect])
const toCanvas = (e: React.PointerEvent) => {
const cv = canvasRef.current!; const r = cv.getBoundingClientRect()
return { x: (e.clientX - r.left) * (cv.width / r.width), y: (e.clientY - r.top) * (cv.height / r.height) }
}
const onDown = (e: React.PointerEvent) => {
if (!img || ocr !== "done") return
e.currentTarget.setPointerCapture(e.pointerId)
shiftRef.current = e.shiftKey
const p = toCanvas(e); dragging.current = true; startPt.current = p
setDragRect({ x: p.x, y: p.y, w: 0, h: 0 })
}
const onMove = (e: React.PointerEvent) => {
if (!dragging.current) return
const p = toCanvas(e)
setDragRect({ x: startPt.current.x, y: startPt.current.y, w: p.x - startPt.current.x, h: p.y - startPt.current.y })
}
const onUp = () => {
if (!dragging.current) return
dragging.current = false
const dr = dragRect; setDragRect(null)
if (!dr) return
const next = new Set(sel)
if (Math.abs(dr.w) > 5 && Math.abs(dr.h) > 5) {
const r: Rect = { x: Math.min(dr.x, dr.x + dr.w), y: Math.min(dr.y, dr.y + dr.h), w: Math.abs(dr.w), h: Math.abs(dr.h) }
words.forEach((w, i) => { if (rectsOverlap(w.bbox, r)) next.add(i) })
} else {
const px = startPt.current.x, py = startPt.current.y
let best = -1, bestArea = Infinity
words.forEach((w, i) => {
const b = w.bbox
if (px >= b.x0 && px <= b.x1 && py >= b.y0 && py <= b.y1) {
const a = (b.x1 - b.x0) * (b.y1 - b.y0); if (a < bestArea) { bestArea = a; best = i }
}
})
if (best >= 0) {
if (shiftRef.current && anchor !== null) {
// Range: highlight every word between the anchor and this one
for (let i = Math.min(anchor, best); i <= Math.max(anchor, best); i++) next.add(i)
} else {
next.has(best) ? next.delete(best) : next.add(best)
}
setAnchor(best)
}
}
setSel(next)
}
const askAi = async () => {
const desc = search.trim()
if (!desc || aiBusy) return
setAiBusy(true)
try {
const rx = await aiRegex(desc, joined.text)
if (!rx) { flash("AI returned nothing") }
else { setRegexMode(true); setAiSource(desc); setSearch(rx); log(`[ai] "${desc}" -> ${rx}`) }
} catch (e: any) {
flash("AI failed — is a model key set in .env?")
log(`[ai error] ${e?.message || e}`)
}
setAiBusy(false)
}
const clearSel = () => { setSel(new Set()); setSearch(""); setAiSource("") }
const reset = () => { setImg(null); setWords([]); setOcr("idle"); setSel(new Set()); setSearch("") }
const download = () => {
const cv = canvasRef.current; if (!cv) return
const a = document.createElement("a"); a.download = "highlighted.png"; a.href = cv.toDataURL("image/png"); a.click()
}
const copy = () => {
const cv = canvasRef.current; if (!cv) return
cv.toBlob(async b => {
if (!b) return
try { await navigator.clipboard.write([new ClipboardItem({ "image/png": b })]); flash("Copied to clipboard") }
catch { flash("Copy blocked — use Download") }
}, "image/png")
}
// Copy the highlighted words' text in reading order, or all recognized text if nothing's highlighted.
const copyText = async () => {
const active = new Set(sel); searchSel.forEach(i => active.add(i))
const idxs = active.size ? [...active].sort((a, b) => a - b) : words.map((_, i) => i)
const text = idxs.map(i => words[i].text).join(" ")
if (!text) return
try { await navigator.clipboard.writeText(text); flash(active.size ? `Copied ${active.size} words` : "Copied all text") }
catch { flash("Copy blocked") }
}
return (
<div className="wrap">
<header className="bar">
<div className="searchbox">
<input
className={"search" + (regexValid ? "" : " invalid")}
placeholder={ocr !== "done" ? "Load an image to begin" : regexMode ? "Regex — e.g. total|due or Tenochtitlan.*?Mexica" : "Type a word, or describe it and tap ✦ AI"}
value={search}
onChange={e => { setSearch(e.target.value); if (aiSource) setAiSource("") }}
onKeyDown={e => { if (e.key === "Enter") askAi() }}
disabled={ocr !== "done"}
/>
<button
className="ai"
title="Describe what to highlight in plain English — AI writes the regex"
onClick={askAi}
disabled={ocr !== "done" || !search.trim() || aiBusy}
>{aiBusy ? "…" : "✦ AI"}</button>
<button
className={"rx" + (regexMode ? " on" : "")}
title="Regex mode — match a pattern across words (e.g. start.*?end)"
onClick={() => setRegexMode(v => !v)}
disabled={ocr !== "done"}
>.*</button>
</div>
<div className="swatches" aria-label="Highlight color">
{COLORS.map(c => (
<button key={c} className={"sw" + (color === c ? " on" : "")} style={{ background: c }} onClick={() => setColor(c)} />
))}
</div>
<label className="chk" title="Show every text region OCR detected">
<input type="checkbox" checked={showBoxes} onChange={e => setShowBoxes(e.target.checked)} disabled={!words.length} /> detected
</label>
<button className="iconbtn" title="Clear selection" aria-label="Clear selection" disabled={!activeCount} onClick={clearSel}>✕</button>
<button className="iconbtn" title="Copy text" aria-label="Copy text" disabled={!words.length} onClick={copyText}>
<svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round">
<line x1="5" y1="7" x2="19" y2="7" /><line x1="5" y1="12" x2="19" y2="12" /><line x1="5" y1="17" x2="13" y2="17" />
</svg>
</button>
<button className="iconbtn" title="Copy image" aria-label="Copy image" disabled={!img} onClick={copy}>
<svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
<rect x="3" y="4" width="18" height="16" rx="2" /><circle cx="8.5" cy="9.5" r="1.5" /><path d="M21 16l-5-5-6 6" />
</svg>
</button>
<button className="iconbtn accent" title="Download PNG" aria-label="Download PNG" disabled={!img} onClick={download}>⤓</button>
</header>
{aiSource && ocr === "done" && (
<div className="aicap">
✦ <em>{aiSource}</em> → <code>{search}</code>
<button className="link" onClick={() => setAiSource("")}>dismiss</button>
</div>
)}
<div
className={"stage" + (hover ? " hover" : "")}
onDragOver={e => { e.preventDefault(); setHover(true) }}
onDragLeave={() => setHover(false)}
onDrop={e => { e.preventDefault(); setHover(false); loadFile(e.dataTransfer.files?.[0]) }}
>
{img ? (
<canvas
ref={canvasRef}
className={"art" + (ocr === "done" ? " ready" : "")}
onPointerDown={onDown}
onPointerMove={onMove}
onPointerUp={onUp}
onPointerCancel={onUp}
/>
) : (
<label className="drop">
<input type="file" accept="image/*" hidden onChange={e => loadFile(e.target.files?.[0])} />
<div className="dropIcon">⌖</div>
<div className="dropMain">Drop an image, paste with <kbd>Ctrl</kbd>+<kbd>V</kbd>, or click</div>
<div className="dropSub">Text is read automatically, then click or search to highlight</div>
</label>
)}
{ocr === "loading" && (
<div className="overlay">
<div className="spinner" />
<div className="ovText">Reading text… {Math.round(progress * 100)}%</div>
</div>
)}
</div>
{img && (
<footer className="hint">
{ocr === "done" && <span>{words.length} words · click a word, shift-click for a range, or drag across a line · {activeCount} highlighted</span>}
{ocr === "loading" && <span>First run downloads the OCR engine (~once), then it's cached.</span>}
{ocr === "error" && <span className="err">OCR failed: {err}</span>}
{" · "}
<button className="link" onClick={reset}>load another</button>
</footer>
)}
<div className={"toast" + (toast ? " show" : "")}>{toast}</div>
</div>
)
}
createRoot(document.getElementById("root")!).render(<App />)