main
1/**
2 * Kitty Image Display Extension
3 *
4 * Displays images directly in the Kitty terminal using the Kitty Graphics Protocol
5 * (raw escape sequences via process.stdout). No kitten or external tools required
6 * for PNG files. Other formats are auto-converted to PNG via Pi's native
7 * (Photon/WASM) converter, falling back to python3 or ImageMagick if unavailable.
8 *
9 * Provides:
10 * - show_image tool: LLM can display any image in the terminal
11 * - /show-image command: quick display from the command line
12 */
13
14import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
15import { convertToPng as convertToPngNative } from "@mariozechner/pi-coding-agent";
16import { Text } from "@mariozechner/pi-tui";
17import { Type } from "@sinclair/typebox";
18import { readFileSync, existsSync, statSync } from "node:fs";
19import { execSync, execFileSync } from "node:child_process";
20import { extname, resolve } from "node:path";
21import { homedir } from "node:os";
22
23// ═══════════════════════════════════════════════════════════════════════════
24// Kitty Graphics Protocol Implementation
25// ═══════════════════════════════════════════════════════════════════════════
26
27const CHUNK_SIZE = 4096;
28
29/**
30 * Expand ~ and ~user in paths, and resolve relative paths against cwd.
31 */
32function expandPath(rawPath: string, cwd: string): string {
33 let p = rawPath.trim().replace(/^@/, "");
34 if (p.startsWith("~/") || p === "~") {
35 p = p.replace(/^~/, homedir());
36 }
37 return resolve(cwd, p);
38}
39
40/**
41 * Send image bytes to Kitty using the Graphics Protocol.
42 * Data must be raw PNG bytes.
43 *
44 * Uses direct transmission mode (a=T, f=100).
45 */
46function sendPngToKitty(pngBytes: Buffer): void {
47 const b64 = pngBytes.toString("base64");
48 const chunks: string[] = [];
49
50 for (let i = 0; i < b64.length; i += CHUNK_SIZE) {
51 chunks.push(b64.slice(i, i + CHUNK_SIZE));
52 }
53
54 for (let i = 0; i < chunks.length; i++) {
55 const isLast = i === chunks.length - 1;
56 const m = isLast ? 0 : 1;
57 // First chunk: a=T (transmit + display), f=100 (PNG format)
58 const ctrl = i === 0 ? `a=T,f=100,m=${m}` : `m=${m}`;
59 const seq = `\x1b_G${ctrl};${chunks[i]}\x1b\\`;
60 process.stdout.write(seq);
61 }
62
63 // Move to next line after the image
64 process.stdout.write("\n");
65}
66
67// Map file extensions to MIME types for the native converter.
68const MIME_BY_EXT: Record<string, string> = {
69 ".png": "image/png",
70 ".jpg": "image/jpeg",
71 ".jpeg": "image/jpeg",
72 ".gif": "image/gif",
73 ".webp": "image/webp",
74 ".bmp": "image/bmp",
75 ".tiff": "image/tiff",
76 ".tif": "image/tiff",
77};
78
79/**
80 * Convert image bytes to PNG using Pi's native (Photon/WASM) converter.
81 * Returns null if the native converter is unavailable or fails.
82 */
83async function convertToPngNativeBytes(imagePath: string): Promise<Buffer | null> {
84 const ext = extname(imagePath).toLowerCase();
85 const mimeType = MIME_BY_EXT[ext] ?? "application/octet-stream";
86 const base64 = readFileSync(imagePath).toString("base64");
87 const result = await convertToPngNative(base64, mimeType);
88 if (!result) return null;
89 return Buffer.from(result.data, "base64");
90}
91
92/**
93 * Fallback: convert any image format to PNG bytes via external tools.
94 * Tries python3 (Pillow) first, then ImageMagick `convert`.
95 */
96function convertToPngExternal(imagePath: string): Buffer {
97 // Try python3 with Pillow
98 try {
99 const script = `
100import sys
101from PIL import Image
102import io
103img = Image.open(sys.argv[1]).convert('RGBA')
104buf = io.BytesIO()
105img.save(buf, format='PNG')
106sys.stdout.buffer.write(buf.getvalue())
107`;
108 const result = execFileSync("python3", ["-c", script, imagePath], {
109 stdio: ["pipe", "pipe", "pipe"],
110 timeout: 15000,
111 });
112 return result;
113 } catch {
114 // Pillow not available, try ImageMagick
115 }
116
117 try {
118 const result = execSync(`convert "${imagePath}" PNG:-`, {
119 stdio: ["pipe", "pipe", "pipe"],
120 timeout: 15000,
121 });
122 return result;
123 } catch {
124 throw new Error("Could not convert image. Install Pillow (pip install Pillow) or ImageMagick.");
125 }
126}
127
128/**
129 * Load image at path and return PNG bytes, converting if necessary.
130 */
131async function loadAsPng(imagePath: string): Promise<{ png: Buffer; originalFormat: string }> {
132 const ext = extname(imagePath).toLowerCase();
133
134 if (ext === ".png") {
135 return { png: readFileSync(imagePath), originalFormat: "png" };
136 }
137
138 // Prefer Pi's native converter (no external dependency), fall back to tools.
139 const native = await convertToPngNativeBytes(imagePath);
140 const png = native ?? convertToPngExternal(imagePath);
141 return { png, originalFormat: ext.slice(1) || "unknown" };
142}
143
144// ═══════════════════════════════════════════════════════════════════════════
145// Extension
146// ═══════════════════════════════════════════════════════════════════════════
147
148export default function (pi: ExtensionAPI) {
149 // ─── Tool ───────────────────────────────────────────────────────────
150
151 pi.registerTool({
152 name: "show_image",
153 label: "Show Image",
154 description: `Display an image directly in the Kitty terminal using the Kitty Graphics Protocol.
155Supports PNG natively. JPEG, GIF, WEBP, BMP, and other formats are auto-converted.
156
157Use this when:
158- User asks to "display", "show", or "view" an image
159- You want to visualize a chart, diagram, screenshot, or photo
160- User is in Kitty terminal and wants inline image preview`,
161 parameters: Type.Object({
162 path: Type.String({
163 description: "Absolute or relative path to the image file to display",
164 }),
165 }),
166
167 async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
168 const imagePath = expandPath(params.path, ctx.cwd);
169
170 if (!existsSync(imagePath)) {
171 return {
172 content: [{ type: "text", text: `Image not found: ${imagePath}` }],
173 isError: true,
174 };
175 }
176
177 const stat = statSync(imagePath);
178 if (!stat.isFile()) {
179 return {
180 content: [{ type: "text", text: `Not a file: ${imagePath}` }],
181 isError: true,
182 };
183 }
184
185 try {
186 const { png, originalFormat } = await loadAsPng(imagePath);
187 const sizeMB = (stat.size / 1024 / 1024).toFixed(2);
188
189 // Write the image to the terminal via Kitty Graphics Protocol
190 process.stdout.write("\n"); // Blank line before image
191 sendPngToKitty(png);
192
193 const converted = originalFormat !== "png" ? ` (converted from ${originalFormat})` : "";
194
195 return {
196 content: [{
197 type: "text",
198 text: `Displayed image: ${imagePath} (${sizeMB} MB${converted})`,
199 }],
200 details: {
201 path: imagePath,
202 format: originalFormat,
203 sizeMB,
204 pngBytes: png.length,
205 },
206 };
207 } catch (err: any) {
208 return {
209 content: [{
210 type: "text",
211 text: `Failed to display image: ${err.message}`,
212 }],
213 isError: true,
214 };
215 }
216 },
217
218 renderCall(args, theme) {
219 return new Text(
220 theme.fg("toolTitle", theme.bold("show_image ")) + theme.fg("accent", args.path),
221 0,
222 0,
223 );
224 },
225
226 renderResult(result, _options, theme) {
227 if (result.isError) {
228 const msg = result.content?.[0]?.type === "text" ? result.content[0].text : "Error";
229 return new Text(theme.fg("error", "✗ " + msg), 0, 0);
230 }
231
232 const { details } = result;
233 if (details?.path) {
234 const fmt = details.format !== "png" ? ` → png` : "";
235 return new Text(
236 theme.fg("success", "✓ ") +
237 theme.fg("accent", details.format + fmt) +
238 theme.fg("dim", ` ${details.sizeMB} MB — `) +
239 theme.fg("muted", details.path),
240 0,
241 0,
242 );
243 }
244
245 const msg = result.content?.[0]?.type === "text" ? result.content[0].text : "";
246 return new Text(theme.fg("success", "✓ ") + msg, 0, 0);
247 },
248 });
249
250 // ─── Command ────────────────────────────────────────────────────────
251
252 pi.registerCommand("show-image", {
253 description: "Display an image in the terminal (Kitty Graphics Protocol)",
254 handler: async (args, ctx) => {
255 if (!args) {
256 ctx.ui.notify("Usage: /show-image <path>", "warning");
257 return;
258 }
259
260 const imagePath = expandPath(args, ctx.cwd);
261
262 if (!existsSync(imagePath)) {
263 ctx.ui.notify(`Image not found: ${imagePath}`, "error");
264 return;
265 }
266
267 try {
268 const { png, originalFormat } = await loadAsPng(imagePath);
269 const converted = originalFormat !== "png" ? ` (from ${originalFormat})` : "";
270
271 process.stdout.write("\n");
272 sendPngToKitty(png);
273
274 ctx.ui.notify(`Displayed ${imagePath}${converted}`, "info");
275 } catch (err: any) {
276 ctx.ui.notify(`Failed: ${err.message}`, "error");
277 }
278 },
279 });
280}