[PR] avformat: add iTerm2 inline image protocol muxer (PR #23541)

PR #23541 opened by Zhao Zhili (quink)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23541
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23541.patch

# Summary of changes

Add a muxer that wraps encoded image in the iTerm2 inline image protocol
(OSC 1337) so ffmpeg can play video directly in an iTerm2 terminal. The
output is a self-contained byte stream: it can be played live or saved
to a file and replayed with cat.

A screenshot of ssh to a remote machine then run `./ffmpeg -re -i ~/video/bunny.mp4 -f iterm2 -tmux 1 -` on that machine
![image](/attachments/85e4b522-71ee-40d4-8e38-d0e5ec8c42f9)


>From 866e045f8b06160f18650039c29e7fbec64c7227 Mon Sep 17 00:00:00 2001
From: Zhao Zhili <[email protected]>
Date: Fri, 19 Jun 2026 21:34:06 +0800
Subject: [PATCH] avformat: add iTerm2 inline image protocol muxer

Add a muxer that wraps encoded image in the iTerm2 inline image protocol
(OSC 1337) so ffmpeg can play video directly in an iTerm2 terminal. The
output is a self-contained byte stream: it can be played live or saved
to a file and replayed with cat.
---
 Changelog                |   1 +
 doc/muxers.texi          |  52 +++++++++++
 libavformat/Makefile     |   1 +
 libavformat/allformats.c |   1 +
 libavformat/iterm2enc.c  | 180 +++++++++++++++++++++++++++++++++++++++
 libavformat/version.h    |   4 +-
 6 files changed, 237 insertions(+), 2 deletions(-)
 create mode 100644 libavformat/iterm2enc.c

diff --git a/Changelog b/Changelog
index 2ad3ee255f..9782fae96c 100644
--- a/Changelog
+++ b/Changelog
@@ -18,6 +18,7 @@ version <next>:
 - Remove ogg/celt parsing
 - Bitstream filter to split Dolby Vision multi-layer HEVC
 - Add AMF hardware memory mapping support.
+- iTerm2 inline image protocol muxer
 
 
 version 8.1:
diff --git a/doc/muxers.texi b/doc/muxers.texi
index 5056a3e3d6..092dc13ed9 100644
--- a/doc/muxers.texi
+++ b/doc/muxers.texi
@@ -2722,6 +2722,58 @@ computer-generated compositions.
 
 This muxer accepts a single audio stream containing PCM data.
 
+@section iterm2
+iTerm2 inline image protocol muxer.
+
+This muxer writes video frames as OSC 1337 inline images for display in
+terminals that support the iTerm2 image protocol. Use @option{-re} to limit
+the output rate to the source framerate; without it, frames are emitted as
+fast as they are encoded, which is usually not desired for live display.
+
+Frames are sent with the multipart form of the protocol, which splits each
+image across several short control sequences. This avoids the per-sequence
+size limit that otherwise discards large frames, and requires iTerm2 3.5 or
+newer.
+
+The output is a self-contained byte stream and can be redirected to a file.
+Replaying the file with @command{cat} displays the images in the terminal.
+
+@subsection Options
+@table @option
+@item display_width @var{size}
+Set the displayed image width. @var{size} can be @samp{auto}, @var{N} terminal
+cells, @var{N}px pixels, or @var{N}% of the terminal width. When unset, the
+terminal derives the width from the image.
+
+@item display_height @var{size}
+Set the displayed image height. @var{size} uses the same syntax as
+@option{display_width}. When unset, the terminal derives the height from the
+image.
+
+@item keep_aspect @var{bool}
+Preserve the input aspect ratio when scaling. Default is enabled.
+
+@item tmux @var{bool}
+Wrap image data in tmux DCS passthrough. This requires a tmux version whose
+passthrough sequence size limit is large enough for image data, with
+passthrough enabled via @command{tmux set -g allow-passthrough on}. Default is
+disabled.
+@end table
+
+@subsection Examples
+
+Display a video in an iTerm2 terminal:
+@example
+ffmpeg -re -i input.mp4 -f iterm2 -
+@end example
+
+Scale the displayed image to 40 terminal cells tall. Inside tmux, enable
+passthrough first with @command{tmux set -g allow-passthrough on}, then add
+@option{tmux}:
+@example
+ffmpeg -re -i input.mp4 -f iterm2 -display_height 40 -tmux 1 -
+@end example
+
 @section ivf
 On2 IVF muxer.
 
diff --git a/libavformat/Makefile b/libavformat/Makefile
index 0db0c7c2a9..70b62f9724 100644
--- a/libavformat/Makefile
+++ b/libavformat/Makefile
@@ -337,6 +337,7 @@ OBJS-$(CONFIG_IPU_DEMUXER)               += ipudec.o rawdec.o
 OBJS-$(CONFIG_IRCAM_DEMUXER)             += ircamdec.o ircam.o pcm.o
 OBJS-$(CONFIG_IRCAM_MUXER)               += ircamenc.o ircam.o rawenc.o
 OBJS-$(CONFIG_ISS_DEMUXER)               += iss.o
+OBJS-$(CONFIG_ITERM2_MUXER)              += iterm2enc.o
 OBJS-$(CONFIG_IV8_DEMUXER)               += iv8.o
 OBJS-$(CONFIG_IVF_DEMUXER)               += ivfdec.o
 OBJS-$(CONFIG_IVF_MUXER)                 += ivfenc.o
diff --git a/libavformat/allformats.c b/libavformat/allformats.c
index af7eea5e5c..05624e66eb 100644
--- a/libavformat/allformats.c
+++ b/libavformat/allformats.c
@@ -242,6 +242,7 @@ extern const FFInputFormat  ff_ircam_demuxer;
 extern const FFOutputFormat ff_ircam_muxer;
 extern const FFOutputFormat ff_ismv_muxer;
 extern const FFInputFormat  ff_iss_demuxer;
+extern const FFOutputFormat ff_iterm2_muxer;
 extern const FFInputFormat  ff_iv8_demuxer;
 extern const FFInputFormat  ff_ivf_demuxer;
 extern const FFOutputFormat ff_ivf_muxer;
diff --git a/libavformat/iterm2enc.c b/libavformat/iterm2enc.c
new file mode 100644
index 0000000000..5503b08266
--- /dev/null
+++ b/libavformat/iterm2enc.c
@@ -0,0 +1,180 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * Copyright (c) 2026 Zhao Zhili <[email protected]>
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <string.h>
+
+#include "libavutil/base64.h"
+#include "libavutil/macros.h"
+#include "libavutil/mem.h"
+#include "libavutil/opt.h"
+#include "avformat.h"
+#include "mux.h"
+
+/* iTerm2 inline image protocol: https://iterm2.com/documentation-images.html */
+
+#define ESC "\033"
+
+#define SYNC_BEGIN    ESC "[?2026h"
+#define SYNC_END      ESC "[?2026l"
+#define CURSOR_SAVE   ESC "7"
+#define CURSOR_RESTORE ESC "8"
+#define CURSOR_HOME   ESC "[H"
+#define CURSOR_BOTTOM ESC "[999H"
+
+#define OSC_START     ESC "]1337;"
+#define BEL           "\a"
+#define ST            ESC "\\"
+
+/* tmux requires DCS passthrough with ST termination and ESC doubling */
+#define TMUX_DCS ESC "Ptmux;"
+
+/* iTerm2 and tmux silently drop a single OSC sequence >= 1 MiB, so split the
+ * image into chunks below that limit. Old tmux capped a sequence at 256 bytes,
+ * but the tiny chunks that would require flood the tmux parser and freeze the
+ * terminal, so we do not support such versions. */
+#define FILEPART_CHUNK   ((1 << 20) - 4096)
+
+#define WRITE_LITERAL(pb, str) avio_write(pb, (const unsigned char *)(str), \
+                                           sizeof(str) - 1)
+
+typedef struct ITerm2Context {
+    const AVClass *class;
+    char *display_width;
+    char *display_height;
+    int  keep_aspect;
+    int  tmux;
+    char *b64;
+    unsigned b64_size;
+} ITerm2Context;
+
+static void osc_open(ITerm2Context *c, AVIOContext *pb)
+{
+    if (c->tmux)
+        WRITE_LITERAL(pb, TMUX_DCS ESC);
+    WRITE_LITERAL(pb, OSC_START);
+}
+
+static void osc_close(ITerm2Context *c, AVIOContext *pb)
+{
+    WRITE_LITERAL(pb, BEL);
+    if (c->tmux)
+        WRITE_LITERAL(pb, ST);
+}
+
+static void write_image(ITerm2Context *c, AVIOContext *pb, int size)
+{
+    size_t b64_len = strlen(c->b64);
+
+    osc_open(c, pb);
+    WRITE_LITERAL(pb, "MultipartFile=");
+    avio_printf(pb, "inline=1;size=%d", size);
+    if (c->display_width && c->display_width[0])
+        avio_printf(pb, ";width=%s", c->display_width);
+    if (c->display_height && c->display_height[0])
+        avio_printf(pb, ";height=%s", c->display_height);
+    if (!c->keep_aspect)
+        WRITE_LITERAL(pb, ";preserveAspectRatio=0");
+    osc_close(c, pb);
+
+    for (size_t off = 0; off < b64_len; off += FILEPART_CHUNK) {
+        size_t n = FFMIN(FILEPART_CHUNK, b64_len - off);
+
+        osc_open(c, pb);
+        WRITE_LITERAL(pb, "FilePart=");
+        avio_write(pb, c->b64 + off, n);
+        osc_close(c, pb);
+    }
+
+    osc_open(c, pb);
+    WRITE_LITERAL(pb, "FileEnd");
+    osc_close(c, pb);
+}
+
+static int iterm2_write_packet(AVFormatContext *s, AVPacket *pkt)
+{
+    ITerm2Context *c = s->priv_data;
+
+    av_fast_malloc(&c->b64, &c->b64_size, AV_BASE64_SIZE(pkt->size));
+    if (!c->b64)
+        return AVERROR(ENOMEM);
+    if (!av_base64_encode(c->b64, c->b64_size, pkt->data, pkt->size))
+        return AVERROR(EINVAL);
+
+    /* Synchronized output swaps the frame in atomically. */
+    WRITE_LITERAL(s->pb, SYNC_BEGIN CURSOR_SAVE CURSOR_HOME);
+
+    write_image(c, s->pb, pkt->size);
+
+    WRITE_LITERAL(s->pb, CURSOR_RESTORE SYNC_END);
+
+    avio_flush(s->pb);
+
+    return 0;
+}
+
+static int iterm2_write_trailer(AVFormatContext *s)
+{
+    WRITE_LITERAL(s->pb, CURSOR_BOTTOM "\n");
+
+    return 0;
+}
+
+static av_cold void iterm2_deinit(AVFormatContext *s)
+{
+    ITerm2Context *c = s->priv_data;
+    av_freep(&c->b64);
+}
+
+#define OFFSET(x) offsetof(ITerm2Context, x)
+#define ENC AV_OPT_FLAG_ENCODING_PARAM
+
+static const AVOption options[] = {
+    { "display_width", "on-screen width (auto, N cells, Npx, N%%)",
+        OFFSET(display_width),  AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+    { "display_height", "on-screen height (auto, N cells, Npx, N%%)",
+        OFFSET(display_height), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC },
+    { "keep_aspect", "preserve aspect ratio when scaling",
+        OFFSET(keep_aspect), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, ENC },
+    { "tmux", "wrap image in tmux DCS passthrough, requires tmux set -g allow-passthrough on",
+        OFFSET(tmux), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, ENC },
+    { NULL },
+};
+
+static const AVClass iterm2_class = {
+    .class_name = "iTerm2 muxer",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+    .category   = AV_CLASS_CATEGORY_MUXER,
+};
+
+const FFOutputFormat ff_iterm2_muxer = {
+    .p.name         = "iterm2",
+    .p.long_name    = NULL_IF_CONFIG_SMALL("iTerm2 inline image protocol"),
+    .priv_data_size = sizeof(ITerm2Context),
+    .p.audio_codec  = AV_CODEC_ID_NONE,
+    .p.video_codec  = AV_CODEC_ID_MJPEG,
+    .write_packet   = iterm2_write_packet,
+    .write_trailer  = iterm2_write_trailer,
+    .deinit         = iterm2_deinit,
+    .flags_internal = FF_OFMT_FLAG_MAX_ONE_OF_EACH,
+    .p.flags        = AVFMT_NOTIMESTAMPS | AVFMT_NODIMENSIONS,
+    .p.priv_class   = &iterm2_class,
+};
diff --git a/libavformat/version.h b/libavformat/version.h
index bbb2fc7d87..de9cc8e31d 100644
--- a/libavformat/version.h
+++ b/libavformat/version.h
@@ -31,8 +31,8 @@
 
 #include "version_major.h"
 
-#define LIBAVFORMAT_VERSION_MINOR  19
-#define LIBAVFORMAT_VERSION_MICRO 101
+#define LIBAVFORMAT_VERSION_MINOR  20
+#define LIBAVFORMAT_VERSION_MICRO 100
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
                                                LIBAVFORMAT_VERSION_MINOR, \
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]