B252: keep closed textures valid until the engine is done with them

Windows: closeTexture hands the texture to UnregisterTexture's callback and
frees its frame buffers there; frames pushed after close are dropped. Without
"recycle" the object is kept for the process lifetime (the session may still
hold its pointer); an engine that shuts down keeps its open textures the same
way. A duplicate createTexture key gets -1, getTexturePtr of an unknown key 0.

macOS: closeTexture drops the frame and closes the texture to later frames;
only "recycle" removes it from the plugin.

Linux: the frame handed to the engine last is freed at that engine's next
copy_pixels or once the engine has shut down, not at close: from Flutter 3.32
the raster thread can still be uploading it.

Tests: windows/test/run_tests.sh (plain, --asan, --tsan) and
linux/test/run_tests.sh (plain, --asan).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
yunchan 2026-10-27 03:19:05 +09:00
parent 8cd34a80ff
commit f38b9420ac
15 changed files with 1388 additions and 31 deletions

View file

@ -1,7 +1,8 @@
// Modified by the Kitedesk fork (2026, B212): frame buffers are freed when the
// texture is closed and when the object is finalized; a closed texture can be
// restarted for reuse; with no frame waiting, one transparent pixel is handed
// out instead of no buffer.
// Modified by the Kitedesk fork (2026, B212, B252): frame buffers are freed
// when the texture is closed (the frame last handed to the engine once the
// engine cannot be uploading it any more) and when the object is finalized; a
// closed texture can be restarted for reuse; with no frame waiting, one
// transparent pixel is handed out instead of no buffer.
#ifndef TEXTURE_RGBA_RENDERER_H
#define TEXTURE_RGBA_RENDERER_H
@ -9,6 +10,8 @@
#include <flutter_linux/flutter_linux.h>
#include <gtk/gtk.h>
#include <vector>
#include "texture_rgba_renderer_plugin.h"
G_BEGIN_DECLS
@ -59,11 +62,46 @@ static void texture_rgba_free_buffers(TextureRgba* self) {
g_atomic_int_set(&self->buffer_ready, FALSE);
}
// Frames that copy_pixels handed to an engine and whose texture was closed
// since. The engine can still be uploading such a frame: populate calls
// glTexImage2D after copy_pixels returns, and unregistering a texture does not
// wait for that (from Flutter 3.32 the engine rasterizes on its own thread). A
// frame is freed at the engine's next copy_pixels, which comes after the
// upload on the same raster thread, or once the engine has shut down.
struct RetiredFrame {
FlTextureRegistrar* engine;
uint8_t* buffer;
};
static GMutex retired_frames_mutex;
// Never destroyed: a raster thread can still run while the process exits.
static std::vector<RetiredFrame>& retired_frames =
*new std::vector<RetiredFrame>();
static void texture_rgba_free_retired_frames(FlTextureRegistrar* engine) {
g_mutex_lock(&retired_frames_mutex);
for (auto it = retired_frames.begin(); it != retired_frames.end();) {
if (it->engine == engine) {
delete[] it->buffer;
it = retired_frames.erase(it);
} else {
++it;
}
}
g_mutex_unlock(&retired_frames_mutex);
}
static void texture_rgba_terminate(TextureRgba* self) {
g_mutex_lock(&self->mutex);
// OnRgba stops copying frames and copy_pixels stops handing them out once
// terminated is set. The frame waiting was never handed out; the one handed
// out last waits in retired_frames.
g_atomic_int_set(&self->terminated, TRUE);
// The texture is unregistered, so the engine reads neither frame again, and
// OnRgba stops copying frames once terminated is set.
if (self->prior_buffer != nullptr) {
g_mutex_lock(&retired_frames_mutex);
retired_frames.push_back({self->texture_registrar, self->prior_buffer});
g_mutex_unlock(&retired_frames_mutex);
self->prior_buffer = nullptr;
}
texture_rgba_free_buffers(self);
g_mutex_unlock(&self->mutex);
}
@ -88,6 +126,8 @@ static gboolean texture_rgba_copy_pixels(FlPixelBufferTexture* texture,
TextureRgba *self = TEXTURE_RGBA(texture);
// This method is called on Render Thread. Be careful with your
// cross-thread operation.
// The engine's earlier uploads on this thread are done.
texture_rgba_free_retired_frames(self->texture_registrar);
g_mutex_lock(&self->mutex);
// @width and @height are initially stored the canvas size in Flutter.
// You must prepare your pixel buffer in RGBA format.

View file

@ -7,7 +7,9 @@
// registrar with the reference rules of the Flutter 3.24 engine, and a plugin
// registrar. Frame buffers that the plugin copies with new[] are counted.
// The engine's GL work (fl_pixel_buffer_texture_populate) is replaced by calls
// to the copy_pixels hook it would make; no GL context is needed.
// to the copy_pixels hook it would make; no GL context is needed. Reading the
// frame copy_pixels returned after the call stands for the glTexImage2D that
// populate makes then.
//
// Build and run: linux/test/run_tests.sh [--asan]
@ -47,6 +49,19 @@ int live_frames() {
return n;
}
bool frame_is_live(const void* p) {
lock_frames();
bool live = false;
for (int i = 0; i < kSlots && !live; i++) {
live = g_frames[i] == p;
}
unlock_frames();
return live;
}
// Keeps the reads of an "upload" from being optimized away.
volatile uint8_t g_uploaded;
} // namespace
void* operator new[](size_t size) {
@ -565,8 +580,9 @@ int g_failed_checks = 0;
} \
} while (0)
// D1: closing a texture frees the frame it shows and the frame still waiting
// for the next raster frame.
// D1: closing a texture frees the frame still waiting for the next raster
// frame at once, and the frame it shows at the window's next raster frame,
// when the engine cannot be uploading that frame any more.
static void test_close_frees_frame_buffers() {
const int base = live_frames();
Engine* e = engine_new();
@ -580,10 +596,80 @@ static void test_close_frees_frame_buffers() {
send_frame(ptr, 2);
CHECK(live_frames() == base + 2);
CHECK(close_texture(e, 101, 0) == 1);
CHECK(live_frames() == base + 1);
int64_t other = create_texture(e, 102);
CHECK(pull_frame(other).ok);
CHECK(live_frames() == base);
engine_shutdown(e);
}
// B252: from Flutter 3.32 the Linux engine rasterizes on its own thread (up to
// 3.29 on the platform thread, where a close cannot come in between).
// fl_pixel_buffer_texture_populate uploads the frame copy_pixels returned
// after the call, and unregistering the texture does not wait for that, so a
// close on the platform thread can come while the frame is uploading.
static void test_close_keeps_frame_being_uploaded() {
const int base = live_frames();
for (int recycle = 0; recycle <= 1; recycle++) {
Engine* e = engine_new();
int64_t key = 150 + recycle;
int64_t ptr = create_texture(e, key);
send_frame(ptr, 0x5A);
Pulled uploading = pull_frame(ptr);
CHECK(uploading.ok && uploading.width == kWidth);
send_frame(ptr, 0x5B);
CHECK(close_texture(e, key, recycle) == 1);
CHECK(frame_is_live(uploading.buffer));
CHECK(uploading.buffer[0] == 0x5A &&
uploading.buffer[kFrameBytes - 1] == 0x5A);
// A raster frame that had looked the texture up before the close gets no
// frame, and the frame uploaded before is freed.
CHECK(!pull_frame(ptr).ok);
CHECK(live_frames() == base);
engine_shutdown(e);
}
}
// B252: a closed texture's frame is freed by its own window's raster thread
// only; another window's raster thread runs while this one uploads.
static void test_shown_frame_waits_for_its_own_window() {
const int base = live_frames();
Engine* w1 = engine_new();
Engine* w2 = engine_new();
int64_t a = create_texture(w1, 170);
int64_t b = create_texture(w2, 171);
send_frame(a, 40);
Pulled uploading = pull_frame(a);
CHECK(uploading.ok);
CHECK(close_texture(w1, 170, 1) == 1);
CHECK(pull_frame(b).ok);
CHECK(frame_is_live(uploading.buffer));
int64_t c = create_texture(w1, 172);
CHECK(c == a);
CHECK(pull_frame(c).ok);
CHECK(live_frames() == base);
engine_shutdown(w1);
engine_shutdown(w2);
}
// B252: when a window goes away (its engine stops rasterizing first), the
// frames its closed textures showed last are freed.
static void test_engine_shutdown_frees_shown_frames() {
const int base = live_frames();
Engine* e = engine_new();
int64_t a = create_texture(e, 160);
int64_t b = create_texture(e, 161);
send_frame(a, 30);
CHECK(pull_frame(a).ok);
send_frame(b, 31);
CHECK(pull_frame(b).ok);
CHECK(close_texture(e, 160, 0) == 1);
CHECK(close_texture(e, 161, 1) == 1);
CHECK(live_frames() == base + 2);
engine_shutdown(e);
CHECK(live_frames() == base);
}
// D1: after close, late frames are dropped and the texture never hands out a
// frame again (it would be a freed buffer).
static void test_closed_texture_ignores_late_frames() {
@ -643,11 +729,13 @@ static void test_recycled_texture_is_reused_in_same_window() {
send_frame(ptr, 8);
CHECK(close_texture(e, key, 1) == 1);
CHECK(!registered(e->textures, ptr));
CHECK(live_frames() == base);
// The frame shown last waits for the window's next raster frame.
CHECK(live_frames() == base + 1);
}
CHECK(seen.size() == 1);
CHECK(!finalized);
engine_shutdown(e);
CHECK(live_frames() == base);
}
// D2, multi-window: windows never get each other's textures back.
@ -790,21 +878,68 @@ static void test_multi_display_tab() {
// Close the tab, open another one with two displays.
CHECK(close_texture(e, 800, 1) == 1);
CHECK(close_texture(e, 802, 1) == 1);
CHECK(live_frames() == base);
CHECK(live_frames() == base + 2);
int64_t n0 = create_texture(e, 803);
int64_t n1 = create_texture(e, 804);
CHECK(std::set<int64_t>({n0, n1}) == std::set<int64_t>({d0, d1}));
CHECK(pull_frame(n0).ok);
CHECK(live_frames() == base);
CHECK(close_texture(e, 803, 1) == 1);
CHECK(close_texture(e, 804, 1) == 1);
engine_shutdown(e);
}
// B252 with threads (Flutter 3.32 and later): a raster thread uploads each
// frame it gets while the video thread pushes frames and the platform thread
// closes the texture.
static void test_raster_thread_uploads_while_closing() {
const int base = live_frames();
for (int round = 0; round < 100; round++) {
Engine* e = engine_new();
int64_t key = 900 + round;
int64_t ptr = create_texture(e, key);
std::atomic<bool> stop{false};
std::thread video([ptr, &stop]() {
for (int i = 0; !stop.load(); i++) {
send_frame(ptr, static_cast<uint8_t>(10 + i % 2));
}
});
std::thread raster([ptr, &stop]() {
while (!stop.load()) {
Pulled p = pull_frame(ptr);
if (p.ok && p.width == kWidth) {
uint8_t x = 0;
for (size_t i = 0; i < kFrameBytes; i++) {
x ^= p.buffer[i];
}
g_uploaded = x;
}
}
});
g_usleep(1000);
int64_t closed = close_texture(e, key, round % 2);
g_usleep(1000);
stop = true;
video.join();
raster.join();
engine_shutdown(e);
CHECK(closed == 1);
}
CHECK(live_frames() == base);
}
int main() {
struct {
const char* name;
void (*run)();
} tests[] = {
{"close_frees_frame_buffers", test_close_frees_frame_buffers},
{"close_keeps_frame_being_uploaded",
test_close_keeps_frame_being_uploaded},
{"shown_frame_waits_for_its_own_window",
test_shown_frame_waits_for_its_own_window},
{"engine_shutdown_frees_shown_frames",
test_engine_shutdown_frees_shown_frames},
{"closed_texture_ignores_late_frames",
test_closed_texture_ignores_late_frames},
{"finalize_frees_frame_buffers", test_finalize_frees_frame_buffers},
@ -819,6 +954,8 @@ int main() {
test_engine_shutdown_keeps_recycled_textures},
{"frames_racing_close", test_frames_racing_close},
{"multi_display_tab", test_multi_display_tab},
{"raster_thread_uploads_while_closing",
test_raster_thread_uploads_while_closing},
};
int failed = 0;
int count = 0;

View file

@ -1,6 +1,7 @@
// Modified by the Kitedesk fork (2026, B212): closeTexture with
// Modified by the Kitedesk fork (2026, B212, B252): closeTexture with
// "recycle": true keeps the texture for the next createTexture of the same
// engine instead of leaking it.
// engine instead of leaking it; an engine going away frees the frames its
// closed textures showed last.
#include "include/texture_rgba_renderer/texture_rgba_renderer_plugin.h"
#include "include/texture_rgba_renderer/texture_rgba_renderer.h"
@ -131,6 +132,9 @@ static void texture_rgba_renderer_plugin_dispose(GObject *object)
// purpose: releasing one makes the engine delete its GL texture in whatever
// GL context is current, which can belong to another window.
g_clear_pointer(&self->recycled, g_ptr_array_unref);
// The engine shuts down before it releases its plugins, so it uploads no
// frame any more.
texture_rgba_free_retired_frames(self->texture_registrar);
G_OBJECT_CLASS(texture_rgba_renderer_plugin_parent_class)->dispose(object);
}