B252: keep closed textures valid until the engine is done with them
Windows: closeTexture hands the texture to UnregisterTexture's callback and frees its frame buffers there; frames pushed after close are dropped. Without "recycle" the object is kept for the process lifetime (the session may still hold its pointer); an engine that shuts down keeps its open textures the same way. A duplicate createTexture key gets -1, getTexturePtr of an unknown key 0. macOS: closeTexture drops the frame and closes the texture to later frames; only "recycle" removes it from the plugin. Linux: the frame handed to the engine last is freed at that engine's next copy_pixels or once the engine has shut down, not at close: from Flutter 3.32 the raster thread can still be uploading it. Tests: windows/test/run_tests.sh (plain, --asan, --tsan) and linux/test/run_tests.sh (plain, --asan). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8cd34a80ff
commit
f38b9420ac
15 changed files with 1388 additions and 31 deletions
58
windows/test/run_tests.sh
Executable file
58
windows/test/run_tests.sh
Executable file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env bash
|
||||
# Kitedesk fork, 2026: builds windows/test/texture_rgba_renderer_plugin_test.cc
|
||||
# on Linux with the plugin sources and the C++ client wrapper of the Flutter
|
||||
# Windows engine, and runs each test in its own process. The engine files come
|
||||
# from FLUTTER_WINDOWS_ENGINE, by default the windows-x64 engine artifacts of the
|
||||
# Flutter SDK on PATH (or FLUTTER_ROOT) as `flutter precache --windows` leaves
|
||||
# them: the C headers and cpp_client_wrapper/. No display, GPU or Windows needed.
|
||||
# run_tests.sh -Wall -Werror build
|
||||
# run_tests.sh --asan the same with AddressSanitizer and UBSan
|
||||
# run_tests.sh --tsan the same with ThreadSanitizer
|
||||
# OUT_DIR (default: a new temporary directory) receives the test binary.
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
if [ -z "${FLUTTER_WINDOWS_ENGINE:-}" ]; then
|
||||
root=${FLUTTER_ROOT:-$(dirname "$(dirname "$(readlink -f "$(command -v flutter)")")")}
|
||||
FLUTTER_WINDOWS_ENGINE=$root/bin/cache/artifacts/engine/windows-x64
|
||||
fi
|
||||
engine=$FLUTTER_WINDOWS_ENGINE
|
||||
wrapper=$engine/cpp_client_wrapper
|
||||
[ -f "$engine/flutter_texture_registrar.h" ] && [ -f "$wrapper/core_implementations.cc" ] || {
|
||||
echo "no Flutter Windows engine headers and client wrapper in $engine" \
|
||||
"(flutter precache --windows, or set FLUTTER_WINDOWS_ENGINE)" >&2
|
||||
exit 2
|
||||
}
|
||||
out=${OUT_DIR:-$(mktemp -d)}
|
||||
mkdir -p "$out"
|
||||
# The client wrapper headers expect <cstdint> to come with the standard headers,
|
||||
# as MSVC's do; libstdc++'s do not.
|
||||
flags=(-std=c++17 -Wall -Werror -O1 -g -pthread -include cstdint -DFLUTTER_PLUGIN_IMPL '-D__declspec(x)=')
|
||||
case "${1:-}" in
|
||||
"") ;;
|
||||
--asan) flags+=('-fsanitize=address,undefined' -fno-omit-frame-pointer -fno-sanitize-recover=undefined) ;;
|
||||
--tsan)
|
||||
flags+=(-fsanitize=thread)
|
||||
export TSAN_OPTIONS=${TSAN_OPTIONS:-halt_on_error=1}
|
||||
;;
|
||||
*) echo "usage: $0 [--asan | --tsan]" >&2; exit 2 ;;
|
||||
esac
|
||||
bin=$out/texture_rgba_renderer_plugin_test
|
||||
"${CXX:-clang++}" "${flags[@]}" -I"$here/shim" -I"$wrapper/include" -I"$engine" \
|
||||
"$here/texture_rgba_renderer_plugin_test.cc" \
|
||||
"$here/../texture_rgba.cpp" "$here/../texture_rgba_renderer_plugin.cpp" \
|
||||
"$here/../texture_rgba_renderer_plugin_c_api.cpp" \
|
||||
"$wrapper/core_implementations.cc" "$wrapper/plugin_registrar.cc" "$wrapper/standard_codec.cc" \
|
||||
-o "$bin"
|
||||
count=0
|
||||
failed=0
|
||||
for test in $("$bin" --list); do
|
||||
count=$((count + 1))
|
||||
if "$bin" "$test"; then
|
||||
echo "PASS $test"
|
||||
else
|
||||
echo "FAIL $test"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
done
|
||||
echo "$count tests, $((count - failed)) passed, $failed failed"
|
||||
[ "$failed" -eq 0 ]
|
||||
3
windows/test/shim/VersionHelpers.h
Normal file
3
windows/test/shim/VersionHelpers.h
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
// Kitedesk fork, 2026: test-only stand-in for <VersionHelpers.h> (windows/test).
|
||||
// The plugin includes it but uses nothing from it. Licensed under the Apache
|
||||
// License, Version 2.0, like the rest of this package.
|
||||
63
windows/test/shim/flutter_windows.h
Normal file
63
windows/test/shim/flutter_windows.h
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
// Kitedesk fork, 2026: test-only stand-in for flutter_windows.h of the Flutter
|
||||
// Windows engine (a18df97c) with the declarations that the client wrapper's
|
||||
// plugin_registrar_windows.h and flutter_view.h use; windows/test defines them.
|
||||
// Licensed under the Apache License, Version 2.0, like the rest of this
|
||||
// package.
|
||||
|
||||
#ifndef KD_TEST_SHIM_FLUTTER_WINDOWS_H_
|
||||
#define KD_TEST_SHIM_FLUTTER_WINDOWS_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <windows.h>
|
||||
|
||||
#include "flutter_export.h"
|
||||
#include "flutter_messenger.h"
|
||||
#include "flutter_plugin_registrar.h"
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct IDXGIAdapter;
|
||||
|
||||
struct FlutterDesktopView;
|
||||
typedef struct FlutterDesktopView* FlutterDesktopViewRef;
|
||||
|
||||
typedef int64_t FlutterDesktopViewId;
|
||||
|
||||
typedef bool (*FlutterDesktopWindowProcCallback)(HWND /* hwnd */,
|
||||
UINT /* uMsg */,
|
||||
WPARAM /*wParam*/,
|
||||
LPARAM /* lParam*/,
|
||||
void* /* user data */,
|
||||
LRESULT* result);
|
||||
|
||||
FLUTTER_EXPORT HWND FlutterDesktopViewGetHWND(FlutterDesktopViewRef view);
|
||||
|
||||
FLUTTER_EXPORT struct IDXGIAdapter* FlutterDesktopViewGetGraphicsAdapter(
|
||||
FlutterDesktopViewRef view);
|
||||
|
||||
FLUTTER_EXPORT FlutterDesktopViewRef FlutterDesktopPluginRegistrarGetView(
|
||||
FlutterDesktopPluginRegistrarRef registrar);
|
||||
|
||||
FLUTTER_EXPORT FlutterDesktopViewRef FlutterDesktopPluginRegistrarGetViewById(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopViewId view_id);
|
||||
|
||||
FLUTTER_EXPORT void
|
||||
FlutterDesktopPluginRegistrarRegisterTopLevelWindowProcDelegate(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopWindowProcCallback delegate,
|
||||
void* user_data);
|
||||
|
||||
FLUTTER_EXPORT void
|
||||
FlutterDesktopPluginRegistrarUnregisterTopLevelWindowProcDelegate(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopWindowProcCallback delegate);
|
||||
|
||||
#if defined(__cplusplus)
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#endif // KD_TEST_SHIM_FLUTTER_WINDOWS_H_
|
||||
17
windows/test/shim/windows.h
Normal file
17
windows/test/shim/windows.h
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
// Kitedesk fork, 2026: test-only stand-in for <windows.h>, so that the plugin
|
||||
// and the engine's C++ client wrapper build on Linux for windows/test. It has
|
||||
// only the types those headers use. Licensed under the Apache License,
|
||||
// Version 2.0, like the rest of this package.
|
||||
|
||||
#ifndef KD_TEST_SHIM_WINDOWS_H_
|
||||
#define KD_TEST_SHIM_WINDOWS_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
typedef struct HWND__* HWND;
|
||||
typedef unsigned int UINT;
|
||||
typedef uintptr_t WPARAM;
|
||||
typedef intptr_t LPARAM;
|
||||
typedef intptr_t LRESULT;
|
||||
|
||||
#endif // KD_TEST_SHIM_WINDOWS_H_
|
||||
899
windows/test/texture_rgba_renderer_plugin_test.cc
Normal file
899
windows/test/texture_rgba_renderer_plugin_test.cc
Normal file
|
|
@ -0,0 +1,899 @@
|
|||
// Kitedesk fork, 2026: lifetime tests for the Windows texture of this plugin
|
||||
// (Kitedesk backlog B252). Licensed under the Apache License, Version 2.0,
|
||||
// like the rest of this package.
|
||||
//
|
||||
// windows/test/run_tests.sh builds this program on Linux together with the
|
||||
// plugin sources and the C++ client wrapper of the Flutter Windows engine
|
||||
// (core_implementations.cc, plugin_registrar.cc, standard_codec.cc); shim/
|
||||
// stands in for <windows.h> and flutter_windows.h. The C API of
|
||||
// flutter_windows.dll is the fake engine below. It follows
|
||||
// FlutterWindowsTextureRegistrar and FlutterWindowsEngine::Stop of engine
|
||||
// a18df97c (Flutter 3.24.5): unregistering a texture only queues a task for
|
||||
// the raster thread, which removes the texture from the engine and then runs
|
||||
// the callback; until that task has run, the raster thread may still call the
|
||||
// texture's pixel buffer callback and read the returned buffer (glTexImage2D).
|
||||
// When a window closes, the registrar is destroyed (and the plugin with it)
|
||||
// while the engine still runs. The Rust video thread is
|
||||
// FlutterRgbaRendererPluginOnRgba with the pointer from getTexturePtr. Frame
|
||||
// buffers the plugin allocates are counted.
|
||||
//
|
||||
// Usage: texture_rgba_renderer_plugin_test [--list | <test>]
|
||||
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <new>
|
||||
#include <set>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <flutter/encodable_value.h>
|
||||
#include <flutter/method_call.h>
|
||||
#include <flutter/method_result_functions.h>
|
||||
#include <flutter/standard_method_codec.h>
|
||||
#include <flutter_windows.h>
|
||||
|
||||
#include "../include/texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h"
|
||||
|
||||
#if defined(__has_feature)
|
||||
#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer)
|
||||
#define KD_SANITIZER 1
|
||||
#endif
|
||||
#endif
|
||||
#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__)
|
||||
#define KD_SANITIZER 1
|
||||
#endif
|
||||
|
||||
#ifdef KD_SANITIZER
|
||||
#include <sanitizer/allocator_interface.h>
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kWidth = 321;
|
||||
constexpr int kHeight = 123;
|
||||
constexpr size_t kFrameBytes = static_cast<size_t>(kWidth) * kHeight * 4;
|
||||
|
||||
// Heap accounting: live frame buffers (blocks of exactly kFrameBytes; nothing
|
||||
// else here allocates that size), live bytes, and whether watched blocks were
|
||||
// freed. The sanitizer runtimes report every allocation through their hooks;
|
||||
// otherwise operators new and delete are replaced below.
|
||||
std::atomic<int> g_live_frames{0};
|
||||
std::atomic<long long> g_live_bytes{0};
|
||||
constexpr int kWatchSlots = 64;
|
||||
std::atomic<const void*> g_watched[kWatchSlots];
|
||||
std::atomic<bool> g_watched_freed[kWatchSlots];
|
||||
|
||||
void on_allocate(size_t size) {
|
||||
if (size == kFrameBytes) {
|
||||
g_live_frames++;
|
||||
}
|
||||
g_live_bytes += static_cast<long long>(size);
|
||||
}
|
||||
|
||||
void on_free(const void* ptr, size_t size) {
|
||||
if (size == kFrameBytes) {
|
||||
g_live_frames--;
|
||||
}
|
||||
g_live_bytes -= static_cast<long long>(size);
|
||||
for (int i = 0; i < kWatchSlots; i++) {
|
||||
if (g_watched[i].load() == ptr) {
|
||||
g_watched_freed[i] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef KD_SANITIZER
|
||||
|
||||
void malloc_hook(const volatile void* ptr, size_t size) {
|
||||
on_allocate(size);
|
||||
}
|
||||
|
||||
void free_hook(const volatile void* ptr) {
|
||||
on_free(const_cast<const void*>(ptr), __sanitizer_get_allocated_size(ptr));
|
||||
}
|
||||
|
||||
void install_heap_accounting() {
|
||||
__sanitizer_install_malloc_and_free_hooks(malloc_hook, free_hook);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
#else // KD_SANITIZER
|
||||
|
||||
constexpr size_t kHeader = 16;
|
||||
|
||||
void* allocate(size_t size) noexcept {
|
||||
char* p = static_cast<char*>(std::malloc(size + kHeader));
|
||||
if (p == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
std::memcpy(p, &size, sizeof(size));
|
||||
on_allocate(size);
|
||||
return p + kHeader;
|
||||
}
|
||||
|
||||
void deallocate(void* ptr) noexcept {
|
||||
if (ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
char* p = static_cast<char*>(ptr) - kHeader;
|
||||
size_t size;
|
||||
std::memcpy(&size, p, sizeof(size));
|
||||
on_free(ptr, size);
|
||||
std::free(p);
|
||||
}
|
||||
|
||||
void install_heap_accounting() {}
|
||||
|
||||
} // namespace
|
||||
|
||||
void* operator new(size_t size) {
|
||||
void* p = allocate(size);
|
||||
if (p == nullptr) {
|
||||
throw std::bad_alloc();
|
||||
}
|
||||
return p;
|
||||
}
|
||||
void* operator new[](size_t size) { return operator new(size); }
|
||||
void* operator new(size_t size, const std::nothrow_t&) noexcept {
|
||||
return allocate(size);
|
||||
}
|
||||
void* operator new[](size_t size, const std::nothrow_t&) noexcept {
|
||||
return allocate(size);
|
||||
}
|
||||
void operator delete(void* p) noexcept { deallocate(p); }
|
||||
void operator delete[](void* p) noexcept { deallocate(p); }
|
||||
void operator delete(void* p, size_t) noexcept { deallocate(p); }
|
||||
void operator delete[](void* p, size_t) noexcept { deallocate(p); }
|
||||
void operator delete(void* p, const std::nothrow_t&) noexcept { deallocate(p); }
|
||||
void operator delete[](void* p, const std::nothrow_t&) noexcept {
|
||||
deallocate(p);
|
||||
}
|
||||
|
||||
#endif // KD_SANITIZER
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fake engine: the C API of flutter_windows.dll that the plugin and the client
|
||||
// wrapper call.
|
||||
|
||||
constexpr int64_t kMaxTextureIds = 1024;
|
||||
|
||||
struct FlutterDesktopMessenger {
|
||||
std::mutex lock;
|
||||
std::map<std::string, std::pair<FlutterDesktopMessageCallback, void*>>
|
||||
handlers;
|
||||
};
|
||||
|
||||
struct _FlutterPlatformMessageResponseHandle {
|
||||
bool replied = false;
|
||||
std::vector<uint8_t> reply;
|
||||
};
|
||||
|
||||
struct FlutterDesktopTextureRegistrar {
|
||||
std::mutex mutex;
|
||||
int64_t next_id = 1;
|
||||
// The engine's textures: texture id -> what PopulateTexture calls.
|
||||
std::map<int64_t, FlutterDesktopPixelBufferTextureConfig> textures;
|
||||
// Tasks posted to the raster thread (PostRasterThreadTask).
|
||||
std::deque<std::function<void()>> raster_tasks;
|
||||
std::array<int, kMaxTextureIds> unregisters{};
|
||||
std::array<int, kMaxTextureIds> marks{};
|
||||
};
|
||||
|
||||
struct FlutterDesktopPluginRegistrar {
|
||||
FlutterDesktopMessenger messenger;
|
||||
FlutterDesktopTextureRegistrar textures;
|
||||
FlutterDesktopOnPluginRegistrarDestroyed on_destroyed = nullptr;
|
||||
};
|
||||
|
||||
extern "C" {
|
||||
|
||||
bool FlutterDesktopMessengerSend(FlutterDesktopMessengerRef messenger,
|
||||
const char* channel,
|
||||
const uint8_t* message,
|
||||
const size_t message_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FlutterDesktopMessengerSendWithReply(FlutterDesktopMessengerRef messenger,
|
||||
const char* channel,
|
||||
const uint8_t* message,
|
||||
const size_t message_size,
|
||||
const FlutterDesktopBinaryReply reply,
|
||||
void* user_data) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void FlutterDesktopMessengerSendResponse(
|
||||
FlutterDesktopMessengerRef messenger,
|
||||
const FlutterDesktopMessageResponseHandle* handle,
|
||||
const uint8_t* data,
|
||||
size_t data_length) {
|
||||
auto* h = const_cast<FlutterDesktopMessageResponseHandle*>(handle);
|
||||
h->replied = true;
|
||||
h->reply.assign(data, data + data_length);
|
||||
}
|
||||
|
||||
void FlutterDesktopMessengerSetCallback(FlutterDesktopMessengerRef messenger,
|
||||
const char* channel,
|
||||
FlutterDesktopMessageCallback callback,
|
||||
void* user_data) {
|
||||
if (callback == nullptr) {
|
||||
messenger->handlers.erase(channel);
|
||||
} else {
|
||||
messenger->handlers[channel] = {callback, user_data};
|
||||
}
|
||||
}
|
||||
|
||||
FlutterDesktopMessengerRef FlutterDesktopMessengerAddRef(
|
||||
FlutterDesktopMessengerRef messenger) {
|
||||
return messenger;
|
||||
}
|
||||
|
||||
void FlutterDesktopMessengerRelease(FlutterDesktopMessengerRef messenger) {}
|
||||
|
||||
bool FlutterDesktopMessengerIsAvailable(FlutterDesktopMessengerRef messenger) {
|
||||
return true;
|
||||
}
|
||||
|
||||
FlutterDesktopMessengerRef FlutterDesktopMessengerLock(
|
||||
FlutterDesktopMessengerRef messenger) {
|
||||
messenger->lock.lock();
|
||||
return messenger;
|
||||
}
|
||||
|
||||
void FlutterDesktopMessengerUnlock(FlutterDesktopMessengerRef messenger) {
|
||||
messenger->lock.unlock();
|
||||
}
|
||||
|
||||
int64_t FlutterDesktopTextureRegistrarRegisterExternalTexture(
|
||||
FlutterDesktopTextureRegistrarRef registrar,
|
||||
const FlutterDesktopTextureInfo* info) {
|
||||
if (info->type != kFlutterDesktopPixelBufferTexture ||
|
||||
info->pixel_buffer_config.callback == nullptr) {
|
||||
return -1;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(registrar->mutex);
|
||||
int64_t id = registrar->next_id++;
|
||||
if (id >= kMaxTextureIds) {
|
||||
std::abort();
|
||||
}
|
||||
registrar->textures[id] = info->pixel_buffer_config;
|
||||
return id;
|
||||
}
|
||||
|
||||
// FlutterWindowsTextureRegistrar::UnregisterTexture: the engine drops the
|
||||
// texture on the raster thread, then calls back.
|
||||
void FlutterDesktopTextureRegistrarUnregisterExternalTexture(
|
||||
FlutterDesktopTextureRegistrarRef registrar,
|
||||
int64_t texture_id,
|
||||
void (*callback)(void* user_data),
|
||||
void* user_data) {
|
||||
std::lock_guard<std::mutex> lock(registrar->mutex);
|
||||
if (texture_id > 0 && texture_id < kMaxTextureIds) {
|
||||
registrar->unregisters[texture_id]++;
|
||||
}
|
||||
registrar->raster_tasks.push_back(
|
||||
[registrar, texture_id, callback, user_data]() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(registrar->mutex);
|
||||
registrar->textures.erase(texture_id);
|
||||
}
|
||||
if (callback != nullptr) {
|
||||
callback(user_data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
bool FlutterDesktopTextureRegistrarMarkExternalTextureFrameAvailable(
|
||||
FlutterDesktopTextureRegistrarRef registrar,
|
||||
int64_t texture_id) {
|
||||
std::lock_guard<std::mutex> lock(registrar->mutex);
|
||||
if (texture_id > 0 && texture_id < kMaxTextureIds) {
|
||||
registrar->marks[texture_id]++;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
FlutterDesktopMessengerRef FlutterDesktopPluginRegistrarGetMessenger(
|
||||
FlutterDesktopPluginRegistrarRef registrar) {
|
||||
return ®istrar->messenger;
|
||||
}
|
||||
|
||||
FlutterDesktopTextureRegistrarRef FlutterDesktopRegistrarGetTextureRegistrar(
|
||||
FlutterDesktopPluginRegistrarRef registrar) {
|
||||
return ®istrar->textures;
|
||||
}
|
||||
|
||||
void FlutterDesktopPluginRegistrarSetDestructionHandler(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopOnPluginRegistrarDestroyed callback) {
|
||||
registrar->on_destroyed = callback;
|
||||
}
|
||||
|
||||
HWND FlutterDesktopViewGetHWND(FlutterDesktopViewRef view) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
struct IDXGIAdapter* FlutterDesktopViewGetGraphicsAdapter(
|
||||
FlutterDesktopViewRef view) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
FlutterDesktopViewRef FlutterDesktopPluginRegistrarGetView(
|
||||
FlutterDesktopPluginRegistrarRef registrar) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
FlutterDesktopViewRef FlutterDesktopPluginRegistrarGetViewById(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopViewId view_id) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void FlutterDesktopPluginRegistrarRegisterTopLevelWindowProcDelegate(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopWindowProcCallback delegate,
|
||||
void* user_data) {}
|
||||
|
||||
void FlutterDesktopPluginRegistrarUnregisterTopLevelWindowProcDelegate(
|
||||
FlutterDesktopPluginRegistrarRef registrar,
|
||||
FlutterDesktopWindowProcCallback delegate) {}
|
||||
|
||||
} // extern "C"
|
||||
|
||||
namespace {
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One engine (= one Flutter window) with the plugin registered on it.
|
||||
|
||||
struct Engine {
|
||||
FlutterDesktopPluginRegistrar* registrar;
|
||||
};
|
||||
|
||||
Engine* engine_new() {
|
||||
Engine* e = new Engine{new FlutterDesktopPluginRegistrar()};
|
||||
TextureRgbaRendererPluginCApiRegisterWithRegistrar(e->registrar);
|
||||
return e;
|
||||
}
|
||||
|
||||
FlutterDesktopTextureRegistrar& textures(Engine* e) {
|
||||
return e->registrar->textures;
|
||||
}
|
||||
|
||||
// Runs the tasks queued for the raster thread (unregistrations).
|
||||
void run_raster_tasks(Engine* e) {
|
||||
for (;;) {
|
||||
std::function<void()> task;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
if (textures(e).raster_tasks.empty()) {
|
||||
return;
|
||||
}
|
||||
task = std::move(textures(e).raster_tasks.front());
|
||||
textures(e).raster_tasks.pop_front();
|
||||
}
|
||||
task();
|
||||
}
|
||||
}
|
||||
|
||||
// The window closes. As in FlutterWindowsEngine::Stop, the registrar's
|
||||
// destruction handler runs first, which destroys the plugin, while the
|
||||
// engine and its raster thread keep running.
|
||||
void engine_destroy_plugins(Engine* e) {
|
||||
e->registrar->on_destroyed(e->registrar);
|
||||
}
|
||||
|
||||
// Then the engine shuts down: its raster thread runs what was queued and the
|
||||
// engine's memory is freed.
|
||||
void engine_free(Engine* e) {
|
||||
run_raster_tasks(e);
|
||||
delete e->registrar;
|
||||
delete e;
|
||||
}
|
||||
|
||||
void engine_shutdown(Engine* e) {
|
||||
engine_destroy_plugins(e);
|
||||
engine_free(e);
|
||||
}
|
||||
|
||||
constexpr int64_t kNoReply = INT64_MIN;
|
||||
|
||||
// Calls a plugin method like Dart does. Returns the int or bool result, or
|
||||
// kNoReply when the plugin did not answer.
|
||||
int64_t invoke(Engine* e, const char* method, flutter::EncodableMap args) {
|
||||
const auto& codec = flutter::StandardMethodCodec::GetInstance();
|
||||
flutter::MethodCall<flutter::EncodableValue> call(
|
||||
method, std::make_unique<flutter::EncodableValue>(std::move(args)));
|
||||
std::unique_ptr<std::vector<uint8_t>> message = codec.EncodeMethodCall(call);
|
||||
_FlutterPlatformMessageResponseHandle handle;
|
||||
FlutterDesktopMessage m = {sizeof(FlutterDesktopMessage),
|
||||
"texture_rgba_renderer", message->data(),
|
||||
message->size(), &handle};
|
||||
FlutterDesktopMessenger& messenger = e->registrar->messenger;
|
||||
auto handler = messenger.handlers.at("texture_rgba_renderer");
|
||||
handler.first(&messenger, &m, handler.second);
|
||||
if (!handle.replied) {
|
||||
return kNoReply;
|
||||
}
|
||||
int64_t out = kNoReply;
|
||||
flutter::MethodResultFunctions<flutter::EncodableValue> result(
|
||||
[&out](const flutter::EncodableValue* value) {
|
||||
if (value == nullptr) {
|
||||
return;
|
||||
}
|
||||
if (auto v = std::get_if<int32_t>(value)) {
|
||||
out = *v;
|
||||
} else if (auto v64 = std::get_if<int64_t>(value)) {
|
||||
out = *v64;
|
||||
} else if (auto b = std::get_if<bool>(value)) {
|
||||
out = *b ? 1 : 0;
|
||||
}
|
||||
},
|
||||
nullptr, nullptr);
|
||||
codec.DecodeAndProcessResponseEnvelope(handle.reply.data(),
|
||||
handle.reply.size(), &result);
|
||||
return out;
|
||||
}
|
||||
|
||||
flutter::EncodableMap key_args(int key) {
|
||||
return {{flutter::EncodableValue("key"), flutter::EncodableValue(key)}};
|
||||
}
|
||||
|
||||
int64_t create_texture(Engine* e, int key) {
|
||||
return invoke(e, "createTexture", key_args(key));
|
||||
}
|
||||
|
||||
int64_t texture_ptr(Engine* e, int key) {
|
||||
return invoke(e, "getTexturePtr", key_args(key));
|
||||
}
|
||||
|
||||
// recycle < 0: no "recycle" argument (callers before the Kitedesk change).
|
||||
int64_t close_texture(Engine* e, int key, int recycle) {
|
||||
flutter::EncodableMap args = key_args(key);
|
||||
if (recycle >= 0) {
|
||||
args[flutter::EncodableValue("recycle")] =
|
||||
flutter::EncodableValue(recycle != 0);
|
||||
}
|
||||
return invoke(e, "closeTexture", std::move(args));
|
||||
}
|
||||
|
||||
// Decoded frames, filled before any test runs (frame i is all bytes i).
|
||||
constexpr int kFills = 16;
|
||||
uint8_t g_frames[kFills][kFrameBytes];
|
||||
|
||||
// A decoded frame from the Rust video thread.
|
||||
void send_frame(int64_t ptr, uint8_t fill) {
|
||||
FlutterRgbaRendererPluginOnRgba(reinterpret_cast<void*>(ptr),
|
||||
g_frames[fill % kFills],
|
||||
static_cast<int>(kFrameBytes), kWidth,
|
||||
kHeight, 0);
|
||||
}
|
||||
|
||||
volatile uint8_t g_sink;
|
||||
|
||||
struct Pulled {
|
||||
bool ok;
|
||||
uint8_t first;
|
||||
size_t width;
|
||||
size_t height;
|
||||
};
|
||||
|
||||
// What the raster thread does for a texture on a frame
|
||||
// (ExternalTexturePixelBuffer::CopyPixelBuffer): ask for the pixel buffer and
|
||||
// upload it, which reads all of it.
|
||||
Pulled populate(const FlutterDesktopPixelBufferTextureConfig& config) {
|
||||
Pulled p = {false, 0, 0, 0};
|
||||
const FlutterDesktopPixelBuffer* buffer =
|
||||
config.callback(1920, 1080, config.user_data);
|
||||
if (buffer == nullptr || buffer->buffer == nullptr) {
|
||||
return p;
|
||||
}
|
||||
uint8_t x = 0;
|
||||
for (size_t i = 0; i < buffer->width * buffer->height * 4; i++) {
|
||||
x ^= buffer->buffer[i];
|
||||
}
|
||||
g_sink = x;
|
||||
p = {true, buffer->buffer[0], buffer->width, buffer->height};
|
||||
if (buffer->release_callback != nullptr) {
|
||||
buffer->release_callback(buffer->release_context);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
Pulled pull(Engine* e, int64_t id) {
|
||||
FlutterDesktopPixelBufferTextureConfig config;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
auto it = textures(e).textures.find(id);
|
||||
if (it == textures(e).textures.end()) {
|
||||
return {false, 0, 0, 0};
|
||||
}
|
||||
config = it->second;
|
||||
}
|
||||
return populate(config);
|
||||
}
|
||||
|
||||
// One raster frame: every texture the engine still has is drawn.
|
||||
void pull_all(Engine* e) {
|
||||
std::vector<FlutterDesktopPixelBufferTextureConfig> configs;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
for (const auto& entry : textures(e).textures) {
|
||||
configs.push_back(entry.second);
|
||||
}
|
||||
}
|
||||
for (const auto& config : configs) {
|
||||
populate(config);
|
||||
}
|
||||
}
|
||||
|
||||
bool registered(Engine* e, int64_t id) {
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
return textures(e).textures.count(id) != 0;
|
||||
}
|
||||
|
||||
int unregisters(Engine* e, int64_t id) {
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
return id > 0 && id < kMaxTextureIds ? textures(e).unregisters[id] : -1;
|
||||
}
|
||||
|
||||
int marks(Engine* e, int64_t id) {
|
||||
std::lock_guard<std::mutex> lock(textures(e).mutex);
|
||||
return id > 0 && id < kMaxTextureIds ? textures(e).marks[id] : -1;
|
||||
}
|
||||
|
||||
int live_frames() {
|
||||
return g_live_frames.load();
|
||||
}
|
||||
|
||||
// Watches the heap block at ptr; freed(slot) tells whether it was freed since.
|
||||
int watch(int64_t ptr) {
|
||||
for (int i = 0; i < kWatchSlots; i++) {
|
||||
if (g_watched[i].load() == nullptr) {
|
||||
g_watched_freed[i] = false;
|
||||
g_watched[i] = reinterpret_cast<const void*>(ptr);
|
||||
return i;
|
||||
}
|
||||
}
|
||||
std::abort();
|
||||
}
|
||||
|
||||
bool freed(int slot) {
|
||||
return g_watched_freed[slot].load();
|
||||
}
|
||||
|
||||
// The Rust session's VideoRenderer (src/flutter.rs): frames go to the
|
||||
// registered pointer under a read lock, and registerPixelbufferTexture(..., 0)
|
||||
// clears it under the write lock.
|
||||
struct Session {
|
||||
std::shared_mutex lock;
|
||||
int64_t ptr = 0;
|
||||
|
||||
void set(int64_t p) {
|
||||
std::unique_lock<std::shared_mutex> l(lock);
|
||||
ptr = p;
|
||||
}
|
||||
|
||||
void push(uint8_t fill) {
|
||||
std::shared_lock<std::shared_mutex> l(lock);
|
||||
if (ptr != 0) {
|
||||
send_frame(ptr, fill);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
int g_failed_checks = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::fprintf(stderr, " check failed at line %d: %s\n", __LINE__, \
|
||||
#cond); \
|
||||
g_failed_checks++; \
|
||||
return; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
// N-01, raster thread: closeTexture only queues the engine's unregistration,
|
||||
// and the engine may draw the texture once more before it is done. The texture
|
||||
// must stay valid and hand out no frame until the engine calls back; then its
|
||||
// frame buffers are freed. With "recycle" (nothing pushes to the pointer any
|
||||
// more) it is destroyed, without it, it is kept.
|
||||
void test_close_keeps_texture_until_engine_unregisters() {
|
||||
for (int recycle = 1; recycle >= 0; recycle--) {
|
||||
const int base = live_frames();
|
||||
Engine* e = engine_new();
|
||||
int64_t id = create_texture(e, 10);
|
||||
int64_t ptr = texture_ptr(e, 10);
|
||||
CHECK(id > 0 && ptr != 0 && ptr != -1 && registered(e, id));
|
||||
send_frame(ptr, 1);
|
||||
Pulled shown = pull(e, id);
|
||||
CHECK(shown.ok && shown.first == 1 && shown.width == kWidth &&
|
||||
shown.height == kHeight);
|
||||
send_frame(ptr, 2);
|
||||
CHECK(live_frames() == base + 2);
|
||||
const int slot = watch(ptr);
|
||||
CHECK(close_texture(e, 10, recycle) == 1);
|
||||
CHECK(registered(e, id));
|
||||
Pulled late = pull(e, id);
|
||||
CHECK(!late.ok);
|
||||
CHECK(!freed(slot));
|
||||
run_raster_tasks(e);
|
||||
CHECK(!registered(e, id));
|
||||
CHECK(live_frames() == base);
|
||||
CHECK(freed(slot) == (recycle == 1));
|
||||
engine_shutdown(e);
|
||||
}
|
||||
}
|
||||
|
||||
// N-01, video thread: without "recycle" the session keeps the pointer (a tab
|
||||
// moving to another window), so frames keep arriving after closeTexture. They
|
||||
// are dropped without telling the engine, before and after it unregistered the
|
||||
// texture, and after the engine itself is gone. The closed texture's address
|
||||
// is never handed out again (ABA: a late frame would show in another session).
|
||||
void test_late_frames_after_close_are_dropped() {
|
||||
for (int recycle = 0; recycle >= -1; recycle--) {
|
||||
const int base = live_frames();
|
||||
Engine* e = engine_new();
|
||||
int64_t a = create_texture(e, 20);
|
||||
int64_t pa = texture_ptr(e, 20);
|
||||
CHECK(a > 0 && pa != 0 && pa != -1);
|
||||
send_frame(pa, 3);
|
||||
CHECK(pull(e, a).ok);
|
||||
CHECK(close_texture(e, 20, recycle) == 1);
|
||||
const int announced = marks(e, a);
|
||||
send_frame(pa, 4);
|
||||
CHECK(marks(e, a) == announced);
|
||||
run_raster_tasks(e);
|
||||
CHECK(live_frames() == base);
|
||||
send_frame(pa, 5);
|
||||
CHECK(marks(e, a) == announced && live_frames() == base);
|
||||
std::set<int64_t> ids;
|
||||
for (int key = 21; key < 53; key++) {
|
||||
int64_t id = create_texture(e, key);
|
||||
CHECK(id > 0 && texture_ptr(e, key) != pa);
|
||||
ids.insert(id);
|
||||
}
|
||||
send_frame(pa, 6);
|
||||
for (int64_t id : ids) {
|
||||
CHECK(marks(e, id) == 0);
|
||||
}
|
||||
engine_shutdown(e);
|
||||
send_frame(pa, 7);
|
||||
CHECK(live_frames() == base);
|
||||
}
|
||||
}
|
||||
|
||||
// A key that is in use gets -1. Without an answer the Dart future would never
|
||||
// complete.
|
||||
void test_duplicate_key_gets_minus_one() {
|
||||
Engine* e = engine_new();
|
||||
int64_t a = create_texture(e, 30);
|
||||
int64_t pa = texture_ptr(e, 30);
|
||||
CHECK(a > 0);
|
||||
CHECK(create_texture(e, 30) == -1);
|
||||
CHECK(texture_ptr(e, 30) == pa);
|
||||
CHECK(close_texture(e, 30, 1) == 1);
|
||||
run_raster_tasks(e);
|
||||
engine_shutdown(e);
|
||||
}
|
||||
|
||||
// The texture is unregistered once: by closeTexture, or when the window goes
|
||||
// away while it is open, not again when the object is destroyed.
|
||||
void test_texture_is_unregistered_once() {
|
||||
Engine* e = engine_new();
|
||||
int64_t a = create_texture(e, 40);
|
||||
CHECK(close_texture(e, 40, 1) == 1);
|
||||
int64_t b = create_texture(e, 41);
|
||||
CHECK(close_texture(e, 41, 0) == 1);
|
||||
int64_t c = create_texture(e, 42);
|
||||
run_raster_tasks(e);
|
||||
CHECK(unregisters(e, a) == 1 && unregisters(e, b) == 1);
|
||||
engine_destroy_plugins(e);
|
||||
run_raster_tasks(e);
|
||||
CHECK(unregisters(e, c) == 1 && !registered(e, c));
|
||||
engine_free(e);
|
||||
}
|
||||
|
||||
// When the window closes, the textures it still has are unregistered but not
|
||||
// destroyed: the sessions may still push frames to them, and the raster
|
||||
// thread may draw them until the engine has dropped them.
|
||||
void test_engine_shutdown_retires_open_textures() {
|
||||
const int base = live_frames();
|
||||
Engine* e = engine_new();
|
||||
int64_t a = create_texture(e, 50);
|
||||
int64_t pa = texture_ptr(e, 50);
|
||||
CHECK(a > 0 && pa != 0 && pa != -1);
|
||||
send_frame(pa, 8);
|
||||
CHECK(pull(e, a).ok);
|
||||
send_frame(pa, 9);
|
||||
engine_destroy_plugins(e);
|
||||
send_frame(pa, 10);
|
||||
CHECK(registered(e, a));
|
||||
CHECK(!pull(e, a).ok);
|
||||
engine_free(e);
|
||||
send_frame(pa, 11);
|
||||
CHECK(live_frames() == base);
|
||||
}
|
||||
|
||||
// A texture closed without "recycle" is kept for the rest of the process. It
|
||||
// keeps no frame buffer once the engine has unregistered it; what it keeps is
|
||||
// measured here (the retired bound).
|
||||
void test_retired_textures_keep_no_frame_buffers() {
|
||||
Engine* e = engine_new();
|
||||
auto cycle = [e](int key) {
|
||||
int64_t id = create_texture(e, key);
|
||||
int64_t ptr = texture_ptr(e, key);
|
||||
if (id <= 0 || ptr == 0 || ptr == -1) {
|
||||
return false;
|
||||
}
|
||||
send_frame(ptr, 12);
|
||||
pull(e, id);
|
||||
send_frame(ptr, 13);
|
||||
const bool closed = close_texture(e, key, 0) == 1;
|
||||
run_raster_tasks(e);
|
||||
return closed;
|
||||
};
|
||||
CHECK(cycle(60));
|
||||
const int base_frames = live_frames();
|
||||
const long long base_bytes = g_live_bytes.load();
|
||||
constexpr int kRetired = 16;
|
||||
for (int i = 0; i < kRetired; i++) {
|
||||
CHECK(cycle(61 + i));
|
||||
}
|
||||
CHECK(live_frames() == base_frames);
|
||||
const long long per_texture = (g_live_bytes.load() - base_bytes) / kRetired;
|
||||
std::printf(" retired texture keeps %lld bytes of heap (%d textures)\n",
|
||||
per_texture, kRetired);
|
||||
CHECK(per_texture < 1024);
|
||||
engine_shutdown(e);
|
||||
}
|
||||
|
||||
// getTexturePtr of a key that is not open (closed meanwhile) answers 0, which
|
||||
// the session treats as no texture, like on Linux and macOS.
|
||||
void test_texture_ptr_of_closed_key_is_null() {
|
||||
Engine* e = engine_new();
|
||||
CHECK(texture_ptr(e, 80) == 0);
|
||||
CHECK(create_texture(e, 81) > 0);
|
||||
CHECK(close_texture(e, 81, 0) == 1);
|
||||
CHECK(texture_ptr(e, 81) == 0);
|
||||
run_raster_tasks(e);
|
||||
engine_shutdown(e);
|
||||
}
|
||||
|
||||
// N-01 with threads: the video thread pushes frames and the raster thread draws
|
||||
// every texture and runs the engine's tasks while the platform thread closes
|
||||
// textures. With release_first the session first clears its pointer (like
|
||||
// destroy(true) in Dart) and closes with "recycle"; otherwise it keeps pushing
|
||||
// to the closed texture.
|
||||
void racing_close(bool release_first) {
|
||||
const int base = live_frames();
|
||||
Engine* e = engine_new();
|
||||
std::atomic<bool> stop{false};
|
||||
std::thread raster([e, &stop]() {
|
||||
while (!stop.load()) {
|
||||
pull_all(e);
|
||||
run_raster_tasks(e);
|
||||
}
|
||||
});
|
||||
constexpr int kTextures = 20;
|
||||
int64_t closed[kTextures] = {};
|
||||
bool created = true;
|
||||
for (int i = 0; i < kTextures && created; i++) {
|
||||
const int key = 700 + i;
|
||||
int64_t id = create_texture(e, key);
|
||||
int64_t ptr = texture_ptr(e, key);
|
||||
created = id > 0 && ptr != 0 && ptr != -1;
|
||||
if (!created) {
|
||||
break;
|
||||
}
|
||||
Session session;
|
||||
session.set(ptr);
|
||||
std::atomic<bool> done{false};
|
||||
std::thread video([&session, &done]() {
|
||||
for (int n = 0; !done.load(); n++) {
|
||||
session.push(static_cast<uint8_t>(1 + n % 2));
|
||||
}
|
||||
});
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||
if (release_first) {
|
||||
session.set(0);
|
||||
}
|
||||
closed[i] = close_texture(e, key, release_first ? 1 : 0);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||
done = true;
|
||||
video.join();
|
||||
}
|
||||
stop = true;
|
||||
raster.join();
|
||||
run_raster_tasks(e);
|
||||
CHECK(created);
|
||||
for (int i = 0; i < kTextures; i++) {
|
||||
CHECK(closed[i] == 1);
|
||||
}
|
||||
CHECK(live_frames() == base);
|
||||
engine_shutdown(e);
|
||||
}
|
||||
|
||||
void test_frames_racing_close_without_recycle() {
|
||||
racing_close(false);
|
||||
}
|
||||
|
||||
void test_frames_racing_close_with_recycle() {
|
||||
racing_close(true);
|
||||
}
|
||||
|
||||
struct Test {
|
||||
const char* name;
|
||||
void (*run)();
|
||||
};
|
||||
|
||||
const Test kTests[] = {
|
||||
{"close_keeps_texture_until_engine_unregisters",
|
||||
test_close_keeps_texture_until_engine_unregisters},
|
||||
{"late_frames_after_close_are_dropped",
|
||||
test_late_frames_after_close_are_dropped},
|
||||
{"duplicate_key_gets_minus_one", test_duplicate_key_gets_minus_one},
|
||||
{"texture_is_unregistered_once", test_texture_is_unregistered_once},
|
||||
{"engine_shutdown_retires_open_textures",
|
||||
test_engine_shutdown_retires_open_textures},
|
||||
{"retired_textures_keep_no_frame_buffers",
|
||||
test_retired_textures_keep_no_frame_buffers},
|
||||
{"texture_ptr_of_closed_key_is_null",
|
||||
test_texture_ptr_of_closed_key_is_null},
|
||||
{"frames_racing_close_without_recycle",
|
||||
test_frames_racing_close_without_recycle},
|
||||
{"frames_racing_close_with_recycle", test_frames_racing_close_with_recycle},
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
install_heap_accounting();
|
||||
for (int i = 0; i < kFills; i++) {
|
||||
std::memset(g_frames[i], i, kFrameBytes);
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--list") == 0) {
|
||||
for (const Test& t : kTests) {
|
||||
std::printf("%s\n", t.name);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
int count = 0;
|
||||
int failed = 0;
|
||||
for (const Test& t : kTests) {
|
||||
if (argc == 2 && std::strcmp(argv[1], t.name) != 0) {
|
||||
continue;
|
||||
}
|
||||
const int before = g_failed_checks;
|
||||
t.run();
|
||||
const bool ok = g_failed_checks == before;
|
||||
if (argc != 2) {
|
||||
std::printf("%s %s\n", ok ? "PASS" : "FAIL", t.name);
|
||||
}
|
||||
std::fflush(stdout);
|
||||
failed += ok ? 0 : 1;
|
||||
count++;
|
||||
}
|
||||
if (count == 0) {
|
||||
std::fprintf(stderr, "unknown test %s\n", argv[1]);
|
||||
return 2;
|
||||
}
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue