From 873919902ee28fef3b62034e85f60fb2761a8fdd Mon Sep 17 00:00:00 2001 From: fufesou Date: Fri, 3 Mar 2023 09:30:05 +0800 Subject: [PATCH] Feat/row align bytes (#7) * better bytes alignment Signed-off-by: fufesou * debug macos Signed-off-by: fufesou * debug linux Signed-off-by: fufesou * windows switch_rgba Signed-off-by: fufesou * better switch_rgba Signed-off-by: fufesou --------- Signed-off-by: fufesou --- README.md | 10 ++--- example/lib/main.dart | 43 +++++++++++-------- lib/texture_rgba_ffi.dart | 6 +-- lib/texture_rgba_renderer.dart | 4 +- lib/texture_rgba_renderer_method_channel.dart | 6 +-- ...ture_rgba_renderer_platform_interface.dart | 2 +- .../texture_rgba_renderer.h | 17 +++++--- .../texture_rgba_renderer_plugin.h | 2 +- linux/texture_rgba_renderer_plugin.cc | 16 +++---- macos/Classes/TextRgba.swift | 4 +- macos/Classes/TextureRgbaApi.m | 6 +-- macos/Classes/TextureRgbaRendererPlugin.swift | 4 +- .../texture_rgba_renderer_plugin_c_api.h | 2 +- windows/texture_rgba.cpp | 25 ++++++++++- windows/texture_rgba.h | 3 ++ .../texture_rgba_renderer_plugin_c_api.cpp | 4 +- 16 files changed, 94 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index fdd46ae..13b6f21 100644 --- a/README.md +++ b/README.md @@ -118,17 +118,17 @@ class TextureRgbaRenderer { Note that the `onRgba` function is not called immediately. It depends on the execution scheduler of Flutter. To solve the problem, the plugin provide the low level function on all 3 pc platforms, which can make them be executed immediately. ```c -void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height) +void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes) ``` You could use the native API in other programing languages. A dart quick example: ```dart /// The `Native` class is bundled in this plugin. -typedef F1 = Void Function( - Pointer ptr, Pointer buffer, Int32 width, Int32 height); -typedef F1Dart = void Function( - Pointer ptr, Pointer buffer, int width, int height); +typedef F1 = Void Function(Pointer ptr, Pointer buffer, Int len, + Int32 width, Int32 height, Int row_align_bytes); +typedef F1Dart = void Function(Pointer ptr, Pointer buffer, + int len, int width, int height, int row_align_bytes); class Native { Native._(); diff --git a/example/lib/main.dart b/example/lib/main.dart index 8c3f4b9..53ae0ae 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,3 +1,4 @@ +import 'dart:io'; import 'dart:ffi'; import 'dart:math'; import 'dart:typed_data'; @@ -24,7 +25,7 @@ class _MyAppState extends State { final _textureRgbaRendererPlugin = TextureRgbaRenderer(); int textureId = -1; int height = 500; - int width = 528; + int width = 500; int cnt = 0; var key = 0; int texturePtr = 0; @@ -33,6 +34,7 @@ class _MyAppState extends State { Timer? _timer; int time = 0; int method = 0; + final rowAlignBytes = Platform.isMacOS ? 64 : 1; @override void initState() { @@ -58,16 +60,18 @@ class _MyAppState extends State { void start(int methodId) { debugPrint("start mockPic"); method = methodId; + final rowBytes = (width * 4 + rowAlignBytes - 1) & (~(rowAlignBytes - 1)); + final picDataLength = rowBytes * height; _timer?.cancel(); // 60 fps _timer = Timer.periodic(const Duration(milliseconds: 1000 ~/ 60), (timer) async { if (methodId == 0) { // Method.1: with MethodChannel - data = mockPicture(width, height); + data = mockPicture(width, height, rowBytes, picDataLength); final t1 = DateTime.now().microsecondsSinceEpoch; final res = await _textureRgbaRendererPlugin.onRgba( - key, data!, height, width, 16); + key, data!, height, width, rowAlignBytes); final t2 = DateTime.now().microsecondsSinceEpoch; setState(() { time = t2 - t1; @@ -76,11 +80,11 @@ class _MyAppState extends State { debugPrint("WARN: render failed"); } } else { - final dataPtr = mockPicturePtr(width, height); + final dataPtr = mockPicturePtr(width, height, rowBytes, picDataLength); // Method.2: with native ffi final t1 = DateTime.now().microsecondsSinceEpoch; Native.instance.onRgba(Pointer.fromAddress(texturePtr).cast(), - dataPtr, width, height, 16); + dataPtr, picDataLength, width, height, rowAlignBytes); final t2 = DateTime.now().microsecondsSinceEpoch; setState(() { time = t2 - t1; @@ -99,27 +103,27 @@ class _MyAppState extends State { super.dispose(); } - Uint8List mockPicture(int width, int height) { - final pic = List.generate(width * height * 4, (index) { - final h = (index / 4) / width; - final w = (index / 4) % width; + Uint8List mockPicture(int width, int height, int rowBytes, int length) { + final pic = List.generate(length, (index) { + final r = index / rowBytes; + final c = (index % rowBytes) / 4; final p = index & 0x03; - if (h > 20 && h < 30) { - if (w > 20 && w < 25) { + if (c > 20 && c < 30) { + if (r > 20 && r < 25) { if (p == 0 || p == 3) { return 255; } else { return 0; } } - if (w > 40 && w < 45) { + if (r > 40 && r < 45) { if (p == 1 || p == 3) { return 255; } else { return 0; } } - if (w > 60 && w < 65) { + if (r > 60 && r < 65) { if (p == 2 || p == 3) { return 255; } else { @@ -132,13 +136,14 @@ class _MyAppState extends State { return Uint8List.fromList(pic); } - Pointer mockPicturePtr(int width, int height) { - final pic = List.generate(width * height * 4, (index) { - final h = (index / 4) / width; - final w = (index / 4) % width; + Pointer mockPicturePtr( + int width, int height, int rowBytes, int length) { + final pic = List.generate(length, (index) { + final r = index / rowBytes; + final c = (index % rowBytes) / 4; final p = index & 0x03; - final edgeH = (h >= 0 && h < 10) || ((h >= height - 10) && h < height); - final edgeW = (w >= 0 && w < 10) || ((w >= width - 10) && w < width); + final edgeH = (c >= 0 && c < 10) || ((c >= height - 10) && c < height); + final edgeW = (r >= 0 && r < 10) || ((r >= width - 10) && r < width); if (edgeH || edgeW) { if (p == 0 || p == 3) { return 255; diff --git a/lib/texture_rgba_ffi.dart b/lib/texture_rgba_ffi.dart index 7c26601..6c341eb 100644 --- a/lib/texture_rgba_ffi.dart +++ b/lib/texture_rgba_ffi.dart @@ -1,9 +1,9 @@ import 'dart:ffi'; -typedef F1 = Void Function(Pointer ptr, Pointer buffer, - Int32 width, Int32 height, Int stride_align); +typedef F1 = Void Function(Pointer ptr, Pointer buffer, Int len, + Int32 width, Int32 height, Int row_align_bytes); typedef F1Dart = void Function(Pointer ptr, Pointer buffer, - int width, int height, int stride_align); + int len, int width, int height, int row_align_bytes); class Native { Native._(); diff --git a/lib/texture_rgba_renderer.dart b/lib/texture_rgba_renderer.dart index f3a1f02..e66a61e 100644 --- a/lib/texture_rgba_renderer.dart +++ b/lib/texture_rgba_renderer.dart @@ -24,9 +24,9 @@ class TextureRgbaRenderer { /// Provide the rgba data to the texture. Future onRgba( - int key, Uint8List data, int height, int width, int stride_align) { + int key, Uint8List data, int height, int width, int row_align_bytes) { return TextureRgbaRendererPlatform.instance - .onRgba(key, data, height, width, stride_align); + .onRgba(key, data, height, width, row_align_bytes); } Future getTexturePtr(int key) { diff --git a/lib/texture_rgba_renderer_method_channel.dart b/lib/texture_rgba_renderer_method_channel.dart index f45fcc4..da6c2d8 100644 --- a/lib/texture_rgba_renderer_method_channel.dart +++ b/lib/texture_rgba_renderer_method_channel.dart @@ -37,14 +37,14 @@ class MethodChannelTextureRgbaRenderer extends TextureRgbaRendererPlatform { } @override - Future onRgba( - int key, Uint8List data, int height, int width, int stride_align) async { + Future onRgba(int key, Uint8List data, int height, int width, + int row_align_bytes) async { return await methodChannel.invokeMethod('onRgba', { "data": data, "height": height, "width": width, "key": key, - "stride_align": stride_align, + "row_align_bytes": row_align_bytes, }) ?? false; } diff --git a/lib/texture_rgba_renderer_platform_interface.dart b/lib/texture_rgba_renderer_platform_interface.dart index 6e56981..77af47a 100644 --- a/lib/texture_rgba_renderer_platform_interface.dart +++ b/lib/texture_rgba_renderer_platform_interface.dart @@ -40,7 +40,7 @@ abstract class TextureRgbaRendererPlatform extends PlatformInterface { } Future onRgba( - int key, Uint8List data, int height, int width, int stride_align) { + int key, Uint8List data, int height, int width, int row_align_bytes) { throw UnimplementedError('onRgba() has not been implemented.'); } diff --git a/linux/include/texture_rgba_renderer/texture_rgba_renderer.h b/linux/include/texture_rgba_renderer/texture_rgba_renderer.h index a7c5781..e026d8a 100644 --- a/linux/include/texture_rgba_renderer/texture_rgba_renderer.h +++ b/linux/include/texture_rgba_renderer/texture_rgba_renderer.h @@ -39,13 +39,18 @@ G_DEFINE_TYPE(TextureRgba, texture_rgba, (G_TYPE_CHECK_INSTANCE_CAST((obj), texture_rgba_get_type(), \ TextureRgba)) -static inline void switch_rgba(uint8_t* pixels, int width, int height) { +static inline void switch_rgba(uint8_t* pixels, int len, int height) { uint8_t temp; - - for (int i = 0; i < width * height; i++) { - temp = pixels[4 * i]; - pixels[4 * i] = pixels[4 * i + 2]; - pixels[4 * i + 2] = temp; + int byes_per_row = len / height; + int width = byes_per_row >> 2; + for (int i = 0; i < height; ++i) { + int row_idx_base = i * byes_per_row; + for (int k = 0; k < width; ++k) { + int idx = row_idx_base + k * 4; + temp = pixels[idx]; + pixels[idx] = pixels[idx + 2]; + pixels[idx + 2] = temp; + } } } diff --git a/linux/include/texture_rgba_renderer/texture_rgba_renderer_plugin.h b/linux/include/texture_rgba_renderer/texture_rgba_renderer_plugin.h index d6d1558..f7c7745 100644 --- a/linux/include/texture_rgba_renderer/texture_rgba_renderer_plugin.h +++ b/linux/include/texture_rgba_renderer/texture_rgba_renderer_plugin.h @@ -22,7 +22,7 @@ FLUTTER_PLUGIN_EXPORT void texture_rgba_renderer_plugin_register_with_registrar( FlPluginRegistrar* registrar); // Keep the same symbol name with Windows. -FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height); +FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes); G_END_DECLS diff --git a/linux/texture_rgba_renderer_plugin.cc b/linux/texture_rgba_renderer_plugin.cc index 0915dbf..0c6ba4b 100644 --- a/linux/texture_rgba_renderer_plugin.cc +++ b/linux/texture_rgba_renderer_plugin.cc @@ -14,7 +14,7 @@ std::unordered_map g_renderer_map; -void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height); +void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height); struct _TextureRgbaRendererPlugin { GObject parent_instance; @@ -68,11 +68,12 @@ static void texture_rgba_renderer_plugin_handle_method_call( auto args = fl_method_call_get_args(method_call); auto key = fl_value_get_int(fl_value_lookup_string(args, "key")); auto data = fl_value_get_uint8_list(fl_value_lookup_string(args, "data")); - // auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data")); + auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data")); auto width = fl_value_get_int(fl_value_lookup_string(args, "width")); auto height = fl_value_get_int(fl_value_lookup_string(args, "height")); + auto row_align_bytes = fl_value_get_int(fl_value_lookup_string(args, "row_align_bytes")); auto texture_rgba = g_renderer_map[key]; - FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, width, height); + FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, data_length, width, height, row_align_bytes); response = FL_METHOD_RESPONSE(fl_method_success_response_new( fl_value_new_bool(fl_texture_registrar_mark_texture_frame_available(self->texture_registrar, FL_TEXTURE(texture_rgba))))); } @@ -140,16 +141,15 @@ void texture_rgba_renderer_plugin_register_with_registrar(FlPluginRegistrar *reg extern "C" { - void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height) { + void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height, int row_align_bytes) { TextureRgba* self = TEXTURE_RGBA(texture_rgba_ptr); g_mutex_lock(&self->mutex); // selfate has registered a texture_id, if (self->texture_id != 0 && !g_atomic_int_get(&self->buffer_ready) && !texture_rgba_is_terminate(self)) { // copy data to the texture. - auto buffer_length = 4 * width * height; - auto copied_data = new uint8_t[buffer_length]; - memcpy(copied_data, buffer, buffer_length); - switch_rgba(copied_data, width, height); + auto copied_data = new uint8_t[len]; + memcpy(copied_data, buffer, len); + switch_rgba(copied_data, len, height); // It's safe to working on a non reading index g_atomic_pointer_set(&self->buffer, copied_data); g_atomic_int_set(&self->video_height, height); diff --git a/macos/Classes/TextRgba.swift b/macos/Classes/TextRgba.swift index f1c1a3e..ca8afbe 100644 --- a/macos/Classes/TextRgba.swift +++ b/macos/Classes/TextRgba.swift @@ -34,7 +34,7 @@ import CoreVideo } - @objc public func markFrameAvaliable(data: Data, width: Int, height: Int, stride_align: Int) -> Bool { + @objc public func markFrameAvaliable(data: Data, width: Int, height: Int, row_align_bytes: Int) -> Bool { queue.sync { self.data = data.withUnsafeBytes { buffer in var pixelBufferCopy: CVPixelBuffer! @@ -44,7 +44,7 @@ import CoreVideo kCVPixelBufferMetalCompatibilityKey: true, kCVPixelBufferOpenGLCompatibilityKey: true, // https://developer.apple.com/forums/thread/712709 - kCVPixelBufferBytesPerRowAlignmentKey: stride_align * 4 + kCVPixelBufferBytesPerRowAlignmentKey: 64 ] as CFDictionary, &pixelBufferCopy) guard result == kCVReturnSuccess else { return nil diff --git a/macos/Classes/TextureRgbaApi.m b/macos/Classes/TextureRgbaApi.m index 77ba983..2ab02ed 100644 --- a/macos/Classes/TextureRgbaApi.m +++ b/macos/Classes/TextureRgbaApi.m @@ -13,10 +13,10 @@ extern "C" { #endif /// Keep the same name with Windows. -void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height, int stride_align) { +void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes) { TextRgba* texture_rgba = (__bridge TextRgba *)(texture_rgba_ptr); - NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:((width + stride_align - 1) & ~(stride_align - 1)) * height * 4 freeWhenDone:FALSE]; - [texture_rgba markFrameAvaliableWithData:data width:width height:height stride_align: stride_align]; + NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:len freeWhenDone:FALSE]; + [texture_rgba markFrameAvaliableWithData:data width:width height:height row_align_bytes: row_align_bytes]; } #if __cplusplus diff --git a/macos/Classes/TextureRgbaRendererPlugin.swift b/macos/Classes/TextureRgbaRendererPlugin.swift index 8c991a6..16f03f5 100644 --- a/macos/Classes/TextureRgbaRendererPlugin.swift +++ b/macos/Classes/TextureRgbaRendererPlugin.swift @@ -49,12 +49,12 @@ public class TextureRgbaRendererPlugin: NSObject, FlutterPlugin { let data = args["data"] as! FlutterStandardTypedData let height = args["height"] as! Int let width = args["width"] as! Int - let stride_align = args["stride_align"] as! Int + let row_align_bytes = args["row_align_bytes"] as! Int let textureRgba = renderer[key] if (textureRgba == nil) { result(false) } else { - result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, stride_align: stride_align)) + result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, row_align_bytes: row_align_bytes)) } case "getTexturePtr": let args = call.arguments as! Dictionary diff --git a/windows/include/texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h b/windows/include/texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h index 4c08580..0e54994 100644 --- a/windows/include/texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h +++ b/windows/include/texture_rgba_renderer/texture_rgba_renderer_plugin_c_api.h @@ -16,7 +16,7 @@ extern "C" { FLUTTER_PLUGIN_EXPORT void TextureRgbaRendererPluginCApiRegisterWithRegistrar( FlutterDesktopPluginRegistrarRef registrar); -FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int width, int height); +FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int len, int width, int height, int row_align_bytes); #if defined(__cplusplus) } // extern "C" diff --git a/windows/texture_rgba.cpp b/windows/texture_rgba.cpp index d50dcf1..d860b24 100644 --- a/windows/texture_rgba.cpp +++ b/windows/texture_rgba.cpp @@ -1,7 +1,6 @@ #include "texture_rgba.h" #include - TextureRgba::TextureRgba(flutter::TextureRegistrar *texture_registrar) { this->texture_registrar_ = texture_registrar; @@ -17,14 +16,36 @@ TextureRgba::~TextureRgba() texture_registrar_->UnregisterTexture(texture_id_); } +void TextureRgba::switch_rgba(std::vector &buffer, size_t height) +{ + uint8_t temp; + size_t byes_per_row = buffer.size() / height; + size_t width = byes_per_row >> 2; + for (size_t i = 0; i < height; ++i) + { + size_t row_idx_base = i * byes_per_row; + for (size_t k = 0; k < width; ++k) + { + size_t idx = row_idx_base + k * 4; + temp = buffer[idx]; + buffer[idx] = buffer[idx + 2]; + buffer[idx + 2] = temp; + } + } +} + void TextureRgba::MarkVideoFrameAvailable( std::vector &buffer, size_t width, size_t height) { - if (buffer.empty() || buffer.size() != width * height * 4) + if (buffer.empty()) { return; } + // Do not check bellow. Because the data may have paddings per row. + // buffer.size() != width * height * 4 + + switch_rgba(buffer, height); const std::lock_guard lock(mutex_); int bg_index = fg_index_ ^ 1; buffer.swap(buffer_tmp_[bg_index]); diff --git a/windows/texture_rgba.h b/windows/texture_rgba.h index ffcfc47..07a98ef 100644 --- a/windows/texture_rgba.h +++ b/windows/texture_rgba.h @@ -17,6 +17,9 @@ public: const FlutterDesktopPixelBuffer* buffer(); +private: + void switch_rgba(std::vector& buffer, size_t height); + private: FlutterDesktopPixelBuffer flutter_pixel_buffer_{}; flutter::TextureRegistrar* texture_registrar_ = nullptr; diff --git a/windows/texture_rgba_renderer_plugin_c_api.cpp b/windows/texture_rgba_renderer_plugin_c_api.cpp index 58285f0..9e9a667 100644 --- a/windows/texture_rgba_renderer_plugin_c_api.cpp +++ b/windows/texture_rgba_renderer_plugin_c_api.cpp @@ -14,7 +14,7 @@ void TextureRgbaRendererPluginCApiRegisterWithRegistrar( ->GetRegistrar(registrar)); } -void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height) +void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes) { assert(texture_rgba != nullptr); assert(buffer != nullptr); @@ -25,6 +25,6 @@ void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, } TextureRgba *rgba = static_cast(texture_rgba); - std::vector data(buffer, buffer + width * height * 4); + std::vector data(buffer, buffer + len); rgba->MarkVideoFrameAvailable(data, width, height); }