Feat/row align bytes (#7)

* better bytes alignment

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* debug macos

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* debug linux

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* windows switch_rgba

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* better switch_rgba

Signed-off-by: fufesou <shuanglongchen@yeah.net>

---------

Signed-off-by: fufesou <shuanglongchen@yeah.net>
This commit is contained in:
fufesou 2023-03-03 09:30:05 +08:00 • committed by GitHub
parent 044bb260d9
commit 873919902e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 94 additions and 60 deletions

View file

@ -118,17 +118,17 @@ class TextureRgbaRenderer {
Note that the `onRgba` function is not called immediately. It depends on the execution scheduler of Flutter. To solve the problem, the plugin provide the low level function on all 3 pc platforms, which can make them be executed immediately. Note that the `onRgba` function is not called immediately. It depends on the execution scheduler of Flutter. To solve the problem, the plugin provide the low level function on all 3 pc platforms, which can make them be executed immediately.
```c ```c
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height) void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes)
``` ```
You could use the native API in other programing languages. A dart quick example: You could use the native API in other programing languages. A dart quick example:
```dart ```dart
/// The `Native` class is bundled in this plugin. /// The `Native` class is bundled in this plugin.
typedef F1 = Void Function( typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, Int len,
Pointer<Void> ptr, Pointer<Uint8> buffer, Int32 width, Int32 height); Int32 width, Int32 height, Int row_align_bytes);
typedef F1Dart = void Function( typedef F1Dart = void Function(Pointer<Void> ptr, Pointer<Uint8> buffer,
Pointer<Void> ptr, Pointer<Uint8> buffer, int width, int height); int len, int width, int height, int row_align_bytes);
class Native { class Native {
Native._(); Native._();

View file

@ -1,3 +1,4 @@
import 'dart:io';
import 'dart:ffi'; import 'dart:ffi';
import 'dart:math'; import 'dart:math';
import 'dart:typed_data'; import 'dart:typed_data';
@ -24,7 +25,7 @@ class _MyAppState extends State<MyApp> {
final _textureRgbaRendererPlugin = TextureRgbaRenderer(); final _textureRgbaRendererPlugin = TextureRgbaRenderer();
int textureId = -1; int textureId = -1;
int height = 500; int height = 500;
int width = 528; int width = 500;
int cnt = 0; int cnt = 0;
var key = 0; var key = 0;
int texturePtr = 0; int texturePtr = 0;
@ -33,6 +34,7 @@ class _MyAppState extends State<MyApp> {
Timer? _timer; Timer? _timer;
int time = 0; int time = 0;
int method = 0; int method = 0;
final rowAlignBytes = Platform.isMacOS ? 64 : 1;
@override @override
void initState() { void initState() {
@ -58,16 +60,18 @@ class _MyAppState extends State<MyApp> {
void start(int methodId) { void start(int methodId) {
debugPrint("start mockPic"); debugPrint("start mockPic");
method = methodId; method = methodId;
final rowBytes = (width * 4 + rowAlignBytes - 1) & (~(rowAlignBytes - 1));
final picDataLength = rowBytes * height;
_timer?.cancel(); _timer?.cancel();
// 60 fps // 60 fps
_timer = _timer =
Timer.periodic(const Duration(milliseconds: 1000 ~/ 60), (timer) async { Timer.periodic(const Duration(milliseconds: 1000 ~/ 60), (timer) async {
if (methodId == 0) { if (methodId == 0) {
// Method.1: with MethodChannel // Method.1: with MethodChannel
data = mockPicture(width, height); data = mockPicture(width, height, rowBytes, picDataLength);
final t1 = DateTime.now().microsecondsSinceEpoch; final t1 = DateTime.now().microsecondsSinceEpoch;
final res = await _textureRgbaRendererPlugin.onRgba( final res = await _textureRgbaRendererPlugin.onRgba(
key, data!, height, width, 16); key, data!, height, width, rowAlignBytes);
final t2 = DateTime.now().microsecondsSinceEpoch; final t2 = DateTime.now().microsecondsSinceEpoch;
setState(() { setState(() {
time = t2 - t1; time = t2 - t1;
@ -76,11 +80,11 @@ class _MyAppState extends State<MyApp> {
debugPrint("WARN: render failed"); debugPrint("WARN: render failed");
} }
} else { } else {
final dataPtr = mockPicturePtr(width, height); final dataPtr = mockPicturePtr(width, height, rowBytes, picDataLength);
// Method.2: with native ffi // Method.2: with native ffi
final t1 = DateTime.now().microsecondsSinceEpoch; final t1 = DateTime.now().microsecondsSinceEpoch;
Native.instance.onRgba(Pointer.fromAddress(texturePtr).cast<Void>(), Native.instance.onRgba(Pointer.fromAddress(texturePtr).cast<Void>(),
dataPtr, width, height, 16); dataPtr, picDataLength, width, height, rowAlignBytes);
final t2 = DateTime.now().microsecondsSinceEpoch; final t2 = DateTime.now().microsecondsSinceEpoch;
setState(() { setState(() {
time = t2 - t1; time = t2 - t1;
@ -99,27 +103,27 @@ class _MyAppState extends State<MyApp> {
super.dispose(); super.dispose();
} }
Uint8List mockPicture(int width, int height) { Uint8List mockPicture(int width, int height, int rowBytes, int length) {
final pic = List.generate(width * height * 4, (index) { final pic = List.generate(length, (index) {
final h = (index / 4) / width; final r = index / rowBytes;
final w = (index / 4) % width; final c = (index % rowBytes) / 4;
final p = index & 0x03; final p = index & 0x03;
if (h > 20 && h < 30) { if (c > 20 && c < 30) {
if (w > 20 && w < 25) { if (r > 20 && r < 25) {
if (p == 0 || p == 3) { if (p == 0 || p == 3) {
return 255; return 255;
} else { } else {
return 0; return 0;
} }
} }
if (w > 40 && w < 45) { if (r > 40 && r < 45) {
if (p == 1 || p == 3) { if (p == 1 || p == 3) {
return 255; return 255;
} else { } else {
return 0; return 0;
} }
} }
if (w > 60 && w < 65) { if (r > 60 && r < 65) {
if (p == 2 || p == 3) { if (p == 2 || p == 3) {
return 255; return 255;
} else { } else {
@ -132,13 +136,14 @@ class _MyAppState extends State<MyApp> {
return Uint8List.fromList(pic); return Uint8List.fromList(pic);
} }
Pointer<Uint8> mockPicturePtr(int width, int height) { Pointer<Uint8> mockPicturePtr(
final pic = List.generate(width * height * 4, (index) { int width, int height, int rowBytes, int length) {
final h = (index / 4) / width; final pic = List.generate(length, (index) {
final w = (index / 4) % width; final r = index / rowBytes;
final c = (index % rowBytes) / 4;
final p = index & 0x03; final p = index & 0x03;
final edgeH = (h >= 0 && h < 10) || ((h >= height - 10) && h < height); final edgeH = (c >= 0 && c < 10) || ((c >= height - 10) && c < height);
final edgeW = (w >= 0 && w < 10) || ((w >= width - 10) && w < width); final edgeW = (r >= 0 && r < 10) || ((r >= width - 10) && r < width);
if (edgeH || edgeW) { if (edgeH || edgeW) {
if (p == 0 || p == 3) { if (p == 0 || p == 3) {
return 255; return 255;

View file

@ -1,9 +1,9 @@
import 'dart:ffi'; import 'dart:ffi';
typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, Int len,
Int32 width, Int32 height, Int stride_align); Int32 width, Int32 height, Int row_align_bytes);
typedef F1Dart = void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, typedef F1Dart = void Function(Pointer<Void> ptr, Pointer<Uint8> buffer,
int width, int height, int stride_align); int len, int width, int height, int row_align_bytes);
class Native { class Native {
Native._(); Native._();

View file

@ -24,9 +24,9 @@ class TextureRgbaRenderer {
/// Provide the rgba data to the texture. /// Provide the rgba data to the texture.
Future<bool> onRgba( Future<bool> onRgba(
int key, Uint8List data, int height, int width, int stride_align) { int key, Uint8List data, int height, int width, int row_align_bytes) {
return TextureRgbaRendererPlatform.instance return TextureRgbaRendererPlatform.instance
.onRgba(key, data, height, width, stride_align); .onRgba(key, data, height, width, row_align_bytes);
} }
Future<int> getTexturePtr(int key) { Future<int> getTexturePtr(int key) {

View file

@ -37,14 +37,14 @@ class MethodChannelTextureRgbaRenderer extends TextureRgbaRendererPlatform {
} }
@override @override
Future<bool> onRgba( Future<bool> onRgba(int key, Uint8List data, int height, int width,
int key, Uint8List data, int height, int width, int stride_align) async { int row_align_bytes) async {
return await methodChannel.invokeMethod<bool>('onRgba', { return await methodChannel.invokeMethod<bool>('onRgba', {
"data": data, "data": data,
"height": height, "height": height,
"width": width, "width": width,
"key": key, "key": key,
"stride_align": stride_align, "row_align_bytes": row_align_bytes,
}) ?? }) ??
false; false;
} }

View file

@ -40,7 +40,7 @@ abstract class TextureRgbaRendererPlatform extends PlatformInterface {
} }
Future<bool> onRgba( Future<bool> onRgba(
int key, Uint8List data, int height, int width, int stride_align) { int key, Uint8List data, int height, int width, int row_align_bytes) {
throw UnimplementedError('onRgba() has not been implemented.'); throw UnimplementedError('onRgba() has not been implemented.');
} }

View file

@ -39,13 +39,18 @@ G_DEFINE_TYPE(TextureRgba, texture_rgba,
(G_TYPE_CHECK_INSTANCE_CAST((obj), texture_rgba_get_type(), \ (G_TYPE_CHECK_INSTANCE_CAST((obj), texture_rgba_get_type(), \
TextureRgba)) TextureRgba))
static inline void switch_rgba(uint8_t* pixels, int width, int height) { static inline void switch_rgba(uint8_t* pixels, int len, int height) {
uint8_t temp; uint8_t temp;
int byes_per_row = len / height;
for (int i = 0; i < width * height; i++) { int width = byes_per_row >> 2;
temp = pixels[4 * i]; for (int i = 0; i < height; ++i) {
pixels[4 * i] = pixels[4 * i + 2]; int row_idx_base = i * byes_per_row;
pixels[4 * i + 2] = temp; for (int k = 0; k < width; ++k) {
int idx = row_idx_base + k * 4;
temp = pixels[idx];
pixels[idx] = pixels[idx + 2];
pixels[idx + 2] = temp;
}
} }
} }

View file

@ -22,7 +22,7 @@ FLUTTER_PLUGIN_EXPORT void texture_rgba_renderer_plugin_register_with_registrar(
FlPluginRegistrar* registrar); FlPluginRegistrar* registrar);
// Keep the same symbol name with Windows. // Keep the same symbol name with Windows.
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height); FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes);
G_END_DECLS G_END_DECLS

View file

@ -14,7 +14,7 @@
std::unordered_map<int64_t, TextureRgba *> g_renderer_map; std::unordered_map<int64_t, TextureRgba *> g_renderer_map;
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height); void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height);
struct _TextureRgbaRendererPlugin struct _TextureRgbaRendererPlugin
{ {
GObject parent_instance; GObject parent_instance;
@ -68,11 +68,12 @@ static void texture_rgba_renderer_plugin_handle_method_call(
auto args = fl_method_call_get_args(method_call); auto args = fl_method_call_get_args(method_call);
auto key = fl_value_get_int(fl_value_lookup_string(args, "key")); auto key = fl_value_get_int(fl_value_lookup_string(args, "key"));
auto data = fl_value_get_uint8_list(fl_value_lookup_string(args, "data")); auto data = fl_value_get_uint8_list(fl_value_lookup_string(args, "data"));
// auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data")); auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data"));
auto width = fl_value_get_int(fl_value_lookup_string(args, "width")); auto width = fl_value_get_int(fl_value_lookup_string(args, "width"));
auto height = fl_value_get_int(fl_value_lookup_string(args, "height")); auto height = fl_value_get_int(fl_value_lookup_string(args, "height"));
auto row_align_bytes = fl_value_get_int(fl_value_lookup_string(args, "row_align_bytes"));
auto texture_rgba = g_renderer_map[key]; auto texture_rgba = g_renderer_map[key];
FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, width, height); FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, data_length, width, height, row_align_bytes);
response = FL_METHOD_RESPONSE(fl_method_success_response_new( response = FL_METHOD_RESPONSE(fl_method_success_response_new(
fl_value_new_bool(fl_texture_registrar_mark_texture_frame_available(self->texture_registrar, FL_TEXTURE(texture_rgba))))); fl_value_new_bool(fl_texture_registrar_mark_texture_frame_available(self->texture_registrar, FL_TEXTURE(texture_rgba)))));
} }
@ -140,16 +141,15 @@ void texture_rgba_renderer_plugin_register_with_registrar(FlPluginRegistrar *reg
extern "C" { extern "C" {
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height) { void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height, int row_align_bytes) {
TextureRgba* self = TEXTURE_RGBA(texture_rgba_ptr); TextureRgba* self = TEXTURE_RGBA(texture_rgba_ptr);
g_mutex_lock(&self->mutex); g_mutex_lock(&self->mutex);
// selfate has registered a texture_id, // selfate has registered a texture_id,
if (self->texture_id != 0 && !g_atomic_int_get(&self->buffer_ready) && !texture_rgba_is_terminate(self)) { if (self->texture_id != 0 && !g_atomic_int_get(&self->buffer_ready) && !texture_rgba_is_terminate(self)) {
// copy data to the texture. // copy data to the texture.
auto buffer_length = 4 * width * height; auto copied_data = new uint8_t[len];
auto copied_data = new uint8_t[buffer_length]; memcpy(copied_data, buffer, len);
memcpy(copied_data, buffer, buffer_length); switch_rgba(copied_data, len, height);
switch_rgba(copied_data, width, height);
// It's safe to working on a non reading index // It's safe to working on a non reading index
g_atomic_pointer_set(&self->buffer, copied_data); g_atomic_pointer_set(&self->buffer, copied_data);
g_atomic_int_set(&self->video_height, height); g_atomic_int_set(&self->video_height, height);

View file

@ -34,7 +34,7 @@ import CoreVideo
} }
@objc public func markFrameAvaliable(data: Data, width: Int, height: Int, stride_align: Int) -> Bool { @objc public func markFrameAvaliable(data: Data, width: Int, height: Int, row_align_bytes: Int) -> Bool {
queue.sync { queue.sync {
self.data = data.withUnsafeBytes { buffer in self.data = data.withUnsafeBytes { buffer in
var pixelBufferCopy: CVPixelBuffer! var pixelBufferCopy: CVPixelBuffer!
@ -44,7 +44,7 @@ import CoreVideo
kCVPixelBufferMetalCompatibilityKey: true, kCVPixelBufferMetalCompatibilityKey: true,
kCVPixelBufferOpenGLCompatibilityKey: true, kCVPixelBufferOpenGLCompatibilityKey: true,
// https://developer.apple.com/forums/thread/712709 // https://developer.apple.com/forums/thread/712709
kCVPixelBufferBytesPerRowAlignmentKey: stride_align * 4 kCVPixelBufferBytesPerRowAlignmentKey: 64
] as CFDictionary, &pixelBufferCopy) ] as CFDictionary, &pixelBufferCopy)
guard result == kCVReturnSuccess else { guard result == kCVReturnSuccess else {
return nil return nil

View file

@ -13,10 +13,10 @@ extern "C" {
#endif #endif
/// Keep the same name with Windows. /// Keep the same name with Windows.
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height, int stride_align) { void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes) {
TextRgba* texture_rgba = (__bridge TextRgba *)(texture_rgba_ptr); TextRgba* texture_rgba = (__bridge TextRgba *)(texture_rgba_ptr);
NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:((width + stride_align - 1) & ~(stride_align - 1)) * height * 4 freeWhenDone:FALSE]; NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:len freeWhenDone:FALSE];
[texture_rgba markFrameAvaliableWithData:data width:width height:height stride_align: stride_align]; [texture_rgba markFrameAvaliableWithData:data width:width height:height row_align_bytes: row_align_bytes];
} }
#if __cplusplus #if __cplusplus

View file

@ -49,12 +49,12 @@ public class TextureRgbaRendererPlugin: NSObject, FlutterPlugin {
let data = args["data"] as! FlutterStandardTypedData let data = args["data"] as! FlutterStandardTypedData
let height = args["height"] as! Int let height = args["height"] as! Int
let width = args["width"] as! Int let width = args["width"] as! Int
let stride_align = args["stride_align"] as! Int let row_align_bytes = args["row_align_bytes"] as! Int
let textureRgba = renderer[key] let textureRgba = renderer[key]
if (textureRgba == nil) { if (textureRgba == nil) {
result(false) result(false)
} else { } else {
result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, stride_align: stride_align)) result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, row_align_bytes: row_align_bytes))
} }
case "getTexturePtr": case "getTexturePtr":
let args = call.arguments as! Dictionary<String, Any?> let args = call.arguments as! Dictionary<String, Any?>

View file

@ -16,7 +16,7 @@ extern "C" {
FLUTTER_PLUGIN_EXPORT void TextureRgbaRendererPluginCApiRegisterWithRegistrar( FLUTTER_PLUGIN_EXPORT void TextureRgbaRendererPluginCApiRegisterWithRegistrar(
FlutterDesktopPluginRegistrarRef registrar); FlutterDesktopPluginRegistrarRef registrar);
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int width, int height); FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int len, int width, int height, int row_align_bytes);
#if defined(__cplusplus) #if defined(__cplusplus)
} // extern "C" } // extern "C"

View file

@ -1,7 +1,6 @@
#include "texture_rgba.h" #include "texture_rgba.h"
#include <iostream> #include <iostream>
TextureRgba::TextureRgba(flutter::TextureRegistrar *texture_registrar) TextureRgba::TextureRgba(flutter::TextureRegistrar *texture_registrar)
{ {
this->texture_registrar_ = texture_registrar; this->texture_registrar_ = texture_registrar;
@ -17,14 +16,36 @@ TextureRgba::~TextureRgba()
texture_registrar_->UnregisterTexture(texture_id_); texture_registrar_->UnregisterTexture(texture_id_);
} }
void TextureRgba::switch_rgba(std::vector<uint8_t> &buffer, size_t height)
{
uint8_t temp;
size_t byes_per_row = buffer.size() / height;
size_t width = byes_per_row >> 2;
for (size_t i = 0; i < height; ++i)
{
size_t row_idx_base = i * byes_per_row;
for (size_t k = 0; k < width; ++k)
{
size_t idx = row_idx_base + k * 4;
temp = buffer[idx];
buffer[idx] = buffer[idx + 2];
buffer[idx + 2] = temp;
}
}
}
void TextureRgba::MarkVideoFrameAvailable( void TextureRgba::MarkVideoFrameAvailable(
std::vector<uint8_t> &buffer, size_t width, size_t height) std::vector<uint8_t> &buffer, size_t width, size_t height)
{ {
if (buffer.empty() || buffer.size() != width * height * 4) if (buffer.empty())
{ {
return; return;
} }
// Do not check bellow. Because the data may have paddings per row.
// buffer.size() != width * height * 4
switch_rgba(buffer, height);
const std::lock_guard<std::mutex> lock(mutex_); const std::lock_guard<std::mutex> lock(mutex_);
int bg_index = fg_index_ ^ 1; int bg_index = fg_index_ ^ 1;
buffer.swap(buffer_tmp_[bg_index]); buffer.swap(buffer_tmp_[bg_index]);

View file

@ -17,6 +17,9 @@ public:
const FlutterDesktopPixelBuffer* buffer(); const FlutterDesktopPixelBuffer* buffer();
private:
void switch_rgba(std::vector<uint8_t>& buffer, size_t height);
private: private:
FlutterDesktopPixelBuffer flutter_pixel_buffer_{}; FlutterDesktopPixelBuffer flutter_pixel_buffer_{};
flutter::TextureRegistrar* texture_registrar_ = nullptr; flutter::TextureRegistrar* texture_registrar_ = nullptr;

View file

@ -14,7 +14,7 @@ void TextureRgbaRendererPluginCApiRegisterWithRegistrar(
->GetRegistrar<flutter::PluginRegistrarWindows>(registrar)); ->GetRegistrar<flutter::PluginRegistrarWindows>(registrar));
} }
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height) void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes)
{ {
assert(texture_rgba != nullptr); assert(texture_rgba != nullptr);
assert(buffer != nullptr); assert(buffer != nullptr);
@ -25,6 +25,6 @@ void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer,
} }
TextureRgba *rgba = static_cast<TextureRgba *>(texture_rgba); TextureRgba *rgba = static_cast<TextureRgba *>(texture_rgba);
std::vector<uint8_t> data(buffer, buffer + width * height * 4); std::vector<uint8_t> data(buffer, buffer + len);
rgba->MarkVideoFrameAvailable(data, width, height); rgba->MarkVideoFrameAvailable(data, width, height);
} }