Feat/row align bytes (#7)

* better bytes alignment

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* debug macos

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* debug linux

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* windows switch_rgba

Signed-off-by: fufesou <shuanglongchen@yeah.net>

* better switch_rgba

Signed-off-by: fufesou <shuanglongchen@yeah.net>

---------

Signed-off-by: fufesou <shuanglongchen@yeah.net>
This commit is contained in:
fufesou 2023-03-03 09:30:05 +08:00 • committed by GitHub
parent 044bb260d9
commit 873919902e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 94 additions and 60 deletions

View file

@ -118,17 +118,17 @@ class TextureRgbaRenderer {
Note that the `onRgba` function is not called immediately. It depends on the execution scheduler of Flutter. To solve the problem, the plugin provide the low level function on all 3 pc platforms, which can make them be executed immediately.
```c
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height)
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes)
```
You could use the native API in other programing languages. A dart quick example:
```dart
/// The `Native` class is bundled in this plugin.
typedef F1 = Void Function(
Pointer<Void> ptr, Pointer<Uint8> buffer, Int32 width, Int32 height);
typedef F1Dart = void Function(
Pointer<Void> ptr, Pointer<Uint8> buffer, int width, int height);
typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, Int len,
Int32 width, Int32 height, Int row_align_bytes);
typedef F1Dart = void Function(Pointer<Void> ptr, Pointer<Uint8> buffer,
int len, int width, int height, int row_align_bytes);
class Native {
Native._();

View file

@ -1,3 +1,4 @@
import 'dart:io';
import 'dart:ffi';
import 'dart:math';
import 'dart:typed_data';
@ -24,7 +25,7 @@ class _MyAppState extends State<MyApp> {
final _textureRgbaRendererPlugin = TextureRgbaRenderer();
int textureId = -1;
int height = 500;
int width = 528;
int width = 500;
int cnt = 0;
var key = 0;
int texturePtr = 0;
@ -33,6 +34,7 @@ class _MyAppState extends State<MyApp> {
Timer? _timer;
int time = 0;
int method = 0;
final rowAlignBytes = Platform.isMacOS ? 64 : 1;
@override
void initState() {
@ -58,16 +60,18 @@ class _MyAppState extends State<MyApp> {
void start(int methodId) {
debugPrint("start mockPic");
method = methodId;
final rowBytes = (width * 4 + rowAlignBytes - 1) & (~(rowAlignBytes - 1));
final picDataLength = rowBytes * height;
_timer?.cancel();
// 60 fps
_timer =
Timer.periodic(const Duration(milliseconds: 1000 ~/ 60), (timer) async {
if (methodId == 0) {
// Method.1: with MethodChannel
data = mockPicture(width, height);
data = mockPicture(width, height, rowBytes, picDataLength);
final t1 = DateTime.now().microsecondsSinceEpoch;
final res = await _textureRgbaRendererPlugin.onRgba(
key, data!, height, width, 16);
key, data!, height, width, rowAlignBytes);
final t2 = DateTime.now().microsecondsSinceEpoch;
setState(() {
time = t2 - t1;
@ -76,11 +80,11 @@ class _MyAppState extends State<MyApp> {
debugPrint("WARN: render failed");
}
} else {
final dataPtr = mockPicturePtr(width, height);
final dataPtr = mockPicturePtr(width, height, rowBytes, picDataLength);
// Method.2: with native ffi
final t1 = DateTime.now().microsecondsSinceEpoch;
Native.instance.onRgba(Pointer.fromAddress(texturePtr).cast<Void>(),
dataPtr, width, height, 16);
dataPtr, picDataLength, width, height, rowAlignBytes);
final t2 = DateTime.now().microsecondsSinceEpoch;
setState(() {
time = t2 - t1;
@ -99,27 +103,27 @@ class _MyAppState extends State<MyApp> {
super.dispose();
}
Uint8List mockPicture(int width, int height) {
final pic = List.generate(width * height * 4, (index) {
final h = (index / 4) / width;
final w = (index / 4) % width;
Uint8List mockPicture(int width, int height, int rowBytes, int length) {
final pic = List.generate(length, (index) {
final r = index / rowBytes;
final c = (index % rowBytes) / 4;
final p = index & 0x03;
if (h > 20 && h < 30) {
if (w > 20 && w < 25) {
if (c > 20 && c < 30) {
if (r > 20 && r < 25) {
if (p == 0 || p == 3) {
return 255;
} else {
return 0;
}
}
if (w > 40 && w < 45) {
if (r > 40 && r < 45) {
if (p == 1 || p == 3) {
return 255;
} else {
return 0;
}
}
if (w > 60 && w < 65) {
if (r > 60 && r < 65) {
if (p == 2 || p == 3) {
return 255;
} else {
@ -132,13 +136,14 @@ class _MyAppState extends State<MyApp> {
return Uint8List.fromList(pic);
}
Pointer<Uint8> mockPicturePtr(int width, int height) {
final pic = List.generate(width * height * 4, (index) {
final h = (index / 4) / width;
final w = (index / 4) % width;
Pointer<Uint8> mockPicturePtr(
int width, int height, int rowBytes, int length) {
final pic = List.generate(length, (index) {
final r = index / rowBytes;
final c = (index % rowBytes) / 4;
final p = index & 0x03;
final edgeH = (h >= 0 && h < 10) || ((h >= height - 10) && h < height);
final edgeW = (w >= 0 && w < 10) || ((w >= width - 10) && w < width);
final edgeH = (c >= 0 && c < 10) || ((c >= height - 10) && c < height);
final edgeW = (r >= 0 && r < 10) || ((r >= width - 10) && r < width);
if (edgeH || edgeW) {
if (p == 0 || p == 3) {
return 255;

View file

@ -1,9 +1,9 @@
import 'dart:ffi';
typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer,
Int32 width, Int32 height, Int stride_align);
typedef F1 = Void Function(Pointer<Void> ptr, Pointer<Uint8> buffer, Int len,
Int32 width, Int32 height, Int row_align_bytes);
typedef F1Dart = void Function(Pointer<Void> ptr, Pointer<Uint8> buffer,
int width, int height, int stride_align);
int len, int width, int height, int row_align_bytes);
class Native {
Native._();

View file

@ -24,9 +24,9 @@ class TextureRgbaRenderer {
/// Provide the rgba data to the texture.
Future<bool> onRgba(
int key, Uint8List data, int height, int width, int stride_align) {
int key, Uint8List data, int height, int width, int row_align_bytes) {
return TextureRgbaRendererPlatform.instance
.onRgba(key, data, height, width, stride_align);
.onRgba(key, data, height, width, row_align_bytes);
}
Future<int> getTexturePtr(int key) {

View file

@ -37,14 +37,14 @@ class MethodChannelTextureRgbaRenderer extends TextureRgbaRendererPlatform {
}
@override
Future<bool> onRgba(
int key, Uint8List data, int height, int width, int stride_align) async {
Future<bool> onRgba(int key, Uint8List data, int height, int width,
int row_align_bytes) async {
return await methodChannel.invokeMethod<bool>('onRgba', {
"data": data,
"height": height,
"width": width,
"key": key,
"stride_align": stride_align,
"row_align_bytes": row_align_bytes,
}) ??
false;
}

View file

@ -40,7 +40,7 @@ abstract class TextureRgbaRendererPlatform extends PlatformInterface {
}
Future<bool> onRgba(
int key, Uint8List data, int height, int width, int stride_align) {
int key, Uint8List data, int height, int width, int row_align_bytes) {
throw UnimplementedError('onRgba() has not been implemented.');
}

View file

@ -39,13 +39,18 @@ G_DEFINE_TYPE(TextureRgba, texture_rgba,
(G_TYPE_CHECK_INSTANCE_CAST((obj), texture_rgba_get_type(), \
TextureRgba))
static inline void switch_rgba(uint8_t* pixels, int width, int height) {
static inline void switch_rgba(uint8_t* pixels, int len, int height) {
uint8_t temp;
for (int i = 0; i < width * height; i++) {
temp = pixels[4 * i];
pixels[4 * i] = pixels[4 * i + 2];
pixels[4 * i + 2] = temp;
int byes_per_row = len / height;
int width = byes_per_row >> 2;
for (int i = 0; i < height; ++i) {
int row_idx_base = i * byes_per_row;
for (int k = 0; k < width; ++k) {
int idx = row_idx_base + k * 4;
temp = pixels[idx];
pixels[idx] = pixels[idx + 2];
pixels[idx + 2] = temp;
}
}
}

View file

@ -22,7 +22,7 @@ FLUTTER_PLUGIN_EXPORT void texture_rgba_renderer_plugin_register_with_registrar(
FlPluginRegistrar* registrar);
// Keep the same symbol name with Windows.
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height);
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes);
G_END_DECLS

View file

@ -14,7 +14,7 @@
std::unordered_map<int64_t, TextureRgba *> g_renderer_map;
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height);
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height);
struct _TextureRgbaRendererPlugin
{
GObject parent_instance;
@ -68,11 +68,12 @@ static void texture_rgba_renderer_plugin_handle_method_call(
auto args = fl_method_call_get_args(method_call);
auto key = fl_value_get_int(fl_value_lookup_string(args, "key"));
auto data = fl_value_get_uint8_list(fl_value_lookup_string(args, "data"));
// auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data"));
auto data_length = fl_value_get_length(fl_value_lookup_string(args, "data"));
auto width = fl_value_get_int(fl_value_lookup_string(args, "width"));
auto height = fl_value_get_int(fl_value_lookup_string(args, "height"));
auto row_align_bytes = fl_value_get_int(fl_value_lookup_string(args, "row_align_bytes"));
auto texture_rgba = g_renderer_map[key];
FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, width, height);
FlutterRgbaRendererPluginOnRgba((void*)texture_rgba, data, data_length, width, height, row_align_bytes);
response = FL_METHOD_RESPONSE(fl_method_success_response_new(
fl_value_new_bool(fl_texture_registrar_mark_texture_frame_available(self->texture_registrar, FL_TEXTURE(texture_rgba)))));
}
@ -140,16 +141,15 @@ void texture_rgba_renderer_plugin_register_with_registrar(FlPluginRegistrar *reg
extern "C" {
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int width, int height) {
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba_ptr, const uint8_t *buffer, int len, int width, int height, int row_align_bytes) {
TextureRgba* self = TEXTURE_RGBA(texture_rgba_ptr);
g_mutex_lock(&self->mutex);
// selfate has registered a texture_id,
if (self->texture_id != 0 && !g_atomic_int_get(&self->buffer_ready) && !texture_rgba_is_terminate(self)) {
// copy data to the texture.
auto buffer_length = 4 * width * height;
auto copied_data = new uint8_t[buffer_length];
memcpy(copied_data, buffer, buffer_length);
switch_rgba(copied_data, width, height);
auto copied_data = new uint8_t[len];
memcpy(copied_data, buffer, len);
switch_rgba(copied_data, len, height);
// It's safe to working on a non reading index
g_atomic_pointer_set(&self->buffer, copied_data);
g_atomic_int_set(&self->video_height, height);

View file

@ -34,7 +34,7 @@ import CoreVideo
}
@objc public func markFrameAvaliable(data: Data, width: Int, height: Int, stride_align: Int) -> Bool {
@objc public func markFrameAvaliable(data: Data, width: Int, height: Int, row_align_bytes: Int) -> Bool {
queue.sync {
self.data = data.withUnsafeBytes { buffer in
var pixelBufferCopy: CVPixelBuffer!
@ -44,7 +44,7 @@ import CoreVideo
kCVPixelBufferMetalCompatibilityKey: true,
kCVPixelBufferOpenGLCompatibilityKey: true,
// https://developer.apple.com/forums/thread/712709
kCVPixelBufferBytesPerRowAlignmentKey: stride_align * 4
kCVPixelBufferBytesPerRowAlignmentKey: 64
] as CFDictionary, &pixelBufferCopy)
guard result == kCVReturnSuccess else {
return nil

View file

@ -13,10 +13,10 @@ extern "C" {
#endif
/// Keep the same name with Windows.
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int width, int height, int stride_align) {
void FlutterRgbaRendererPluginOnRgba(void* texture_rgba_ptr, const uint8_t* buffer, int len, int width, int height, int row_align_bytes) {
TextRgba* texture_rgba = (__bridge TextRgba *)(texture_rgba_ptr);
NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:((width + stride_align - 1) & ~(stride_align - 1)) * height * 4 freeWhenDone:FALSE];
[texture_rgba markFrameAvaliableWithData:data width:width height:height stride_align: stride_align];
NSData* data = [NSData dataWithBytesNoCopy:(void*)buffer length:len freeWhenDone:FALSE];
[texture_rgba markFrameAvaliableWithData:data width:width height:height row_align_bytes: row_align_bytes];
}
#if __cplusplus

View file

@ -49,12 +49,12 @@ public class TextureRgbaRendererPlugin: NSObject, FlutterPlugin {
let data = args["data"] as! FlutterStandardTypedData
let height = args["height"] as! Int
let width = args["width"] as! Int
let stride_align = args["stride_align"] as! Int
let row_align_bytes = args["row_align_bytes"] as! Int
let textureRgba = renderer[key]
if (textureRgba == nil) {
result(false)
} else {
result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, stride_align: stride_align))
result(textureRgba!.markFrameAvaliable(data: data.data, width: width, height: height, row_align_bytes: row_align_bytes))
}
case "getTexturePtr":
let args = call.arguments as! Dictionary<String, Any?>

View file

@ -16,7 +16,7 @@ extern "C" {
FLUTTER_PLUGIN_EXPORT void TextureRgbaRendererPluginCApiRegisterWithRegistrar(
FlutterDesktopPluginRegistrarRef registrar);
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int width, int height);
FLUTTER_PLUGIN_EXPORT void FlutterRgbaRendererPluginOnRgba(void* texture_rgba, const uint8_t* buffer, int len, int width, int height, int row_align_bytes);
#if defined(__cplusplus)
} // extern "C"

View file

@ -1,7 +1,6 @@
#include "texture_rgba.h"
#include <iostream>
TextureRgba::TextureRgba(flutter::TextureRegistrar *texture_registrar)
{
this->texture_registrar_ = texture_registrar;
@ -17,14 +16,36 @@ TextureRgba::~TextureRgba()
texture_registrar_->UnregisterTexture(texture_id_);
}
void TextureRgba::switch_rgba(std::vector<uint8_t> &buffer, size_t height)
{
uint8_t temp;
size_t byes_per_row = buffer.size() / height;
size_t width = byes_per_row >> 2;
for (size_t i = 0; i < height; ++i)
{
size_t row_idx_base = i * byes_per_row;
for (size_t k = 0; k < width; ++k)
{
size_t idx = row_idx_base + k * 4;
temp = buffer[idx];
buffer[idx] = buffer[idx + 2];
buffer[idx + 2] = temp;
}
}
}
void TextureRgba::MarkVideoFrameAvailable(
std::vector<uint8_t> &buffer, size_t width, size_t height)
{
if (buffer.empty() || buffer.size() != width * height * 4)
if (buffer.empty())
{
return;
}
// Do not check bellow. Because the data may have paddings per row.
// buffer.size() != width * height * 4
switch_rgba(buffer, height);
const std::lock_guard<std::mutex> lock(mutex_);
int bg_index = fg_index_ ^ 1;
buffer.swap(buffer_tmp_[bg_index]);

View file

@ -17,6 +17,9 @@ public:
const FlutterDesktopPixelBuffer* buffer();
private:
void switch_rgba(std::vector<uint8_t>& buffer, size_t height);
private:
FlutterDesktopPixelBuffer flutter_pixel_buffer_{};
flutter::TextureRegistrar* texture_registrar_ = nullptr;

View file

@ -14,7 +14,7 @@ void TextureRgbaRendererPluginCApiRegisterWithRegistrar(
->GetRegistrar<flutter::PluginRegistrarWindows>(registrar));
}
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int width, int height)
void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer, int len, int width, int height, int row_align_bytes)
{
assert(texture_rgba != nullptr);
assert(buffer != nullptr);
@ -25,6 +25,6 @@ void FlutterRgbaRendererPluginOnRgba(void *texture_rgba, const uint8_t *buffer,
}
TextureRgba *rgba = static_cast<TextureRgba *>(texture_rgba);
std::vector<uint8_t> data(buffer, buffer + width * height * 4);
std::vector<uint8_t> data(buffer, buffer + len);
rgba->MarkVideoFrameAvailable(data, width, height);
}