Merge branch 'soru/file-resize-isolate' into 'main'

fix: resize images in a separate isolate

See merge request ChristianPauly/fluffychat-flutter!211

CHANGELOG.md

@@ -2,6 +2,8 @@
 ### Features
 - Add ability to enable / disable emotes globally
 - Add ability to manage emote packs with different state keys
+### Changes
+- Re-scale images in a separate isolate to prevent the UI from freezing
 ### Fixes
 - Fix amoled / theme settings not always saving properly
 - Show device name in account information correctly

lib/components/dialogs/send_file_dialog.dart

@@ -1,14 +1,11 @@
-import 'dart:typed_data';
-import 'dart:ui';
-
 import 'package:famedlysdk/famedlysdk.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_gen/gen_l10n/l10n.dart';
-import 'package:native_imaging/native_imaging.dart' as native;
 
 import '../../components/dialogs/simple_dialogs.dart';
 import '../../utils/matrix_file_extension.dart';
 import '../../utils/room_send_file_extension.dart';
+import '../../utils/resize_image.dart';
 
 class SendFileDialog extends StatefulWidget {
   final Room room;
@@ -26,46 +23,8 @@ class _SendFileDialogState extends State<SendFileDialog> {
   Future<void> _send() async {
     var file = widget.file;
     if (file is MatrixImageFile && !origImage) {
-      final imgFile = file as MatrixImageFile;
-      // resize to max 1600 x 1600
       try {
-        await native.init();
+        file = await resizeImage(file, max: 1600);
-        var nativeImg = native.Image();
-        try {
-          await nativeImg.loadEncoded(imgFile.bytes);
-          imgFile.width = nativeImg.width();
-          imgFile.height = nativeImg.height();
-        } on UnsupportedError {
-          final dartCodec = await instantiateImageCodec(imgFile.bytes);
-          final dartFrame = await dartCodec.getNextFrame();
-          imgFile.width = dartFrame.image.width;
-          imgFile.height = dartFrame.image.height;
-          final rgbaData = await dartFrame.image.toByteData();
-          final rgba = Uint8List.view(
-              rgbaData.buffer, rgbaData.offsetInBytes, rgbaData.lengthInBytes);
-          dartFrame.image.dispose();
-          dartCodec.dispose();
-          nativeImg.loadRGBA(imgFile.width, imgFile.height, rgba);
-        }
-
-        const max = 1600;
-        if (imgFile.width > max || imgFile.height > max) {
-          var w = max, h = max;
-          if (imgFile.width > imgFile.height) {
-            h = max * imgFile.height ~/ imgFile.width;
-          } else {
-            w = max * imgFile.width ~/ imgFile.height;
-          }
-
-          final scaledImg = nativeImg.resample(w, h, native.Transform.lanczos);
-          nativeImg.free();
-          nativeImg = scaledImg;
-        }
-        final jpegBytes = await nativeImg.toJpeg(75);
-        file = MatrixImageFile(
-            bytes: jpegBytes,
-            name: 'scaled_' + imgFile.name.split('.').first + '.jpg');
-        nativeImg.free();
       } catch (e) {
         // couldn't resize
       }

lib/utils/resize_image.dart

@@ -0,0 +1,105 @@
+import 'dart:ui';
+import 'dart:typed_data';
+
+import 'package:famedlysdk/famedlysdk.dart';
+import 'package:native_imaging/native_imaging.dart' as native;
+
+import 'run_in_background.dart';
+
+Future<MatrixImageFile> resizeImage(MatrixImageFile file,
+    {int max = 800}) async {
+  // we want to resize the image in a separate isolate, because otherwise that can
+  // freeze up the UI a bit
+
+  // we can't do width / height fetching in a separate isolate, as that may use the UI stuff
+  await native.init();
+
+  _IsolateArgs args;
+  try {
+    final nativeImg = native.Image();
+    await nativeImg.loadEncoded(file.bytes);
+    file.width = nativeImg.width();
+    file.height = nativeImg.height();
+    args = _IsolateArgs(
+        width: file.width, height: file.height, bytes: file.bytes, max: max);
+    nativeImg.free();
+  } on UnsupportedError {
+    final dartCodec = await instantiateImageCodec(file.bytes);
+    final dartFrame = await dartCodec.getNextFrame();
+    file.width = dartFrame.image.width;
+    file.height = dartFrame.image.height;
+    final rgbaData = await dartFrame.image.toByteData();
+    final rgba = Uint8List.view(
+        rgbaData.buffer, rgbaData.offsetInBytes, rgbaData.lengthInBytes);
+    dartFrame.image.dispose();
+    dartCodec.dispose();
+    args = _IsolateArgs(
+        width: file.width, height: file.height, bytes: rgba, max: max);
+  }
+
+  final res = await runInBackground(_isolateFunction, args);
+  file.blurhash = res.blurhash;
+  final thumbnail = MatrixImageFile(
+    bytes: res.jpegBytes,
+    name: file.name != null
+        ? 'scaled_' + file.name.split('.').first + '.jpg'
+        : 'thumbnail.jpg',
+    mimeType: 'image/jpeg',
+    width: res.width,
+    height: res.height,
+  );
+  // only return the thumbnail if the size actually decreased
+  return thumbnail.size >= file.size ? file : thumbnail;
+}
+
+class _IsolateArgs {
+  final int width;
+  final int height;
+  final Uint8List bytes;
+  final int max;
+  final String name;
+  _IsolateArgs({this.width, this.height, this.bytes, this.max, this.name});
+}
+
+class _IsolateResponse {
+  final String blurhash;
+  final Uint8List jpegBytes;
+  final int width;
+  final int height;
+  _IsolateResponse({this.blurhash, this.jpegBytes, this.width, this.height});
+}
+
+Future<_IsolateResponse> _isolateFunction(_IsolateArgs args) async {
+  await native.init();
+  var nativeImg = native.Image();
+
+  try {
+    await nativeImg.loadEncoded(args.bytes);
+  } on UnsupportedError {
+    nativeImg.loadRGBA(args.width, args.height, args.bytes);
+  }
+  if (args.width > args.max || args.height > args.max) {
+    var w = args.max, h = args.max;
+    if (args.width > args.height) {
+      h = args.max * args.height ~/ args.width;
+    } else {
+      w = args.max * args.width ~/ args.height;
+    }
+
+    final scaledImg = nativeImg.resample(w, h, native.Transform.lanczos);
+    nativeImg.free();
+    nativeImg = scaledImg;
+  }
+  final jpegBytes = await nativeImg.toJpeg(75);
+  final blurhash = nativeImg.toBlurhash(3, 3);
+
+  final ret = _IsolateResponse(
+      blurhash: blurhash,
+      jpegBytes: jpegBytes,
+      width: nativeImg.width(),
+      height: nativeImg.height());
+
+  nativeImg.free();
+
+  return ret;
+}

lib/utils/room_send_file_extension.dart

@@ -16,11 +16,9 @@
  *   along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import 'dart:typed_data';
-import 'dart:ui';
-
 import 'package:famedlysdk/famedlysdk.dart';
-import 'package:native_imaging/native_imaging.dart' as native;
+
+import 'resize_image.dart';
 
 extension RoomSendFileExtension on Room {
   Future<String> sendFileEventWithThumbnail(
@@ -33,50 +31,7 @@ extension RoomSendFileExtension on Room {
     MatrixFile thumbnail;
     try {
       if (file is MatrixImageFile) {
-        await native.init();
+        thumbnail = await resizeImage(file);
-        var nativeImg = native.Image();
-        try {
-          await nativeImg.loadEncoded(file.bytes);
-          file.width = nativeImg.width();
-          file.height = nativeImg.height();
-        } on UnsupportedError {
-          final dartCodec = await instantiateImageCodec(file.bytes);
-          final dartFrame = await dartCodec.getNextFrame();
-          file.width = dartFrame.image.width;
-          file.height = dartFrame.image.height;
-          final rgbaData = await dartFrame.image.toByteData();
-          final rgba = Uint8List.view(
-              rgbaData.buffer, rgbaData.offsetInBytes, rgbaData.lengthInBytes);
-          dartFrame.image.dispose();
-          dartCodec.dispose();
-          nativeImg.loadRGBA(file.width, file.height, rgba);
-        }
-
-        const max = 800;
-        if (file.width > max || file.height > max) {
-          var w = max, h = max;
-          if (file.width > file.height) {
-            h = max * file.height ~/ file.width;
-          } else {
-            w = max * file.width ~/ file.height;
-          }
-
-          final scaledImg = nativeImg.resample(w, h, native.Transform.lanczos);
-          nativeImg.free();
-          nativeImg = scaledImg;
-        }
-        final jpegBytes = await nativeImg.toJpeg(75);
-        file.blurhash = nativeImg.toBlurhash(3, 3);
-
-        thumbnail = MatrixImageFile(
-          bytes: jpegBytes,
-          name: 'thumbnail.jpg',
-          mimeType: 'image/jpeg',
-          width: nativeImg.width(),
-          height: nativeImg.height(),
-        );
-
-        nativeImg.free();
 
         if (thumbnail.size > file.size ~/ 2) {
           thumbnail = null;

lib/utils/run_in_background.dart

@@ -0,0 +1,12 @@
+import 'package:isolate/isolate.dart';
+import 'dart:async';
+
+Future<T> runInBackground<T, U>(
+    FutureOr<T> Function(U arg) function, U arg) async {
+  final isolate = await IsolateRunner.spawn();
+  try {
+    return await isolate.run(function, arg);
+  } finally {
+    await isolate.close();
+  }
+}

lib/views/chat.dart

@@ -227,31 +227,16 @@ class _ChatState extends State<_Chat> {
   }
 
   void sendImageAction(BuildContext context) async {
-    MatrixImageFile file;
-    if (PlatformInfos.isMobile) {
-      final result = await ImagePicker().getImage(
-          source: ImageSource.gallery,
-          imageQuality: 50,
-          maxWidth: 1600,
-          maxHeight: 1600);
-      if (result == null) return;
-      file = MatrixImageFile(
-        bytes: await result.readAsBytes(),
-        name: result.path,
-      );
-    } else {
     final result =
         await FilePickerCross.importFromStorage(type: FileTypeCross.image);
     if (result == null) return;
-      file = MatrixImageFile(
-        bytes: result.toUint8List(),
-        name: result.fileName,
-      );
-    }
     await showDialog(
       context: context,
       builder: (context) => SendFileDialog(
-        file: file,
+        file: MatrixImageFile(
+          bytes: result.toUint8List(),
+          name: result.fileName,
+        ),
         room: room,
       ),
     );