feat: Blurhashes and better thumbnails

CHANGELOG.md

@@ -6,6 +6,9 @@
 - Tapping links, pills, etc. now does stuff
 ### Fixes:
 - Various html rendering and url-ifying fixes
+- Added support for blurhashes
+- Use server-side generated thumbnails in cleartext rooms
+- Image viewer now eventually displays the original image, not only the thumbnail
 
 # Version 0.17.0 - 2020-08-31
 ### Features

lib/components/image_bubble.dart

@@ -3,6 +3,10 @@ import 'package:famedlysdk/famedlysdk.dart';
 import 'package:fluffychat/utils/app_route.dart';
 import 'package:fluffychat/views/image_view.dart';
 import 'package:flutter/material.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter_blurhash/flutter_blurhash.dart';
+import 'package:flutter_advanced_networkimage/provider.dart';
+import 'package:flutter_advanced_networkimage/transition.dart';
 
 class ImageBubble extends StatefulWidget {
   final Event event;
@@ -11,6 +15,7 @@ class ImageBubble extends StatefulWidget {
   final bool maxSize;
   final Color backgroundColor;
   final double radius;
+  final bool thumbnailOnly;
 
   const ImageBubble(
     this.event, {
@@ -19,6 +24,7 @@ class ImageBubble extends StatefulWidget {
     this.backgroundColor,
     this.fit = BoxFit.cover,
     this.radius = 10.0,
+    this.thumbnailOnly = true,
     Key key,
   }) : super(key: key);
 
@@ -27,16 +33,39 @@ class ImageBubble extends StatefulWidget {
 }
 
 class _ImageBubbleState extends State<ImageBubble> {
+  bool get isUnencrypted => widget.event.content['url'] is String;
+
   static final Map<String, MatrixFile> _matrixFileMap = {};
   MatrixFile get _file => _matrixFileMap[widget.event.eventId];
   set _file(MatrixFile file) {
-    _matrixFileMap[widget.event.eventId] = file;
+    if (file != null) {
+      _matrixFileMap[widget.event.eventId] = file;
+    }
+  }
+
+  static final Map<String, MatrixFile> _matrixThumbnailMap = {};
+  MatrixFile get _thumbnail => _matrixThumbnailMap[widget.event.eventId];
+  set _thumbnail(MatrixFile thumbnail) {
+    if (thumbnail != null) {
+      _matrixThumbnailMap[widget.event.eventId] = thumbnail;
+    }
   }
 
   dynamic _error;
 
+  bool _requestedFile = false;
   Future<MatrixFile> _getFile() async {
+    _requestedFile = true;
+    if (widget.thumbnailOnly) return null;
     if (_file != null) return _file;
+    return widget.event.downloadAndDecryptAttachment();
+  }
+
+  bool _requestedThumbnail = false;
+  Future<MatrixFile> _getThumbnail() async {
+    _requestedThumbnail = true;
+    if (isUnencrypted) return null;
+    if (_thumbnail != null) return _thumbnail;
     return widget.event
         .downloadAndDecryptAttachment(getThumbnail: widget.event.hasThumbnail);
   }
@@ -60,32 +89,71 @@ class _ImageBubbleState extends State<ImageBubble> {
                 ),
               );
             }
-            if (_file != null) {
+            if (_thumbnail == null && !_requestedThumbnail && !isUnencrypted) {
-              return InkWell(
+              _getThumbnail().then((MatrixFile thumbnail) {
-                onTap: () {
+                setState(() => _thumbnail = thumbnail);
-                  if (!widget.tapToView) return;
+              }, onError: (error, stacktrace) {
-                  Navigator.of(context).push(
+                setState(() => _error = error);
-                    AppRoute(
+              });
-                      ImageView(widget.event),
-                    ),
-                  );
-                },
-                child: Hero(
-                  tag: widget.event.eventId,
-                  child: Image.memory(
-                    _file.bytes,
-                    fit: widget.fit,
-                  ),
-                ),
-              );
             }
-            _getFile().then((MatrixFile file) {
+            if (_file == null && !widget.thumbnailOnly && !_requestedFile) {
-              setState(() => _file = file);
+              _getFile().then((MatrixFile file) {
-            }, onError: (error, stacktrace) {
+                setState(() => _file = file);
-              setState(() => _error = error);
+              }, onError: (error, stacktrace) {
-            });
+                setState(() => _error = error);
-            return Center(
+              });
-              child: CircularProgressIndicator(),
+            }
+            final display = _file ?? _thumbnail;
+
+            final generatePlaceholderWidget = () => Stack(
+                  children: <Widget>[
+                    if (widget.event.content['info'] is Map &&
+                        widget.event.content['info']['xyz.amorgan.blurhash']
+                            is String)
+                      BlurHash(
+                          hash: widget.event.content['info']
+                              ['xyz.amorgan.blurhash']),
+                    Center(
+                      child: CircularProgressIndicator(),
+                    ),
+                  ],
+                );
+
+            Widget renderWidget;
+            if (display != null) {
+              renderWidget = Image.memory(
+                display.bytes,
+                fit: widget.fit,
+              );
+            } else if (isUnencrypted) {
+              renderWidget = TransitionToImage(
+                image: AdvancedNetworkImage(
+                  Uri.parse(widget.event.content['url']).getThumbnail(
+                      widget.event.room.client,
+                      width: 800,
+                      height: 800,
+                      method: ThumbnailMethod.scale),
+                  useDiskCache: !kIsWeb,
+                ),
+                loadingWidget: generatePlaceholderWidget(),
+                fit: widget.fit,
+              );
+            } else {
+              renderWidget = generatePlaceholderWidget();
+            }
+            return InkWell(
+              onTap: () {
+                if (!widget.tapToView) return;
+                Navigator.of(context).push(
+                  AppRoute(
+                    ImageView(widget.event),
+                  ),
+                );
+              },
+              child: Hero(
+                tag: widget.event.eventId,
+                child: renderWidget,
+              ),
             );
           },
         ),

lib/utils/event_extension.dart

@@ -3,9 +3,20 @@ import 'package:fluffychat/components/dialogs/simple_dialogs.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'matrix_file_extension.dart';
+import 'app_route.dart';
+import '../views/image_view.dart';
 
 extension LocalizedBody on Event {
-  void openFile(BuildContext context) async {
+  void openFile(BuildContext context, {bool downloadOnly = false}) async {
+    if (!downloadOnly &&
+        [MessageTypes.Image, MessageTypes.Sticker].contains(messageType)) {
+      await Navigator.of(context).push(
+        AppRoute(
+          ImageView(this),
+        ),
+      );
+      return;
+    }
     final MatrixFile matrixFile =
         await SimpleDialogs(context).tryRequestWithLoadingDialog(
       downloadAndDecryptAttachment(),
@@ -32,7 +43,12 @@ extension LocalizedBody on Event {
       [MessageTypes.Image, MessageTypes.Sticker].contains(messageType) &&
       (kIsWeb ||
           (content['info'] is Map &&
-              content['info']['size'] < room.client.database.maxFileSize));
+              content['info']['size'] < room.client.database.maxFileSize) ||
+          (hasThumbnail &&
+              content['info']['thumbnail_info'] is Map &&
+              content['info']['thumbnail_info']['size'] <
+                  room.client.database.maxFileSize) ||
+          (content['url'] is String));
 
   String get sizeString {
     if (content['info'] is Map<String, dynamic> &&

lib/utils/room_send_file_extension.dart

@@ -0,0 +1,98 @@
+/*
+ *   Famedly App
+ *   Copyright (C) 2020 Famedly GmbH
+ *
+ *   This program is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU Affero General Public License as
+ *   published by the Free Software Foundation, either version 3 of the
+ *   License, or (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU Affero General Public License for more details.
+ *
+ *   You should have received a copy of the GNU Affero General Public License
+ *   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import 'dart:typed_data';
+import 'dart:ui';
+
+import 'package:famedlysdk/famedlysdk.dart';
+import 'package:native_imaging/native_imaging.dart' as native;
+
+extension RoomSendFileExtension on Room {
+  Future<String> sendFileEventWithThumbnail(
+    MatrixFile file, {
+    String txid,
+    Event inReplyTo,
+    String editEventId,
+    bool waitUntilSent,
+  }) async {
+    MatrixFile thumbnail;
+    try {
+      if (file is MatrixImageFile) {
+        await native.init();
+        var nativeImg = native.Image();
+        try {
+          await nativeImg.loadEncoded(file.bytes);
+          file.width = nativeImg.width();
+          file.height = nativeImg.height();
+        } on UnsupportedError {
+          final dartCodec = await instantiateImageCodec(file.bytes);
+          final dartFrame = await dartCodec.getNextFrame();
+          file.width = dartFrame.image.width;
+          file.height = dartFrame.image.height;
+          final rgbaData = await dartFrame.image.toByteData();
+          final rgba = Uint8List.view(
+              rgbaData.buffer, rgbaData.offsetInBytes, rgbaData.lengthInBytes);
+          dartFrame.image.dispose();
+          dartCodec.dispose();
+          nativeImg.loadRGBA(file.width, file.height, rgba);
+        }
+
+        const max = 800;
+        if (file.width > max || file.height > max) {
+          var w = max, h = max;
+          if (file.width > file.height) {
+            h = max * file.height ~/ file.width;
+          } else {
+            w = max * file.width ~/ file.height;
+          }
+
+          final scaledImg = nativeImg.resample(w, h, native.Transform.lanczos);
+          nativeImg.free();
+          nativeImg = scaledImg;
+        }
+        final jpegBytes = await nativeImg.toJpeg(75);
+        file.blurhash = nativeImg.toBlurhash(3, 3);
+
+        thumbnail = MatrixImageFile(
+          bytes: jpegBytes,
+          name: 'thumbnail.jpg',
+          mimeType: 'image/jpeg',
+          width: nativeImg.width(),
+          height: nativeImg.height(),
+        );
+
+        nativeImg.free();
+
+        if (thumbnail.size > file.size ~/ 2) {
+          thumbnail = null;
+        }
+      }
+    } catch (e) {
+      // send no thumbnail
+    }
+
+    return sendFileEvent(
+      file,
+      txid: txid,
+      inReplyTo: inReplyTo,
+      editEventId: editEventId,
+      waitUntilSent: waitUntilSent ?? false,
+      thumbnail: thumbnail,
+    );
+  }
+}

lib/views/chat.dart

@@ -27,6 +27,7 @@ import 'package:image_picker/image_picker.dart';
 import 'chat_details.dart';
 import 'chat_list.dart';
 import '../components/input_bar.dart';
+import '../utils/room_send_file_extension.dart';
 
 class ChatView extends StatelessWidget {
   final String id;
@@ -191,7 +192,7 @@ class _ChatState extends State<_Chat> {
     var file = await MemoryFilePicker.getFile();
     if (file == null) return;
     await SimpleDialogs(context).tryRequestWithLoadingDialog(
-      room.sendFileEvent(
+      room.sendFileEventWithThumbnail(
         MatrixFile(bytes: file.bytes, name: file.path),
       ),
     );
@@ -205,7 +206,7 @@ class _ChatState extends State<_Chat> {
         maxHeight: 1600);
     if (file == null) return;
     await SimpleDialogs(context).tryRequestWithLoadingDialog(
-      room.sendFileEvent(
+      room.sendFileEventWithThumbnail(
         MatrixImageFile(bytes: await file.bytes, name: file.path),
       ),
     );
@@ -219,7 +220,7 @@ class _ChatState extends State<_Chat> {
         maxHeight: 1600);
     if (file == null) return;
     await SimpleDialogs(context).tryRequestWithLoadingDialog(
-      room.sendFileEvent(
+      room.sendFileEventWithThumbnail(
         MatrixImageFile(bytes: file.bytes, name: file.path),
       ),
     );

lib/views/image_view.dart

@@ -36,7 +36,7 @@ class ImageView extends StatelessWidget {
           ),
           IconButton(
             icon: Icon(Icons.file_download),
-            onPressed: () => event.openFile(context),
+            onPressed: () => event.openFile(context, downloadOnly: true),
             color: Colors.white,
           ),
         ],
@@ -51,6 +51,7 @@ class ImageView extends StatelessWidget {
           backgroundColor: Colors.black,
           maxSize: false,
           radius: 0.0,
+          thumbnailOnly: false,
         ),
       ),
     );

pubspec.lock

@@ -241,6 +241,13 @@ packages:
       url: "https://github.com/mchome/flutter_advanced_networkimage"
     source: git
     version: "0.8.0"
+  flutter_blurhash:
+    dependency: "direct main"
+    description:
+      name: flutter_blurhash
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.5.0"
   flutter_keyboard_visibility:
     dependency: transitive
     description:
@@ -508,6 +515,15 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.3.1"
+  native_imaging:
+    dependency: "direct main"
+    description:
+      path: "."
+      ref: master
+      resolved-ref: bd24832f96537447174aa34ba78eaed7ff05bb8e
+      url: "https://gitlab.com/famedly/libraries/native_imaging.git"
+    source: git
+    version: "0.0.1"
   node_interop:
     dependency: transitive
     description:

pubspec.yaml

@@ -65,6 +65,12 @@ dependencies:
   flutter_localizations:
     sdk: flutter
   sqflite: ^1.1.7 # Still used to obtain the database location
+  native_imaging:
+    git:
+      url: https://gitlab.com/famedly/libraries/native_imaging.git
+      ref: master
+  flutter_blurhash: ^0.5.0
+
 
 dev_dependencies:
   flutter_test: