Merge branch 'soru/update-database' into 'master'

switch to moor_ffi w/ sqlcipher

See merge request ChristianPauly/fluffychat-flutter!104

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Version 0.16.0 - 2020-07-??
 ### Features
 - Implement web notifications
+### Changes
+- Various performance improvements
 - Added languages: Galician, Croatian, Japanese, Russian
+### Fixes:
+- Various fixes, including key verification fixes
 
 # Version 0.15.1 - 2020-06-26
 ### Fixes:

android/app/build.gradle

@@ -84,6 +84,7 @@ dependencies {
     testImplementation 'junit:junit:4.12'
     androidTestImplementation 'androidx.test:runner:1.1.1'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
+    implementation "net.zetetic:android-database-sqlcipher:4.4.0" // needed for moor_ffi w/ sqlcipher
 }
 
 apply plugin: "com.google.gms.google-services"

ios/Podfile

@@ -63,6 +63,7 @@ target 'Runner' do
 
   # Keep pod path relative so it can be checked into Podfile.lock.
   pod 'Flutter', :path => 'Flutter'
+  pod 'SQLCipher'
 
   # Plugin Pods
 

lib/utils/database/cipher_db.dart

@@ -0,0 +1,121 @@
+// file from https://gist.github.com/simolus3/5097bbd80ce59f9b957961fe851fd95a#file-cipher_db-dart
+
+import 'dart:async';
+import 'dart:ffi';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:moor/backends.dart';
+import 'package:moor/moor.dart';
+import 'package:moor_ffi/moor_ffi.dart';
+import 'package:moor_ffi/open_helper.dart';
+
+/// Tells `moor_ffi` to use `sqlcipher` instead of the regular `sqlite3`.
+///
+/// This needs to be called before using `moor`, for instance in the `main`
+/// method.
+void init() {
+  const sharedLibraryName = 'libsqlcipher.so';
+
+  open.overrideFor(OperatingSystem.android, () {
+    try {
+      return DynamicLibrary.open(sharedLibraryName);
+    } catch (_) {
+      // On some (especially old) Android devices, we somehow can't dlopen
+      // libraries shipped with the apk. We need to find the full path of the
+      // library (/data/data/<id>/lib/libsqlite3.so) and open that one.
+      // For details, see https://github.com/simolus3/moor/issues/420
+      final appIdAsBytes = File('/proc/self/cmdline').readAsBytesSync();
+
+      // app id ends with the first \0 character in here.
+      final endOfAppId = max(appIdAsBytes.indexOf(0), 0);
+      final appId = String.fromCharCodes(appIdAsBytes.sublist(0, endOfAppId));
+
+      return DynamicLibrary.open('/data/data/$appId/lib/$sharedLibraryName');
+    }
+  });
+
+  open.overrideFor(OperatingSystem.iOS, () => DynamicLibrary.executable());
+}
+
+class VmDatabaseEncrypted extends DelegatedDatabase {
+  /// Creates a database that will store its result in the [file], creating it
+  /// if it doesn't exist.
+  factory VmDatabaseEncrypted(
+    File file, {
+    String password = '',
+    bool logStatements = false,
+  }) {
+    final vmDatabase = VmDatabase(file, logStatements: logStatements);
+    return VmDatabaseEncrypted._(vmDatabase, password);
+  }
+
+  factory VmDatabaseEncrypted.memory({
+    String password = '',
+    bool logStatements = false,
+  }) {
+    final vmDatabase = VmDatabase.memory(logStatements: logStatements);
+    return VmDatabaseEncrypted._(vmDatabase, password);
+  }
+
+  VmDatabaseEncrypted._(
+    VmDatabase vmDatabase,
+    String password,
+  ) : super(
+          _VmEncryptedDelegate(vmDatabase.delegate, password),
+          logStatements: vmDatabase.logStatements,
+          isSequential: vmDatabase.isSequential,
+        );
+}
+
+class _VmEncryptedDelegate extends DatabaseDelegate {
+  final String password;
+  final DatabaseDelegate delegate;
+
+  _VmEncryptedDelegate(
+    this.delegate,
+    this.password,
+  );
+
+  @override
+  Future<void> open(QueryExecutorUser db) async {
+    await delegate.open(db);
+    final keyLiteral = const StringType().mapToSqlConstant(password);
+    await delegate.runCustom('PRAGMA KEY = $keyLiteral', const []);
+    return Future.value();
+  }
+
+  @override
+  FutureOr<bool> get isOpen => delegate.isOpen;
+
+  @override
+  Future<void> runBatched(BatchedStatements statements) {
+    return delegate.runBatched(statements);
+  }
+
+  @override
+  Future<void> runCustom(String statement, List args) {
+    return delegate.runCustom(statement, args);
+  }
+
+  @override
+  Future<int> runInsert(String statement, List args) {
+    return delegate.runInsert(statement, args);
+  }
+
+  @override
+  Future<QueryResult> runSelect(String statement, List args) {
+    return delegate.runSelect(statement, args);
+  }
+
+  @override
+  Future<int> runUpdate(String statement, List args) {
+    return delegate.runUpdate(statement, args);
+  }
+
+  @override
+  TransactionDelegate get transactionDelegate => delegate.transactionDelegate;
+
+  @override
+  DbVersionDelegate get versionDelegate => delegate.versionDelegate;
+}

lib/utils/database/mobile.dart

@@ -1,14 +1,63 @@
+import 'dart:io';
+import 'dart:isolate';
 import 'package:famedlysdk/famedlysdk.dart';
-import 'package:encrypted_moor/encrypted_moor.dart';
+import 'package:sqflite/sqflite.dart' show getDatabasesPath;
+import 'package:path/path.dart' as p;
 import 'package:flutter/material.dart';
+import 'package:moor/moor.dart';
+import 'package:moor/isolate.dart';
+import 'cipher_db.dart' as cipher;
 
-Database constructDb(
+bool _inited = false;
+
+// see https://moor.simonbinder.eu/docs/advanced-features/isolates/
+void _startBackground(_IsolateStartRequest request) {
+  // this is the entry point from the background isolate! Let's create
+  // the database from the path we received
+
+  if (!_inited) {
+    cipher.init();
+    _inited = true;
+  }
+  final executor = cipher.VmDatabaseEncrypted(File(request.targetPath),
+      password: request.password, logStatements: request.logStatements);
+  // we're using MoorIsolate.inCurrent here as this method already runs on a
+  // background isolate. If we used MoorIsolate.spawn, a third isolate would be
+  // started which is not what we want!
+  final moorIsolate = MoorIsolate.inCurrent(
+    () => DatabaseConnection.fromExecutor(executor),
+  );
+  // inform the starting isolate about this, so that it can call .connect()
+  request.sendMoorIsolate.send(moorIsolate);
+}
+
+// used to bundle the SendPort and the target path, since isolate entry point
+// functions can only take one parameter.
+class _IsolateStartRequest {
+  final SendPort sendMoorIsolate;
+  final String targetPath;
+  final String password;
+  final bool logStatements;
+
+  _IsolateStartRequest(
+      this.sendMoorIsolate, this.targetPath, this.password, this.logStatements);
+}
+
+Future<Database> constructDb(
     {bool logStatements = false,
     String filename = 'database.sqlite',
-    String password = ''}) {
+    String password = ''}) async {
   debugPrint('[Moor] using encrypted moor');
-  return Database(EncryptedExecutor(
+  final dbFolder = await getDatabasesPath();
-      path: filename, password: password, logStatements: logStatements));
+  final targetPath = p.join(dbFolder, filename);
+  final receivePort = ReceivePort();
+  await Isolate.spawn(
+    _startBackground,
+    _IsolateStartRequest(
+        receivePort.sendPort, targetPath, password, logStatements),
+  );
+  final isolate = (await receivePort.first as MoorIsolate);
+  return Database.connect(await isolate.connect());
 }
 
 Future<String> getLocalstorage(String key) async {

lib/utils/database/unsupported.dart

@@ -1,9 +1,9 @@
 import 'package:famedlysdk/famedlysdk.dart';
 
-Database constructDb(
+Future<Database> constructDb(
     {bool logStatements = false,
     String filename = 'database.sqlite',
-    String password = ''}) {
+    String password = ''}) async {
   throw 'Platform not supported';
 }
 

lib/utils/database/web.dart

@@ -3,10 +3,10 @@ import 'package:moor/moor_web.dart';
 import 'package:flutter/material.dart';
 import 'dart:html';
 
-Database constructDb(
+Future<Database> constructDb(
     {bool logStatements = false,
     String filename = 'database.sqlite',
-    String password = ''}) {
+    String password = ''}) async {
   debugPrint('[Moor] Using moor web');
   return Database(WebDatabase.withStorage(
       MoorWebStorage.indexedDbIfSupported(filename),

lib/utils/famedlysdk_store.dart

@@ -12,6 +12,11 @@ import 'package:olm/olm.dart' as olm; // needed for migration
 import 'package:random_string/random_string.dart';
 
 Future<Database> getDatabase(Client client) async {
+  while (_generateDatabaseLock) {
+    await Future.delayed(Duration(milliseconds: 50));
+  }
+  _generateDatabaseLock = true;
+  try {
     if (_db != null) return _db;
     final store = Store();
     var password = await store.getItem('database-password');
@@ -20,7 +25,7 @@ Future<Database> getDatabase(Client client) async {
       needMigration = true;
       password = randomString(255);
     }
-  _db = constructDb(
+    _db = await constructDb(
       logStatements: false,
       filename: 'moor.sqlite',
       password: password,
@@ -30,9 +35,13 @@ Future<Database> getDatabase(Client client) async {
       await store.setItem('database-password', password);
     }
     return _db;
+  } finally {
+    _generateDatabaseLock = false;
+  }
 }
 
 Database _db;
+bool _generateDatabaseLock = false;
 
 Future<void> migrate(String clientName, Database db, Store store) async {
   debugPrint('[Store] attempting old migration to moor...');

pubspec.lock

@@ -148,15 +148,6 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "4.0.2"
-  encrypted_moor:
-    dependency: "direct main"
-    description:
-      path: "extras/encryption"
-      ref: HEAD
-      resolved-ref: "6f930b011577e5bc8a5e5511691c8fcc43869a1c"
-      url: "https://github.com/simolus3/moor.git"
-    source: git
-    version: "1.0.0"
   fake_async:
     dependency: transitive
     description:
@@ -168,8 +159,8 @@ packages:
     dependency: "direct main"
     description:
       path: "."
-      ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3
+      ref: f4c8cfe992ceed937721af87abbd13fce7700ea5
-      resolved-ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3
+      resolved-ref: f4c8cfe992ceed937721af87abbd13fce7700ea5
       url: "https://gitlab.com/famedly/famedlysdk.git"
     source: git
     version: "0.0.1"
@@ -252,7 +243,14 @@ packages:
       name: flutter_matrix_html
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.1.1"
+    version: "0.1.2"
+  flutter_olm:
+    dependency: "direct main"
+    description:
+      name: flutter_olm
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.1"
   flutter_plugin_android_lifecycle:
     dependency: transitive
     description:
@@ -434,12 +432,10 @@ packages:
   matrix_file_e2ee:
     dependency: transitive
     description:
-      path: "."
+      name: matrix_file_e2ee
-      ref: "1.x.y"
+      url: "https://pub.dartlang.org"
-      resolved-ref: "32edeff765369a7a77a0822f4b19302ca24a017b"
+    source: hosted
-      url: "https://gitlab.com/famedly/libraries/matrix_file_e2ee.git"
+    version: "1.0.4"
-    source: git
-    version: "1.0.3"
   memoryfilepicker:
     dependency: "direct main"
     description:
@@ -475,6 +471,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.0.2"
+  moor_ffi:
+    dependency: "direct main"
+    description:
+      name: moor_ffi
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.5.0"
   multi_server_socket:
     dependency: transitive
     description:
@@ -506,12 +509,10 @@ packages:
   olm:
     dependency: transitive
     description:
-      path: "."
+      name: olm
-      ref: "1.x.y"
+      url: "https://pub.dartlang.org"
-      resolved-ref: "8e4fcccff7a2d4d0bd5142964db092bf45061905"
+    source: hosted
-      url: "https://gitlab.com/famedly/libraries/dart-olm.git"
+    version: "1.2.1"
-    source: git
-    version: "1.2.0"
   open_file:
     dependency: "direct main"
     description:
@@ -706,20 +707,6 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.2.0"
-  sqflite_common:
-    dependency: transitive
-    description:
-      name: sqflite_common
-      url: "https://pub.dartlang.org"
-    source: hosted
-    version: "1.0.1"
-  sqflite_sqlcipher:
-    dependency: transitive
-    description:
-      name: sqflite_sqlcipher
-      url: "https://pub.dartlang.org"
-    source: hosted
-    version: "1.0.0+6"
   stack_trace:
     dependency: transitive
     description:

pubspec.yaml

@@ -27,14 +27,13 @@ dependencies:
   famedlysdk:
     git:
       url: https://gitlab.com/famedly/famedlysdk.git
-      ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3
+      ref: f4c8cfe992ceed937721af87abbd13fce7700ea5
 
   localstorage: ^3.0.1+4
   bubble: ^1.1.9+1
   memoryfilepicker: ^0.1.1
   url_launcher: ^5.4.1
   url_launcher_web: ^0.1.0
-  sqflite: ^1.2.0
   flutter_advanced_networkimage: any
   firebase_messaging: ^6.0.13
   flutter_local_notifications: ^1.4.3
@@ -63,10 +62,8 @@ dependencies:
   flutter_localizations:
     sdk: flutter
 
-  encrypted_moor:
+  moor_ffi: ^0.5.0 # 0.6.0 and  up have a bug that it doesn't build with --profile, see https://github.com/simolus3/moor/issues/581
-    git:
+  sqflite: ^1.1.7 # Still used to obtain the database location
-      url: https://github.com/simolus3/moor.git
-      path: extras/encryption 
 
 dev_dependencies:
   flutter_test: