From 83559066a984f1f7c73e16155ffd98a0daee0611 Mon Sep 17 00:00:00 2001 From: Sorunome Date: Mon, 20 Jul 2020 15:33:52 +0000 Subject: [PATCH] switch to moor_ffi w/ sqlcipher --- CHANGELOG.md | 4 + android/app/build.gradle | 1 + ios/Podfile | 3 +- lib/utils/database/cipher_db.dart | 121 ++++++++++++++++++++++++++++ lib/utils/database/mobile.dart | 59 ++++++++++++-- lib/utils/database/unsupported.dart | 4 +- lib/utils/database/web.dart | 4 +- lib/utils/famedlysdk_store.dart | 41 ++++++---- pubspec.lock | 63 ++++++--------- pubspec.yaml | 9 +-- 10 files changed, 239 insertions(+), 70 deletions(-) create mode 100644 lib/utils/database/cipher_db.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 45379671..41c7a856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # Version 0.16.0 - 2020-07-?? ### Features - Implement web notifications +### Changes +- Various performance improvements - Added languages: Galician, Croatian, Japanese, Russian +### Fixes: +- Various fixes, including key verification fixes # Version 0.15.1 - 2020-06-26 ### Fixes: diff --git a/android/app/build.gradle b/android/app/build.gradle index 0689123c..36caa499 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -84,6 +84,7 @@ dependencies { testImplementation 'junit:junit:4.12' androidTestImplementation 'androidx.test:runner:1.1.1' androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1' + implementation "net.zetetic:android-database-sqlcipher:4.4.0" // needed for moor_ffi w/ sqlcipher } apply plugin: "com.google.gms.google-services" diff --git a/ios/Podfile b/ios/Podfile index 5b12b73d..02cc456f 100644 --- a/ios/Podfile +++ b/ios/Podfile @@ -63,6 +63,7 @@ target 'Runner' do # Keep pod path relative so it can be checked into Podfile.lock. pod 'Flutter', :path => 'Flutter' + pod 'SQLCipher' # Plugin Pods @@ -90,4 +91,4 @@ post_install do |installer| end # add pods for desired Firebase products -# https://firebase.google.com/docs/ios/setup#available-pods \ No newline at end of file +# https://firebase.google.com/docs/ios/setup#available-pods diff --git a/lib/utils/database/cipher_db.dart b/lib/utils/database/cipher_db.dart new file mode 100644 index 00000000..11b54f01 --- /dev/null +++ b/lib/utils/database/cipher_db.dart @@ -0,0 +1,121 @@ +// file from https://gist.github.com/simolus3/5097bbd80ce59f9b957961fe851fd95a#file-cipher_db-dart + +import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:math'; + +import 'package:moor/backends.dart'; +import 'package:moor/moor.dart'; +import 'package:moor_ffi/moor_ffi.dart'; +import 'package:moor_ffi/open_helper.dart'; + +/// Tells `moor_ffi` to use `sqlcipher` instead of the regular `sqlite3`. +/// +/// This needs to be called before using `moor`, for instance in the `main` +/// method. +void init() { + const sharedLibraryName = 'libsqlcipher.so'; + + open.overrideFor(OperatingSystem.android, () { + try { + return DynamicLibrary.open(sharedLibraryName); + } catch (_) { + // On some (especially old) Android devices, we somehow can't dlopen + // libraries shipped with the apk. We need to find the full path of the + // library (/data/data//lib/libsqlite3.so) and open that one. + // For details, see https://github.com/simolus3/moor/issues/420 + final appIdAsBytes = File('/proc/self/cmdline').readAsBytesSync(); + + // app id ends with the first \0 character in here. + final endOfAppId = max(appIdAsBytes.indexOf(0), 0); + final appId = String.fromCharCodes(appIdAsBytes.sublist(0, endOfAppId)); + + return DynamicLibrary.open('/data/data/$appId/lib/$sharedLibraryName'); + } + }); + + open.overrideFor(OperatingSystem.iOS, () => DynamicLibrary.executable()); +} + +class VmDatabaseEncrypted extends DelegatedDatabase { + /// Creates a database that will store its result in the [file], creating it + /// if it doesn't exist. + factory VmDatabaseEncrypted( + File file, { + String password = '', + bool logStatements = false, + }) { + final vmDatabase = VmDatabase(file, logStatements: logStatements); + return VmDatabaseEncrypted._(vmDatabase, password); + } + + factory VmDatabaseEncrypted.memory({ + String password = '', + bool logStatements = false, + }) { + final vmDatabase = VmDatabase.memory(logStatements: logStatements); + return VmDatabaseEncrypted._(vmDatabase, password); + } + + VmDatabaseEncrypted._( + VmDatabase vmDatabase, + String password, + ) : super( + _VmEncryptedDelegate(vmDatabase.delegate, password), + logStatements: vmDatabase.logStatements, + isSequential: vmDatabase.isSequential, + ); +} + +class _VmEncryptedDelegate extends DatabaseDelegate { + final String password; + final DatabaseDelegate delegate; + + _VmEncryptedDelegate( + this.delegate, + this.password, + ); + + @override + Future open(QueryExecutorUser db) async { + await delegate.open(db); + final keyLiteral = const StringType().mapToSqlConstant(password); + await delegate.runCustom('PRAGMA KEY = $keyLiteral', const []); + return Future.value(); + } + + @override + FutureOr get isOpen => delegate.isOpen; + + @override + Future runBatched(BatchedStatements statements) { + return delegate.runBatched(statements); + } + + @override + Future runCustom(String statement, List args) { + return delegate.runCustom(statement, args); + } + + @override + Future runInsert(String statement, List args) { + return delegate.runInsert(statement, args); + } + + @override + Future runSelect(String statement, List args) { + return delegate.runSelect(statement, args); + } + + @override + Future runUpdate(String statement, List args) { + return delegate.runUpdate(statement, args); + } + + @override + TransactionDelegate get transactionDelegate => delegate.transactionDelegate; + + @override + DbVersionDelegate get versionDelegate => delegate.versionDelegate; +} diff --git a/lib/utils/database/mobile.dart b/lib/utils/database/mobile.dart index 295c15ee..81086596 100644 --- a/lib/utils/database/mobile.dart +++ b/lib/utils/database/mobile.dart @@ -1,14 +1,63 @@ +import 'dart:io'; +import 'dart:isolate'; import 'package:famedlysdk/famedlysdk.dart'; -import 'package:encrypted_moor/encrypted_moor.dart'; +import 'package:sqflite/sqflite.dart' show getDatabasesPath; +import 'package:path/path.dart' as p; import 'package:flutter/material.dart'; +import 'package:moor/moor.dart'; +import 'package:moor/isolate.dart'; +import 'cipher_db.dart' as cipher; -Database constructDb( +bool _inited = false; + +// see https://moor.simonbinder.eu/docs/advanced-features/isolates/ +void _startBackground(_IsolateStartRequest request) { + // this is the entry point from the background isolate! Let's create + // the database from the path we received + + if (!_inited) { + cipher.init(); + _inited = true; + } + final executor = cipher.VmDatabaseEncrypted(File(request.targetPath), + password: request.password, logStatements: request.logStatements); + // we're using MoorIsolate.inCurrent here as this method already runs on a + // background isolate. If we used MoorIsolate.spawn, a third isolate would be + // started which is not what we want! + final moorIsolate = MoorIsolate.inCurrent( + () => DatabaseConnection.fromExecutor(executor), + ); + // inform the starting isolate about this, so that it can call .connect() + request.sendMoorIsolate.send(moorIsolate); +} + +// used to bundle the SendPort and the target path, since isolate entry point +// functions can only take one parameter. +class _IsolateStartRequest { + final SendPort sendMoorIsolate; + final String targetPath; + final String password; + final bool logStatements; + + _IsolateStartRequest( + this.sendMoorIsolate, this.targetPath, this.password, this.logStatements); +} + +Future constructDb( {bool logStatements = false, String filename = 'database.sqlite', - String password = ''}) { + String password = ''}) async { debugPrint('[Moor] using encrypted moor'); - return Database(EncryptedExecutor( - path: filename, password: password, logStatements: logStatements)); + final dbFolder = await getDatabasesPath(); + final targetPath = p.join(dbFolder, filename); + final receivePort = ReceivePort(); + await Isolate.spawn( + _startBackground, + _IsolateStartRequest( + receivePort.sendPort, targetPath, password, logStatements), + ); + final isolate = (await receivePort.first as MoorIsolate); + return Database.connect(await isolate.connect()); } Future getLocalstorage(String key) async { diff --git a/lib/utils/database/unsupported.dart b/lib/utils/database/unsupported.dart index cae34995..996b1f58 100644 --- a/lib/utils/database/unsupported.dart +++ b/lib/utils/database/unsupported.dart @@ -1,9 +1,9 @@ import 'package:famedlysdk/famedlysdk.dart'; -Database constructDb( +Future constructDb( {bool logStatements = false, String filename = 'database.sqlite', - String password = ''}) { + String password = ''}) async { throw 'Platform not supported'; } diff --git a/lib/utils/database/web.dart b/lib/utils/database/web.dart index e028e416..cd9e9ef1 100644 --- a/lib/utils/database/web.dart +++ b/lib/utils/database/web.dart @@ -3,10 +3,10 @@ import 'package:moor/moor_web.dart'; import 'package:flutter/material.dart'; import 'dart:html'; -Database constructDb( +Future constructDb( {bool logStatements = false, String filename = 'database.sqlite', - String password = ''}) { + String password = ''}) async { debugPrint('[Moor] Using moor web'); return Database(WebDatabase.withStorage( MoorWebStorage.indexedDbIfSupported(filename), diff --git a/lib/utils/famedlysdk_store.dart b/lib/utils/famedlysdk_store.dart index 43a9924e..0dc906c8 100644 --- a/lib/utils/famedlysdk_store.dart +++ b/lib/utils/famedlysdk_store.dart @@ -12,27 +12,36 @@ import 'package:olm/olm.dart' as olm; // needed for migration import 'package:random_string/random_string.dart'; Future getDatabase(Client client) async { - if (_db != null) return _db; - final store = Store(); - var password = await store.getItem('database-password'); - var needMigration = false; - if (password == null || password.isEmpty) { - needMigration = true; - password = randomString(255); + while (_generateDatabaseLock) { + await Future.delayed(Duration(milliseconds: 50)); } - _db = constructDb( - logStatements: false, - filename: 'moor.sqlite', - password: password, - ); - if (needMigration) { - await migrate(client.clientName, _db, store); - await store.setItem('database-password', password); + _generateDatabaseLock = true; + try { + if (_db != null) return _db; + final store = Store(); + var password = await store.getItem('database-password'); + var needMigration = false; + if (password == null || password.isEmpty) { + needMigration = true; + password = randomString(255); + } + _db = await constructDb( + logStatements: false, + filename: 'moor.sqlite', + password: password, + ); + if (needMigration) { + await migrate(client.clientName, _db, store); + await store.setItem('database-password', password); + } + return _db; + } finally { + _generateDatabaseLock = false; } - return _db; } Database _db; +bool _generateDatabaseLock = false; Future migrate(String clientName, Database db, Store store) async { debugPrint('[Store] attempting old migration to moor...'); diff --git a/pubspec.lock b/pubspec.lock index c7a6603b..93decd72 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -148,15 +148,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "4.0.2" - encrypted_moor: - dependency: "direct main" - description: - path: "extras/encryption" - ref: HEAD - resolved-ref: "6f930b011577e5bc8a5e5511691c8fcc43869a1c" - url: "https://github.com/simolus3/moor.git" - source: git - version: "1.0.0" fake_async: dependency: transitive description: @@ -168,8 +159,8 @@ packages: dependency: "direct main" description: path: "." - ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3 - resolved-ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3 + ref: f4c8cfe992ceed937721af87abbd13fce7700ea5 + resolved-ref: f4c8cfe992ceed937721af87abbd13fce7700ea5 url: "https://gitlab.com/famedly/famedlysdk.git" source: git version: "0.0.1" @@ -252,7 +243,14 @@ packages: name: flutter_matrix_html url: "https://pub.dartlang.org" source: hosted - version: "0.1.1" + version: "0.1.2" + flutter_olm: + dependency: "direct main" + description: + name: flutter_olm + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.1" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -434,12 +432,10 @@ packages: matrix_file_e2ee: dependency: transitive description: - path: "." - ref: "1.x.y" - resolved-ref: "32edeff765369a7a77a0822f4b19302ca24a017b" - url: "https://gitlab.com/famedly/libraries/matrix_file_e2ee.git" - source: git - version: "1.0.3" + name: matrix_file_e2ee + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.4" memoryfilepicker: dependency: "direct main" description: @@ -475,6 +471,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "3.0.2" + moor_ffi: + dependency: "direct main" + description: + name: moor_ffi + url: "https://pub.dartlang.org" + source: hosted + version: "0.5.0" multi_server_socket: dependency: transitive description: @@ -506,12 +509,10 @@ packages: olm: dependency: transitive description: - path: "." - ref: "1.x.y" - resolved-ref: "8e4fcccff7a2d4d0bd5142964db092bf45061905" - url: "https://gitlab.com/famedly/libraries/dart-olm.git" - source: git - version: "1.2.0" + name: olm + url: "https://pub.dartlang.org" + source: hosted + version: "1.2.1" open_file: dependency: "direct main" description: @@ -706,20 +707,6 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.2.0" - sqflite_common: - dependency: transitive - description: - name: sqflite_common - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.1" - sqflite_sqlcipher: - dependency: transitive - description: - name: sqflite_sqlcipher - url: "https://pub.dartlang.org" - source: hosted - version: "1.0.0+6" stack_trace: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 040cc5cf..7cbd99c4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -27,14 +27,13 @@ dependencies: famedlysdk: git: url: https://gitlab.com/famedly/famedlysdk.git - ref: ac720df3d26985faef0b6d6a86a8013f44c5c6e3 + ref: f4c8cfe992ceed937721af87abbd13fce7700ea5 localstorage: ^3.0.1+4 bubble: ^1.1.9+1 memoryfilepicker: ^0.1.1 url_launcher: ^5.4.1 url_launcher_web: ^0.1.0 - sqflite: ^1.2.0 flutter_advanced_networkimage: any firebase_messaging: ^6.0.13 flutter_local_notifications: ^1.4.3 @@ -63,10 +62,8 @@ dependencies: flutter_localizations: sdk: flutter - encrypted_moor: - git: - url: https://github.com/simolus3/moor.git - path: extras/encryption + moor_ffi: ^0.5.0 # 0.6.0 and up have a bug that it doesn't build with --profile, see https://github.com/simolus3/moor/issues/581 + sqflite: ^1.1.7 # Still used to obtain the database location dev_dependencies: flutter_test: