mirror/monocles_chat_clean
1
0
Fork 1
Code Activity

Security: Introduce backup file format v2 2/2

Browse source
This commit is contained in:
Daniel Gultsch 2023-08-17 09:40:17 +02:00 • committed by Arne
parent 94269e5a41
commit a8853bf67f
2 changed files with 232 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

229
src/main/java/eu/siacs/conversations/services/ExportBackupService.java
View file

@ -25,6 +25,10 @@ import android.os.PowerManager;
import android.preference.PreferenceManager;
import android.util.Log;
import com.google.common.base.CharMatcher;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import androidx.core.app.NotificationCompat;
@ -117,124 +121,88 @@ public class ExportBackupService extends Service {
return Arrays.asList(openIntent, amazeIntent, systemFallBack);
}
private static void accountExport(final SQLiteDatabase db, final String uuid, final PrintWriter writer) {
final StringBuilder builder = new StringBuilder();
final Cursor accountCursor = db.query(Account.TABLENAME, null, Account.UUID + "=?", new String[]{uuid}, null, null, null);
private static void accountExport(
final SQLiteDatabase db, final String uuid, final JsonWriter writer)
throws IOException {
final Cursor accountCursor =
db.query(
Account.TABLENAME,
null,
Account.UUID + "=?",
new String[] {uuid},
null,
null,
null);
while (accountCursor != null && accountCursor.moveToNext()) {
builder.append("INSERT INTO ").append(Account.TABLENAME).append("(");
writer.beginObject();
writer.name("table");
writer.value(Account.TABLENAME);
writer.name("values");
writer.beginObject();
for (int i = 0; i < accountCursor.getColumnCount(); ++i) {
if (i != 0) {
builder.append(',');
}
builder.append(accountCursor.getColumnName(i));
}
builder.append(") VALUES(");
for (int i = 0; i < accountCursor.getColumnCount(); ++i) {
if (i != 0) {
builder.append(',');
}
final String name = accountCursor.getColumnName(i);
writer.name(name);
final String value = accountCursor.getString(i);
if (value == null || Account.ROSTERVERSION.equals(accountCursor.getColumnName(i))) {
builder.append("NULL");
} else if (Account.OPTIONS.equals(accountCursor.getColumnName(i)) && value.matches("\\d+")) {
writer.nullValue();
} else if (Account.OPTIONS.equals(accountCursor.getColumnName(i))
&& value.matches("\\d+")) {
int intValue = Integer.parseInt(value);
intValue |= 1 << Account.OPTION_DISABLED;
builder.append(intValue);
writer.value(intValue);
} else {
appendEscapedSQLString(builder, value);
writer.value(value);
}
}
builder.append(")");
builder.append(';');
builder.append('\n');
writer.endObject();
writer.endObject();
}
if (accountCursor != null) {
accountCursor.close();
}
writer.append(builder.toString());
}
private static void appendEscapedSQLString(final StringBuilder sb, final String sqlString) {
DatabaseUtils.appendEscapedSQLString(sb, CharMatcher.is('\u0000').removeFrom(sqlString));
}
private static void simpleExport(SQLiteDatabase db, String table, String column, String uuid, PrintWriter writer) {
final Cursor cursor = db.query(table, null, column + "=?", new String[]{uuid}, null, null, null);
private static void simpleExport(
final SQLiteDatabase db,
final String table,
final String column,
final String uuid,
final JsonWriter writer)
throws IOException {
final Cursor cursor =
db.query(table, null, column + "=?", new String[] {uuid}, null, null, null);
while (cursor != null && cursor.moveToNext()) {
writer.write(cursorToString(table, cursor, PAGE_SIZE));
writer.beginObject();
writer.name("table");
writer.value(table);
writer.name("values");
writer.beginObject();
for (int i = 0; i < cursor.getColumnCount(); ++i) {
final String name = cursor.getColumnName(i);
writer.name(name);
final String value = cursor.getString(i);
writer.value(value);
}
writer.endObject();
writer.endObject();
}
if (cursor != null) {
cursor.close();
}
}
public static byte[] getKey(final String password, final byte[] salt) throws InvalidKeySpecException {
public static byte[] getKey(final String password, final byte[] salt)
throws InvalidKeySpecException {
final SecretKeyFactory factory;
try {
factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(e);
}
return factory.generateSecret(new PBEKeySpec(password.toCharArray(), salt, 1024, 128)).getEncoded();
return factory.generateSecret(new PBEKeySpec(password.toCharArray(), salt, 1024, 128))
.getEncoded();
}
private static String cursorToString(final String table, final Cursor cursor, final int max) {
return cursorToString(table, cursor, max, false);
}
private static String cursorToString(final String table, final Cursor cursor, int max, boolean ignore) {
final boolean identities = SQLiteAxolotlStore.IDENTITIES_TABLENAME.equals(table);
StringBuilder builder = new StringBuilder();
builder.append("INSERT ");
if (ignore) {
builder.append("OR IGNORE ");
}
builder.append("INTO ").append(table).append("(");
int skipColumn = -1;
for (int i = 0; i < cursor.getColumnCount(); ++i) {
final String name = cursor.getColumnName(i);
if (identities && SQLiteAxolotlStore.TRUSTED.equals(name)) {
skipColumn = i;
continue;
}
if (i != 0) {
builder.append(',');
}
builder.append(name);
}
builder.append(") VALUES");
for (int i = 0; i < max; ++i) {
if (i != 0) {
builder.append(',');
}
appendValues(cursor, builder, skipColumn);
if (i < max - 1 && !cursor.moveToNext()) {
break;
}
}
builder.append(';');
builder.append('\n');
return builder.toString();
}
private static void appendValues(final Cursor cursor, final StringBuilder builder, final int skipColumn) {
builder.append("(");
for (int i = 0; i < cursor.getColumnCount(); ++i) {
if (i == skipColumn) {
continue;
}
if (i != 0) {
builder.append(',');
}
final String value = cursor.getString(i);
if (value == null) {
builder.append("NULL");
} else if (value.matches("[0-9]+")) {
builder.append(value);
} else {
appendEscapedSQLString(builder, value);
}
}
builder.append(")");
}
@Override
public void onCreate() {
@ -303,7 +271,8 @@ public class ExportBackupService extends Service {
return START_NOT_STICKY;
}
private void messageExport(SQLiteDatabase db, String uuid, PrintWriter writer, Progress progress) {
private void messageExport(final SQLiteDatabase db, final String uuid, final JsonWriter writer, final Progress progress)
throws IOException {
Cursor cursor;
if (runsTwentySix()) {
try {
@ -329,17 +298,25 @@ public class ExportBackupService extends Service {
int p = 0;
try {
while (cursor != null && cursor.moveToNext()) {
writer.write(cursorToString(Message.TABLENAME, cursor, PAGE_SIZE, false));
if (i + PAGE_SIZE > size) {
i = size;
} else {
i += PAGE_SIZE;
writer.beginObject();
writer.name("table");
writer.value(Message.TABLENAME);
writer.name("values");
writer.beginObject();
for (int j = 0; j < cursor.getColumnCount(); ++j) {
final String name = cursor.getColumnName(j);
writer.name(name);
final String value = cursor.getString(j);
writer.value(value);
}
writer.endObject();
writer.endObject();
final int percentage = i * 100 / size;
if (p < percentage) {
p = percentage;
notificationManager.notify(EXPORT_BACKUP_NOTIFICATION_ID, progress.build(p));
}
i++;
}
} catch (Exception e) {
e.printStackTrace();
@ -350,24 +327,32 @@ public class ExportBackupService extends Service {
}
}
private void messageExportmonocles(SQLiteDatabase db, String uuid, PrintWriter writer, Progress progress) {
private void messageExportmonocles(final SQLiteDatabase db, final String uuid, final JsonWriter writer, final Progress progress) throws IOException {
Cursor cursor = db.rawQuery("select mmessages.* from messages join monocles.messages mmessages using (uuid) join conversations on conversations.uuid=messages.conversationUuid where conversations.accountUuid=?", new String[]{uuid});
int size = cursor != null ? cursor.getCount() : 0;
Log.d(Config.LOGTAG, "exporting " + size + " monocles messages for account " + uuid);
int i = 0;
int p = 0;
while (cursor != null && cursor.moveToNext()) {
writer.write(cursorToString("monocles." + Message.TABLENAME, cursor, PAGE_SIZE, false));
if (i + PAGE_SIZE > size) {
i = size;
} else {
i += PAGE_SIZE;
writer.beginObject();
writer.name("table");
writer.value("monocles." + Message.TABLENAME);
writer.name("values");
writer.beginObject();
for (int j = 0; j < cursor.getColumnCount(); ++j) {
final String name = cursor.getColumnName(j);
writer.name(name);
final String value = cursor.getString(j);
writer.value(value);
}
writer.endObject();
writer.endObject();
final int percentage = i * 100 / size;
if (p < percentage) {
p = percentage;
notificationManager.notify(EXPORT_BACKUP_NOTIFICATION_ID, progress.build(p));
}
i++;
}
if (cursor != null) {
cursor.close();
@ -377,17 +362,25 @@ public class ExportBackupService extends Service {
size = cursor != null ? cursor.getCount() : 0;
Log.d(Config.LOGTAG, "exporting " + size + " WebXDC updates for account " + uuid);
while (cursor != null && cursor.moveToNext()) {
writer.write(cursorToString("monocles.webxdc_updates", cursor, PAGE_SIZE, false));
if (i + PAGE_SIZE > size) {
i = size;
} else {
i += PAGE_SIZE;
writer.beginObject();
writer.name("table");
writer.value("monocles.webxdc_updates");
writer.name("values");
writer.beginObject();
for (int j = 0; j < cursor.getColumnCount(); ++j) {
final String name = cursor.getColumnName(j);
writer.name(name);
final String value = cursor.getString(j);
writer.value(value);
}
writer.endObject();
writer.endObject();
final int percentage = i * 100 / size;
if (p < percentage) {
p = percentage;
notificationManager.notify(EXPORT_BACKUP_NOTIFICATION_ID, progress.build(p));
}
i++;
}
if (cursor != null) {
cursor.close();
@ -439,19 +432,23 @@ public class ExportBackupService extends Service {
IvParameterSpec ivSpec = new IvParameterSpec(IV);
cipher.init(Cipher.ENCRYPT_MODE, keySpec, ivSpec);
CipherOutputStream cipherOutputStream = new CipherOutputStream(fileOutputStream, cipher);
GZIPOutputStream gzipOutputStream = new GZIPOutputStream(cipherOutputStream);
PrintWriter writer = new PrintWriter(gzipOutputStream);
SQLiteDatabase db = this.mDatabaseBackend.getReadableDatabase();
final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(cipherOutputStream);
final JsonWriter jsonWriter =
new JsonWriter(
new OutputStreamWriter(gzipOutputStream, StandardCharsets.UTF_8));
jsonWriter.beginArray();
final SQLiteDatabase db = this.mDatabaseBackend.getReadableDatabase();
final String uuid = account.getUuid();
accountExport(db, uuid, writer);
simpleExport(db, Conversation.TABLENAME, Conversation.ACCOUNT, uuid, writer);
messageExport(db, uuid, writer, progress);
if (withmonoclesDb) messageExportmonocles(db, uuid, writer, progress);
accountExport(db, uuid, jsonWriter);
simpleExport(db, Conversation.TABLENAME, Conversation.ACCOUNT, uuid, jsonWriter);
messageExport(db, uuid, jsonWriter, progress);
if (withmonoclesDb) messageExportmonocles(db, uuid, jsonWriter, progress);
for (String table : Arrays.asList(SQLiteAxolotlStore.PREKEY_TABLENAME, SQLiteAxolotlStore.SIGNED_PREKEY_TABLENAME, SQLiteAxolotlStore.SESSION_TABLENAME, SQLiteAxolotlStore.IDENTITIES_TABLENAME)) {
simpleExport(db, table, SQLiteAxolotlStore.ACCOUNT, uuid, writer);
simpleExport(db, table, SQLiteAxolotlStore.ACCOUNT, uuid, jsonWriter);
}
writer.flush();
writer.close();
jsonWriter.endArray();
jsonWriter.flush();
jsonWriter.close();
mediaScannerScanFile(file);
Log.d(Config.LOGTAG, "written backup to " + file.getAbsoluteFile());
} catch (Exception e) {

167
src/main/java/eu/siacs/conversations/services/ImportBackupService.java
View file

@ -8,6 +8,7 @@ import android.app.Notification;
import android.app.NotificationManager;
import android.app.PendingIntent;
import android.app.Service;
import android.content.ContentValues;
import android.content.Context;
import android.content.Intent;
import android.database.Cursor;
@ -24,6 +25,8 @@ import androidx.core.app.NotificationManagerCompat;
import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
import com.google.common.io.CountingInputStream;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import org.bouncycastle.crypto.engines.AESEngine;
import org.bouncycastle.crypto.io.CipherInputStream;
@ -42,6 +45,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -55,6 +59,10 @@ import javax.crypto.BadPaddingException;
import eu.siacs.conversations.Config;
import eu.siacs.conversations.R;
import eu.siacs.conversations.crypto.axolotl.SQLiteAxolotlStore;
import eu.siacs.conversations.entities.Account;
import eu.siacs.conversations.entities.Conversation;
import eu.siacs.conversations.entities.Message;
import eu.siacs.conversations.persistance.DatabaseBackend;
import eu.siacs.conversations.ui.ManageAccountActivity;
import eu.siacs.conversations.utils.BackupFileHeader;
@ -66,25 +74,28 @@ public class ImportBackupService extends Service {
private static final AtomicBoolean running = new AtomicBoolean(false);
private final ImportBackupServiceBinder binder = new ImportBackupServiceBinder();
private final SerialSingleThreadExecutor executor = new SerialSingleThreadExecutor(getClass().getSimpleName());
private final Set<OnBackupProcessed> mOnBackupProcessedListeners = Collections.newSetFromMap(new WeakHashMap<>());
private final SerialSingleThreadExecutor executor =
new SerialSingleThreadExecutor(getClass().getSimpleName());
private final Set<OnBackupProcessed> mOnBackupProcessedListeners =
Collections.newSetFromMap(new WeakHashMap<>());
private DatabaseBackend mDatabaseBackend;
private NotificationManager notificationManager;
private static int count(String input, char c) {
int count = 0;
for (char aChar : input.toCharArray()) {
if (aChar == c) {
++count;
}
}
return count;
}
private static final Collection<String> TABLE_ALLOW_LIST =
Arrays.asList(
Account.TABLENAME,
Conversation.TABLENAME,
Message.TABLENAME,
SQLiteAxolotlStore.PREKEY_TABLENAME,
SQLiteAxolotlStore.SIGNED_PREKEY_TABLENAME,
SQLiteAxolotlStore.SESSION_TABLENAME,
SQLiteAxolotlStore.IDENTITIES_TABLENAME);
@Override
public void onCreate() {
mDatabaseBackend = DatabaseBackend.getInstance(getBaseContext());
notificationManager = (android.app.NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
notificationManager =
(android.app.NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
}
@Override
@ -106,16 +117,17 @@ public class ImportBackupService extends Service {
return START_NOT_STICKY;
}
if (running.compareAndSet(false, true)) {
executor.execute(() -> {
startForegroundService();
final boolean success = importBackup(uri, password);
stopForeground(true);
running.set(false);
if (success) {
notifySuccess();
}
stopSelf();
});
executor.execute(
() -> {
startForegroundService();
final boolean success = importBackup(uri, password);
stopForeground(true);
running.set(false);
if (success) {
notifySuccess();
}
stopSelf();
});
} else {
Log.d(Config.LOGTAG, "backup already running");
}
@ -216,7 +228,9 @@ public class ImportBackupService extends Service {
fileSize = 0;
} else {
returnCursor.moveToFirst();
fileSize = returnCursor.getLong(returnCursor.getColumnIndex(OpenableColumns.SIZE));
fileSize =
returnCursor.getLong(
returnCursor.getColumnIndexOrThrow(OpenableColumns.SIZE));
returnCursor.close();
}
inputStream = getContentResolver().openInputStream(uri);
@ -244,40 +258,46 @@ public class ImportBackupService extends Service {
final byte[] key = ExportBackupService.getKey(password, backupFileHeader.getSalt());
final AEADBlockCipher cipher = new GCMBlockCipher(new AESEngine());
cipher.init(false, new AEADParameters(new KeyParameter(key), 128, backupFileHeader.getIv()));
final CipherInputStream cipherInputStream = new CipherInputStream(countingInputStream, cipher);
cipher.init(
false,
new AEADParameters(new KeyParameter(key), 128, backupFileHeader.getIv()));
final CipherInputStream cipherInputStream =
new CipherInputStream(countingInputStream, cipher);
final GZIPInputStream gzipInputStream = new GZIPInputStream(cipherInputStream);
final BufferedReader reader = new BufferedReader(new InputStreamReader(gzipInputStream, Charsets.UTF_8));
final BufferedReader reader =
new BufferedReader(new InputStreamReader(gzipInputStream, Charsets.UTF_8));
final JsonReader jsonReader = new JsonReader(reader);
if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) {
jsonReader.beginArray();
} else {
throw new IllegalStateException("Backup file did not begin with array");
}
db.beginTransaction();
String line;
StringBuilder multiLineQuery = null;
while ((line = reader.readLine()) != null) {
int count = count(line, '\'');
if (multiLineQuery != null) {
multiLineQuery.append('\n');
multiLineQuery.append(line);
if (count % 2 == 1) {
db.execSQL(multiLineQuery.toString());
multiLineQuery = null;
updateImportBackupNotification(fileSize, countingInputStream.getCount());
}
} else {
if (count % 2 == 0) {
db.execSQL(line);
updateImportBackupNotification(fileSize, countingInputStream.getCount());
} else {
multiLineQuery = new StringBuilder(line);
}
while (jsonReader.hasNext()) {
if (jsonReader.peek() == JsonToken.BEGIN_OBJECT) {
importRow(db, jsonReader, backupFileHeader.getJid(), password);
} else if (jsonReader.peek() == JsonToken.END_ARRAY) {
jsonReader.endArray();
continue;
}
updateImportBackupNotification(fileSize, countingInputStream.getCount());
}
db.setTransactionSuccessful();
db.endTransaction();
final Jid jid = backupFileHeader.getJid();
final Cursor countCursor = db.rawQuery("select count(messages.uuid) from messages join conversations on conversations.uuid=messages.conversationUuid join accounts on conversations.accountUuid=accounts.uuid where accounts.username=? and accounts.server=?", new String[]{jid.getEscapedLocal(), jid.getDomain().toEscapedString()});
final Cursor countCursor =
db.rawQuery(
"select count(messages.uuid) from messages join conversations on conversations.uuid=messages.conversationUuid join accounts on conversations.accountUuid=accounts.uuid where accounts.username=? and accounts.server=?",
new String[] {
jid.getEscapedLocal(), jid.getDomain().toEscapedString()
});
countCursor.moveToFirst();
final int count = countCursor.getInt(0);
Log.d(Config.LOGTAG, String.format("restored %d messages in %s", count, stopwatch.stop().toString()));
Log.d(
Config.LOGTAG,
String.format(
"restored %d messages in %s", count, stopwatch.stop().toString()));
countCursor.close();
stopBackgroundService();
synchronized (mOnBackupProcessedListeners) {
@ -288,7 +308,8 @@ public class ImportBackupService extends Service {
return true;
} catch (final Exception e) {
final Throwable throwable = e.getCause();
final boolean reasonWasCrypto = throwable instanceof BadPaddingException || e instanceof ZipException;
final boolean reasonWasCrypto =
throwable instanceof BadPaddingException || e instanceof ZipException;
synchronized (mOnBackupProcessedListeners) {
for (OnBackupProcessed l : mOnBackupProcessedListeners) {
if (reasonWasCrypto) {
@ -303,6 +324,56 @@ public class ImportBackupService extends Service {
}
}
private void importRow(
final SQLiteDatabase db,
final JsonReader jsonReader,
final Jid account,
final String passphrase)
throws IOException {
jsonReader.beginObject();
final String firstParameter = jsonReader.nextName();
if (!firstParameter.equals("table")) {
throw new IllegalStateException("Expected key 'table'");
}
final String table = jsonReader.nextString();
if (!TABLE_ALLOW_LIST.contains(table)) {
throw new IOException(String.format("%s is not recognized for import", table));
}
final ContentValues contentValues = new ContentValues();
final String secondParameter = jsonReader.nextName();
if (!secondParameter.equals("values")) {
throw new IllegalStateException("Expected key 'values'");
}
jsonReader.beginObject();
while (jsonReader.peek() != JsonToken.END_OBJECT) {
final String name = jsonReader.nextName();
if (jsonReader.peek() == JsonToken.NULL) {
jsonReader.nextNull();
contentValues.putNull(name);
} else if (jsonReader.peek() == JsonToken.NUMBER) {
contentValues.put(name, jsonReader.nextLong());
} else {
contentValues.put(name, jsonReader.nextString());
}
}
jsonReader.endObject();
jsonReader.endObject();
if (Account.TABLENAME.equals(table)) {
final Jid jid =
Jid.of(
contentValues.getAsString(Account.USERNAME),
contentValues.getAsString(Account.SERVER),
null);
final String password = contentValues.getAsString(Account.PASSWORD);
if (jid.equals(account) && passphrase.equals(password)) {
Log.d(Config.LOGTAG, "jid and password from backup header had matching row");
} else {
throw new IOException("jid or password in table did not match backup");
}
}
db.insert(table, null, contentValues);
}
private void notifySuccess() {
NotificationCompat.Builder mBuilder = new NotificationCompat.Builder(getBaseContext(), "backup");
mBuilder.setContentTitle(getString(R.string.notification_restored_backup_title))