mark deleted files in database and not query them when querying for media

14 files changed, 162 insertions, 97 deletions

diff --git a/src/main/java/de/pixart/messenger/crypto/PgpDecryptionService.java b/src/main/java/de/pixart/messenger/crypto/PgpDecryptionService.java
index 8f01c6982..93cddacc1 100644
--- a/src/main/java/de/pixart/messenger/crypto/PgpDecryptionService.java
+++ b/src/main/java/de/pixart/messenger/crypto/PgpDecryptionService.java
@@ -209,7 +209,9 @@ public class PgpDecryptionService {
                             URL url = message.getFileParams().url;
                             mXmppConnectionService.getFileBackend().updateFileParams(message, url);
                             message.setEncryption(Message.ENCRYPTION_DECRYPTED);
-                            inputFile.delete();
+                            if (!inputFile.delete()) {
+                                Log.w(Config.LOGTAG, "unable to delete pgp encrypted source file " + inputFile.getAbsolutePath());
+                            }
                             mXmppConnectionService.updateMessage(message);
                             skipNotificationPush = true;
                             mXmppConnectionService.getFileBackend().updateMediaScanner(outputFile, () -> notifyIfPending(message));
diff --git a/src/main/java/de/pixart/messenger/entities/Conversation.java b/src/main/java/de/pixart/messenger/entities/Conversation.java
index e64073d61..8c8b22231 100644
--- a/src/main/java/de/pixart/messenger/entities/Conversation.java
+++ b/src/main/java/de/pixart/messenger/entities/Conversation.java
@@ -250,6 +250,19 @@ public class Conversation extends AbstractEntity implements Blockable, Comparabl
         return null;
     }
 
+    public boolean markAsDeleted(final List<String> uuids) {
+        boolean deleted = false;
+        synchronized (this.messages) {
+            for(Message message : this.messages) {
+                if (uuids.contains(message.getUuid())) {
+                    message.setDeleted(true);
+                    deleted = true;
+                }
+            }
+        }
+        return deleted;
+    }
+
     public void clearMessages() {
         synchronized (this.messages) {
             this.messages.clear();
diff --git a/src/main/java/de/pixart/messenger/entities/IndividualMessage.java b/src/main/java/de/pixart/messenger/entities/IndividualMessage.java
index 2a7b4c8da..195ea82c1 100644
--- a/src/main/java/de/pixart/messenger/entities/IndividualMessage.java
+++ b/src/main/java/de/pixart/messenger/entities/IndividualMessage.java
@@ -41,8 +41,8 @@ public class IndividualMessage extends Message {
         super(conversation);
     }
 
-    private IndividualMessage(Conversational conversation, String uuid, String conversationUUid, Jid counterpart, Jid trueCounterpart, String body, long timeSent, int encryption, int status, int type, boolean carbon, String remoteMsgId, String relativeFilePath, String serverMsgId, String fingerprint, boolean read, boolean deleted, String edited, boolean oob, String errorMessage, Set<ReadByMarker> readByMarkers, boolean markable) {
-        super(conversation, uuid, conversationUUid, counterpart, trueCounterpart, body, timeSent, encryption, status, type, carbon, remoteMsgId, relativeFilePath, serverMsgId, fingerprint, read, deleted, edited, oob, errorMessage, readByMarkers, markable);
+    private IndividualMessage(Conversational conversation, String uuid, String conversationUUid, Jid counterpart, Jid trueCounterpart, String body, long timeSent, int encryption, int status, int type, boolean carbon, String remoteMsgId, String relativeFilePath, String serverMsgId, String fingerprint, boolean read, boolean deleted, String edited, boolean oob, String errorMessage, Set<ReadByMarker> readByMarkers, boolean markable, boolean file_deleted) {
+        super(conversation, uuid, conversationUUid, counterpart, trueCounterpart, body, timeSent, encryption, status, type, carbon, remoteMsgId, relativeFilePath, serverMsgId, fingerprint, read, deleted, edited, oob, errorMessage, readByMarkers, markable, file_deleted);
     }
 
     public static Message createDateSeparator(Message message) {
@@ -99,7 +99,8 @@ public class IndividualMessage extends Message {
                 cursor.getInt(cursor.getColumnIndex(OOB)) > 0,
                 cursor.getString(cursor.getColumnIndex(ERROR_MESSAGE)),
                 ReadByMarker.fromJsonString(cursor.getString(cursor.getColumnIndex(READ_BY_MARKERS))),
-                cursor.getInt(cursor.getColumnIndex(MARKABLE)) > 0);
+                cursor.getInt(cursor.getColumnIndex(MARKABLE)) > 0,
+                cursor.getInt(cursor.getColumnIndex(FILE_DELETED)) > 0);
     }
 
     @Override
diff --git a/src/main/java/de/pixart/messenger/entities/Message.java b/src/main/java/de/pixart/messenger/entities/Message.java
index abeda3ad3..8037a022a 100644
--- a/src/main/java/de/pixart/messenger/entities/Message.java
+++ b/src/main/java/de/pixart/messenger/entities/Message.java
@@ -73,6 +73,7 @@ public class Message extends AbstractEntity {
     public static final String ERROR_MESSAGE = "errorMsg";
     public static final String READ_BY_MARKERS = "readByMarkers";
     public static final String MARKABLE = "markable";
+    public static final String FILE_DELETED = "file_deleted";
     public static final String ME_COMMAND = "/me";
     public static final String ERROR_MESSAGE_CANCELLED = "eu.siacs.conversations.cancelled";
 
@@ -86,6 +87,7 @@ public class Message extends AbstractEntity {
     protected int encryption;
     protected int status;
     protected int type;
+    protected boolean file_deleted = false;
     protected boolean carbon = false;
     protected boolean oob = false;
     protected String edited = null;
@@ -139,6 +141,7 @@ public class Message extends AbstractEntity {
                 false,
                 null,
                 null,
+                false,
                 false);
     }
 
@@ -148,7 +151,7 @@ public class Message extends AbstractEntity {
                     final String remoteMsgId, final String relativeFilePath,
                     final String serverMsgId, final String fingerprint, final boolean read, final boolean deleted,
                     final String edited, final boolean oob, final String errorMessage, final Set<ReadByMarker> readByMarkers,
-                    final boolean markable) {
+                    final boolean markable, final boolean file_deleted) {
         this.conversation = conversation;
         this.uuid = uuid;
         this.conversationUuid = conversationUUid;
@@ -171,6 +174,7 @@ public class Message extends AbstractEntity {
         this.errorMessage = errorMessage;
         this.readByMarkers = readByMarkers == null ? new HashSet<ReadByMarker>() : readByMarkers;
         this.markable = markable;
+        this.file_deleted = file_deleted;
     }
 
     public static Message fromCursor(Cursor cursor, Conversation conversation) {
@@ -219,7 +223,8 @@ public class Message extends AbstractEntity {
                 cursor.getInt(cursor.getColumnIndex(OOB)) > 0,
                 cursor.getString(cursor.getColumnIndex(ERROR_MESSAGE)),
                 ReadByMarker.fromJsonString(cursor.getString(cursor.getColumnIndex(READ_BY_MARKERS))),
-                cursor.getInt(cursor.getColumnIndex(MARKABLE)) > 0);
+                cursor.getInt(cursor.getColumnIndex(MARKABLE)) > 0,
+                cursor.getInt(cursor.getColumnIndex(FILE_DELETED)) > 0);
     }
 
     public static Message createStatusMessage(Conversation conversation, String body) {
@@ -269,6 +274,7 @@ public class Message extends AbstractEntity {
         values.put(ERROR_MESSAGE, errorMessage);
         values.put(READ_BY_MARKERS, ReadByMarker.toJson(readByMarkers).toString());
         values.put(MARKABLE, markable ? 1 : 0);
+        values.put(FILE_DELETED, file_deleted ? 1 : 0);
         return values;
     }
 
@@ -390,6 +396,14 @@ public class Message extends AbstractEntity {
         return this.deleted;
     }
 
+    public boolean isDeleted() {
+        return this.deleted;
+    }
+
+    public void setDeleted(boolean deleted) {
+        this.deleted = deleted;
+    }
+
     public void markRead() {
         this.read = true;
     }
@@ -840,10 +854,6 @@ public class Message extends AbstractEntity {
         return type == TYPE_FILE || type == TYPE_IMAGE;
     }
 
-    public boolean isDeleted() {
-        return (type == TYPE_FILE || type == TYPE_IMAGE) && getTransferable() != null && getTransferable().getFileSize() == 0;
-    }
-
     public boolean hasFileOnRemoteHost() {
         return isFileOrImage() && getFileParams().url != null;
     }
diff --git a/src/main/java/de/pixart/messenger/entities/Transferable.java b/src/main/java/de/pixart/messenger/entities/Transferable.java
index c07ec750d..e1deb744d 100644
--- a/src/main/java/de/pixart/messenger/entities/Transferable.java
+++ b/src/main/java/de/pixart/messenger/entities/Transferable.java
@@ -18,50 +18,12 @@ public interface Transferable {
             "gpg",
             "otr"
     );
-    List<String> WELL_KNOWN_EXTENSIONS = Arrays.asList(
-            //documents
-            "pdf",
-            "doc",
-            "docx",
-            "txt",
-            //audio
-            "m4a",
-            "m4b",
-            "mp3",
-            "mp2",
-            "wav",
-            "aac",
-            "aif",
-            "aiff",
-            "aifc",
-            "mid",
-            "midi",
-            "3gpp",
-            //video
-            "avi",
-            "mp4",
-            "mpeg",
-            "mpg",
-            "mpe",
-            "mov",
-            "3gp",
-            //applications
-            "apk",
-            //contact
-            "vcf",
-            //calendar
-            "ics",
-            //compressed
-            "zip",
-            "rar"
-    );
 
     int STATUS_UNKNOWN = 0x200;
     int STATUS_CHECKING = 0x201;
     int STATUS_FAILED = 0x202;
     int STATUS_OFFER = 0x203;
     int STATUS_DOWNLOADING = 0x204;
-    int STATUS_DELETED = 0x205;
     int STATUS_OFFER_CHECK_FILESIZE = 0x206;
     int STATUS_UPLOADING = 0x207;
 
diff --git a/src/main/java/de/pixart/messenger/http/HttpDownloadConnection.java b/src/main/java/de/pixart/messenger/http/HttpDownloadConnection.java
index b20e57e3f..308ec0605 100644
--- a/src/main/java/de/pixart/messenger/http/HttpDownloadConnection.java
+++ b/src/main/java/de/pixart/messenger/http/HttpDownloadConnection.java
@@ -137,10 +137,9 @@ public class HttpDownloadConnection implements Transferable {
     public void cancel() {
         this.canceled = true;
         mHttpConnectionManager.finishConnection(this);
+        message.setTransferable(null);
         if (message.isFileOrImage()) {
-            message.setTransferable(new TransferablePlaceholder(Transferable.STATUS_DELETED));
-        } else {
-            message.setTransferable(null);
+            message.setDeleted(true);
         }
         mHttpConnectionManager.updateConversationUi(true);
     }
diff --git a/src/main/java/de/pixart/messenger/persistance/DatabaseBackend.java b/src/main/java/de/pixart/messenger/persistance/DatabaseBackend.java
index 85d32d55f..22c1ae6e3 100644
--- a/src/main/java/de/pixart/messenger/persistance/DatabaseBackend.java
+++ b/src/main/java/de/pixart/messenger/persistance/DatabaseBackend.java
@@ -60,7 +60,7 @@ import rocks.xmpp.addr.Jid;
 public class DatabaseBackend extends SQLiteOpenHelper {
 
     public static final String DATABASE_NAME = "history";
-    public static final int DATABASE_VERSION = 44; // = Conversations DATABASE_VERSION + 2
+    public static final int DATABASE_VERSION = 45; // = Conversations DATABASE_VERSION + 1
     private static DatabaseBackend instance = null;
 
     private static String CREATE_CONTATCS_STATEMENT = "create table "
@@ -161,6 +161,10 @@ public class DatabaseBackend extends SQLiteOpenHelper {
             + ");";
     private static String CREATE_MESSAGE_TIME_INDEX = "create INDEX message_time_index ON " + Message.TABLENAME + "(" + Message.TIME_SENT + ")";
     private static String CREATE_MESSAGE_CONVERSATION_INDEX = "create INDEX message_conversation_index ON " + Message.TABLENAME + "(" + Message.CONVERSATION + ")";
+    private static String CREATE_MESSAGE_DELETED_INDEX = "create index message_deleted_index ON "+ Message.TABLENAME + "(" + Message.DELETED + ")";
+    private static String CREATE_MESSAGE_FILE_DELETED_INDEX = "create index message_file_deleted_index ON "+ Message.TABLENAME + "(" + Message.FILE_DELETED + ")";
+    private static String CREATE_MESSAGE_RELATIVE_FILE_PATH_INDEX = "create INDEX message_file_path_index ON " + Message.TABLENAME + "(" + Message.RELATIVE_FILE_PATH + ")";
+    private static String CREATE_MESSAGE_TYPE_INDEX = "create INDEX message_type_index ON " + Message.TABLENAME + "(" + Message.TYPE + ")";
 
     private static String CREATE_MESSAGE_INDEX_TABLE = "CREATE VIRTUAL TABLE messages_index USING FTS4(uuid TEXT PRIMARY KEY, body TEXT)";
     private static String CREATE_MESSAGE_INSERT_TRIGGER = "CREATE TRIGGER after_message_insert AFTER INSERT ON " + Message.TABLENAME + " BEGIN INSERT INTO messages_index (uuid,body) VALUES (new.uuid,new.body); END;";
@@ -233,6 +237,7 @@ public class DatabaseBackend extends SQLiteOpenHelper {
                 + Message.ERROR_MESSAGE + " TEXT,"
                 + Message.READ_BY_MARKERS + " TEXT,"
                 + Message.MARKABLE + " NUMBER DEFAULT 0,"
+                + Message.FILE_DELETED + " NUMBER DEFAULT 0,"
                 + Message.REMOTE_MSG_ID + " TEXT, FOREIGN KEY("
                 + Message.CONVERSATION + ") REFERENCES "
                 + Conversation.TABLENAME + "(" + Conversation.UUID
@@ -240,6 +245,10 @@ public class DatabaseBackend extends SQLiteOpenHelper {
 
         db.execSQL(CREATE_MESSAGE_TIME_INDEX);
         db.execSQL(CREATE_MESSAGE_CONVERSATION_INDEX);
+        db.execSQL(CREATE_MESSAGE_DELETED_INDEX);
+        db.execSQL(CREATE_MESSAGE_FILE_DELETED_INDEX);
+        db.execSQL(CREATE_MESSAGE_RELATIVE_FILE_PATH_INDEX);
+        db.execSQL(CREATE_MESSAGE_TYPE_INDEX);
         db.execSQL(CREATE_CONTATCS_STATEMENT);
         db.execSQL(CREATE_DISCOVERY_RESULTS_STATEMENT);
         db.execSQL(CREATE_SESSIONS_STATEMENT);
@@ -535,6 +544,14 @@ public class DatabaseBackend extends SQLiteOpenHelper {
         if (oldVersion < 44 && newVersion >= 44) {
             db.execSQL("ALTER TABLE " + Message.TABLENAME + " ADD COLUMN " + Message.DELETED + " NUMBER DEFAULT 0");
         }
+
+        if (oldVersion < 45 && newVersion >= 45) {
+            db.execSQL("ALTER TABLE " + Message.TABLENAME + " ADD COLUMN " + Message.FILE_DELETED + " NUMBER DEFAULT 0");
+            db.execSQL(CREATE_MESSAGE_DELETED_INDEX);
+            db.execSQL(CREATE_MESSAGE_FILE_DELETED_INDEX);
+            db.execSQL(CREATE_MESSAGE_RELATIVE_FILE_PATH_INDEX);
+            db.execSQL(CREATE_MESSAGE_TYPE_INDEX);
+        }
     }
 
     private void canonicalizeJids(SQLiteDatabase db) {
@@ -797,9 +814,59 @@ public class DatabaseBackend extends SQLiteOpenHelper {
         };
     }
 
+    public List<String> markFileAsDeleted(final File file, final boolean internal) {
+        SQLiteDatabase db = this.getReadableDatabase();
+        String selection;
+        String[] selectionArgs;
+        if (internal) {
+            selection = Message.RELATIVE_FILE_PATH + " IN(?,?) and type in (1,2)";
+            selectionArgs = new String[]{file.getAbsolutePath(), file.getName()};
+        } else {
+            selection = Message.RELATIVE_FILE_PATH + "=? and type in (1,2)";
+            selectionArgs = new String[]{file.getAbsolutePath()};
+        }
+        final List<String> uuids = new ArrayList<>();
+        Cursor cursor = db.query(Message.TABLENAME, new String[]{Message.UUID}, selection, selectionArgs, null, null, null);
+        while (cursor != null && cursor.moveToNext()) {
+            uuids.add(cursor.getString(0));
+        }
+        if (cursor != null) {
+            cursor.close();
+        }
+        markFileAsDeleted(uuids);
+        return uuids;
+    }
+
+    public void markFileAsDeleted(List<String> uuids) {
+        SQLiteDatabase db = this.getReadableDatabase();
+        final ContentValues contentValues = new ContentValues();
+        final String where = Message.UUID + "=?";
+        contentValues.put(Message.FILE_DELETED, 1);
+        db.beginTransaction();
+        for (String uuid : uuids) {
+            db.update(Message.TABLENAME, contentValues, where, new String[]{uuid});
+        }
+        db.setTransactionSuccessful();
+        db.endTransaction();
+    }
+
+    public List<FilePath> getAllNonDeletedFilePath() {
+        final SQLiteDatabase db = this.getReadableDatabase();
+        final Cursor cursor = db.query(Message.TABLENAME, new String[]{Message.UUID, Message.RELATIVE_FILE_PATH}, "type in (1,2) and file_deleted=0", null, null, null, null);
+        final List<FilePath> list = new ArrayList<>();
+        while (cursor != null && cursor.moveToNext()) {
+            list.add(new FilePath(cursor.getString(0), cursor.getString(1)));
+        }
+        if (cursor != null) {
+            cursor.close();
+        }
+        return list;
+    }
+
     public List<FilePath> getRelativeFilePaths(String account, Jid jid, int limit) {
         SQLiteDatabase db = this.getReadableDatabase();
-        final String SQL = "select uuid,relativeFilePath from messages where type in (1,2) and conversationUuid=(select uuid from conversations where accountUuid=? and (contactJid=? or contactJid like ?)) order by timeSent desc";
+        final String SQL = "select uuid,relativeFilePath from messages where type in (1,2) and file_deleted=0 and conversationUuid=(select uuid from conversations where accountUuid=? and (contactJid=? or contactJid like ?)) order by timeSent desc";
+
         final String[] args = {account, jid.toEscapedString(), jid.toEscapedString() + "/%"};
         Cursor cursor = db.rawQuery(SQL + (limit > 0 ? " limit " + String.valueOf(limit) : ""), args);
         List<FilePath> filesPaths = new ArrayList<>();
diff --git a/src/main/java/de/pixart/messenger/persistance/FileBackend.java b/src/main/java/de/pixart/messenger/persistance/FileBackend.java
index 952bb7369..ce3759bb3 100644
--- a/src/main/java/de/pixart/messenger/persistance/FileBackend.java
+++ b/src/main/java/de/pixart/messenger/persistance/FileBackend.java
@@ -173,6 +173,10 @@ public class FileBackend {
         return getFile(message, true);
     }
 
+    public DownloadableFile getFileForPath(String path) {
+        return getFileForPath(path, MimeUtils.guessMimeTypeFromExtension(MimeUtils.extractRelevantExtension(path)));
+    }
+
     public DownloadableFile getFileForPath(String path, String mime) {
         final DownloadableFile file;
         if (path.startsWith("/")) {
@@ -191,6 +195,11 @@ public class FileBackend {
         return file;
     }
 
+    public boolean isInternalFile(final File file) {
+        final File internalFile = getFileForPath(file.getName());
+        return file.getAbsolutePath().equals(internalFile.getAbsolutePath());
+    }
+
     public DownloadableFile getFile(Message message, boolean decrypted) {
         final boolean encrypted = !decrypted
                 && (message.getEncryption() == Message.ENCRYPTION_PGP
@@ -1046,6 +1055,7 @@ public class FileBackend {
             body.append("|0|0|0|").append(getAPK(file, mXmppConnectionService.getApplicationContext()));
         }
         message.setBody(body.toString());
+        message.setDeleted(false);
     }
 
     public int getMediaRuntime(Uri uri) {
diff --git a/src/main/java/de/pixart/messenger/services/NotificationService.java b/src/main/java/de/pixart/messenger/services/NotificationService.java
index 270005dc9..7608976a5 100644
--- a/src/main/java/de/pixart/messenger/services/NotificationService.java
+++ b/src/main/java/de/pixart/messenger/services/NotificationService.java
@@ -737,6 +737,7 @@ public class NotificationService {
     private static boolean isImageMessage(Message message) {
         return message.getType() != Message.TYPE_TEXT
                 && message.getTransferable() == null
+                && !message.isDeleted()
                 && message.getEncryption() != Message.ENCRYPTION_PGP
                 && message.getFileParams().height > 0;
     }
diff --git a/src/main/java/de/pixart/messenger/services/XmppConnectionService.java b/src/main/java/de/pixart/messenger/services/XmppConnectionService.java
index defc6244d..e123e8cff 100644
--- a/src/main/java/de/pixart/messenger/services/XmppConnectionService.java
+++ b/src/main/java/de/pixart/messenger/services/XmppConnectionService.java
@@ -52,6 +52,7 @@ import org.openintents.openpgp.IOpenPgpService2;
 import org.openintents.openpgp.util.OpenPgpApi;
 import org.openintents.openpgp.util.OpenPgpServiceConnection;
 
+import java.io.File;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.security.Security;
@@ -238,7 +239,6 @@ public class XmppConnectionService extends Service {
     ) {
         @Override
         public void onEvent(int event, String path) {
-            Log.d(Config.LOGTAG, "event " + event + " path=" + path);
             markFileDeleted(path);
         }
     };
@@ -375,7 +375,6 @@ public class XmppConnectionService extends Service {
                         if (!conversation.startOtrIfNeeded()) {
                             Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": couldn't start OTR with " + conversation.getContact().getJid() + " when needed");
                         }
-                        checkDeletedFiles(conversation);
                         sendUnsentMessages(conversation);
                         resendFailedFileMessages(conversation);
                     }
@@ -1162,6 +1161,7 @@ public class XmppConnectionService extends Service {
         if (Compatibility.hasStoragePermission(this)) {
             Log.d(Config.LOGTAG, "starting file observer");
             new Thread(fileObserver::startWatching).start();
+            mFileAddingExecutor.execute(this::checkForDeletedFiles);
         }
         if (Config.supportOpenPgp()) {
             this.pgpServiceConnection = new OpenPgpServiceConnection(this, "org.sufficientlysecure.keychain", new OpenPgpServiceConnection.OnBound() {
@@ -1202,6 +1202,22 @@ public class XmppConnectionService extends Service {
         CancelAutomaticExport(false);
     }
 
+    private void checkForDeletedFiles() {
+        final List<String> deletedUuids = new ArrayList<>();
+        final List<DatabaseBackend.FilePath> relativeFilePaths = databaseBackend.getAllNonDeletedFilePath();
+        for (final DatabaseBackend.FilePath filePath : relativeFilePaths) {
+            final File file = fileBackend.getFileForPath(filePath.path);
+            if (!file.exists()) {
+                deletedUuids.add(filePath.uuid.toString());
+            }
+        }
+        Log.d(Config.LOGTAG, "found " + deletedUuids.size() + " deleted files on start up. total=" + relativeFilePaths.size());
+        if (deletedUuids.size() > 0) {
+            databaseBackend.markFileAsDeleted(deletedUuids);
+            markUuidsAsDeletedFiles(deletedUuids);
+        }
+    }
+
     public void startContactObserver() {
         getContentResolver().registerContentObserver(ContactsContract.Contacts.CONTENT_URI, true, new ContentObserver(null) {
             @Override
@@ -1238,6 +1254,7 @@ public class XmppConnectionService extends Service {
 
     public void restartFileObserver() {
         Log.d(Config.LOGTAG, "restarting file observer");
+        mFileAddingExecutor.execute(this::checkForDeletedFiles);
         new Thread(fileObserver::restartWatching).start();
     }
 
@@ -1776,7 +1793,6 @@ public class XmppConnectionService extends Service {
 
     private void restoreMessages(Conversation conversation) {
         conversation.addAll(0, databaseBackend.getMessages(conversation, Config.PAGE_SIZE));
-        checkDeletedFiles(conversation);
         conversation.findUnsentTextMessages(message -> markMessage(message, Message.STATUS_WAITING));
         conversation.findUnreadMessages(message -> mNotificationService.pushFromBacklog(message));
     }
@@ -1817,37 +1833,21 @@ public class XmppConnectionService extends Service {
         return this.conversations;
     }
 
-    private void checkDeletedFiles(Conversation conversation) {
-        conversation.findMessagesWithFiles(message -> {
-            if (!getFileBackend().isFileAvailable(message)) {
-                message.setTransferable(new TransferablePlaceholder(Transferable.STATUS_DELETED));
-                final int s = message.getStatus();
-                if (s == Message.STATUS_WAITING || s == Message.STATUS_OFFERED || s == Message.STATUS_UNSEND) {
-                    markMessage(message, Message.STATUS_SEND_FAILED);
-                }
-            }
-        });
+    private void markFileDeleted(final String path) {
+        final File file = new File(path);
+        final boolean isInternalFile = fileBackend.isInternalFile(file);
+        final List<String> uuids = databaseBackend.markFileAsDeleted(file, isInternalFile);
+        Log.d(Config.LOGTAG, "deleted file " + path + " internal=" + isInternalFile + ", database hits=" + uuids.size());
+        markUuidsAsDeletedFiles(uuids);
     }
 
-    private void markFileDeleted(final String path) {
-        Log.d(Config.LOGTAG, "deleted file " + path);
+    private void markUuidsAsDeletedFiles(List<String> uuids) {
+        boolean deleted = false;
         for (Conversation conversation : getConversations()) {
-            conversation.findMessagesWithFiles(message -> {
-                DownloadableFile file = fileBackend.getFile(message);
-                if (file.getAbsolutePath().equals(path)) {
-                    if (!file.exists()) {
-                        message.setTransferable(new TransferablePlaceholder(Transferable.STATUS_DELETED));
-                        final int s = message.getStatus();
-                        if (s == Message.STATUS_WAITING || s == Message.STATUS_OFFERED || s == Message.STATUS_UNSEND) {
-                            markMessage(message, Message.STATUS_SEND_FAILED);
-                        } else {
-                            updateConversationUi();
-                        }
-                    } else {
-                        Log.d(Config.LOGTAG, "found matching message for file " + path + " but file still exists");
-                    }
-                }
-            });
+            deleted |= conversation.markAsDeleted(uuids);
+        }
+        if (deleted) {
+            updateConversationUi();
         }
     }
 
@@ -1886,7 +1886,6 @@ public class XmppConnectionService extends Service {
             List<Message> messages = databaseBackend.getMessages(conversation, 50, timestamp);
             if (messages.size() > 0) {
                 conversation.addAll(0, messages);
-                checkDeletedFiles(conversation);
                 callback.onMoreMessagesLoaded(messages.size(), conversation);
             } else if (conversation.hasMessagesLeftOnServer()
                     && account.isOnlineAndConnected()
@@ -2033,7 +2032,6 @@ public class XmppConnectionService extends Service {
                         }
                     }
                 }
-                checkDeletedFiles(c);
                 if (joinAfterCreate) {
                     joinMuc(c);
 
@@ -2097,7 +2095,6 @@ public class XmppConnectionService extends Service {
                         updateConversationUi();
                         c.messagesLoaded.set(true);
                     }
-                    checkDeletedFiles(c);
                 }
             });
             updateConversationUi();
diff --git a/src/main/java/de/pixart/messenger/ui/ConversationFragment.java b/src/main/java/de/pixart/messenger/ui/ConversationFragment.java
index 4df6391da..d907f58a4 100644
--- a/src/main/java/de/pixart/messenger/ui/ConversationFragment.java
+++ b/src/main/java/de/pixart/messenger/ui/ConversationFragment.java
@@ -1277,7 +1277,7 @@ public class ConversationFragment extends XmppFragment implements EditMessage.Ke
                 return;
             }
 
-            final boolean deleted = t != null && t instanceof TransferablePlaceholder;
+            final boolean deleted = m.isDeleted();
             final boolean encrypted = m.getEncryption() == Message.ENCRYPTION_DECRYPTION_FAILED
                     || m.getEncryption() == Message.ENCRYPTION_PGP;
             final boolean receiving = m.getStatus() == Message.STATUS_RECEIVED && (t instanceof JingleConnection || t instanceof HttpDownloadConnection);
@@ -1876,7 +1876,8 @@ public class ConversationFragment extends XmppFragment implements EditMessage.Ke
         builder.setMessage(R.string.delete_file_dialog_msg);
         builder.setPositiveButton(R.string.confirm, (dialog, which) -> {
             if (activity.xmppConnectionService.getFileBackend().deleteFile(message)) {
-                message.setTransferable(new TransferablePlaceholder(Transferable.STATUS_DELETED));
+                message.setDeleted(true);
+                activity.xmppConnectionService.updateMessage(message, false);
                 activity.onConversationsListItemUpdated();
                 refresh();
             }
@@ -1909,7 +1910,9 @@ public class ConversationFragment extends XmppFragment implements EditMessage.Ke
                 }
             } else {
                 Toast.makeText(activity, R.string.file_deleted, Toast.LENGTH_SHORT).show();
-                message.setTransferable(new TransferablePlaceholder(Transferable.STATUS_DELETED));
+                //TODO check if we have storage permission
+                message.setDeleted(true);
+                activity.xmppConnectionService.updateMessage(message, false);
                 activity.onConversationsListItemUpdated();
                 refresh();
                 return;
diff --git a/src/main/java/de/pixart/messenger/ui/adapter/ConversationAdapter.java b/src/main/java/de/pixart/messenger/ui/adapter/ConversationAdapter.java
index 3e7bccc75..f2ae272f9 100644
--- a/src/main/java/de/pixart/messenger/ui/adapter/ConversationAdapter.java
+++ b/src/main/java/de/pixart/messenger/ui/adapter/ConversationAdapter.java
@@ -139,7 +139,7 @@ public class ConversationAdapter extends RecyclerView.Adapter<ConversationAdapte
             viewHolder.lastMessage.setTypeface(null, Typeface.NORMAL);
             viewHolder.sender.setTypeface(null, Typeface.ITALIC);
         } else {
-            final boolean fileAvailable = message.getTransferable() == null || message.getTransferable().getStatus() != Transferable.STATUS_DELETED;
+            final boolean fileAvailable = !message.isDeleted();
             final boolean showPreviewText;
             if (fileAvailable && (message.isFileOrImage() || message.treatAsDownloadable() || message.isGeoUri())) {
                 final int imageResource;
diff --git a/src/main/java/de/pixart/messenger/ui/adapter/MessageAdapter.java b/src/main/java/de/pixart/messenger/ui/adapter/MessageAdapter.java
index 1c53f1b5d..04373fb6c 100644
--- a/src/main/java/de/pixart/messenger/ui/adapter/MessageAdapter.java
+++ b/src/main/java/de/pixart/messenger/ui/adapter/MessageAdapter.java
@@ -962,10 +962,10 @@ public class MessageAdapter extends ArrayAdapter<Message> implements CopyTextVie
                 });
 
         final Transferable transferable = message.getTransferable();
-        if (transferable != null && transferable.getStatus() != Transferable.STATUS_UPLOADING) {
-            if (transferable.getStatus() == Transferable.STATUS_OFFER) {
+        if (message.isDeleted() || (transferable != null && transferable.getStatus() != Transferable.STATUS_UPLOADING)) {
+            if (transferable != null && transferable.getStatus() == Transferable.STATUS_OFFER) {
                 displayDownloadableMessage(viewHolder, message, activity.getString(R.string.download_x_file, UIHelper.getFileDescriptionString(activity, message)));
-            } else if (transferable.getStatus() == Transferable.STATUS_OFFER_CHECK_FILESIZE) {
+            } else if (transferable != null && transferable.getStatus() == Transferable.STATUS_OFFER_CHECK_FILESIZE) {
                 displayDownloadableMessage(viewHolder, message, activity.getString(R.string.check_x_filesize, UIHelper.getFileDescriptionString(activity, message)));
             } else {
                 displayInfoMessage(viewHolder, UIHelper.getMessagePreview(activity, message).first, darkBackground);
diff --git a/src/main/java/de/pixart/messenger/utils/UIHelper.java b/src/main/java/de/pixart/messenger/utils/UIHelper.java
index 03af31e23..6ac5ae39d 100644
--- a/src/main/java/de/pixart/messenger/utils/UIHelper.java
+++ b/src/main/java/de/pixart/messenger/utils/UIHelper.java
@@ -263,8 +263,6 @@ public class UIHelper {
                 case Transferable.STATUS_OFFER_CHECK_FILESIZE:
                     return new Pair<>(context.getString(R.string.x_file_offered_for_download,
                             getFileDescriptionString(context, message)), true);
-                case Transferable.STATUS_DELETED:
-                    return new Pair<>(context.getString(R.string.file_deleted), true);
                 case Transferable.STATUS_FAILED:
                     return new Pair<>(context.getString(R.string.file_transmission_failed), true);
                 case Transferable.STATUS_UPLOADING:
@@ -278,6 +276,8 @@ public class UIHelper {
                 default:
                     return new Pair<>("", false);
             }
+        } else if (message.isFileOrImage() && message.isDeleted()) {
+            return new Pair<>(context.getString(R.string.file_deleted), true);
         } else if (message.getEncryption() == Message.ENCRYPTION_PGP) {
             return new Pair<>(context.getString(R.string.pgp_message), true);
         } else if (message.getEncryption() == Message.ENCRYPTION_DECRYPTION_FAILED) {
@@ -286,7 +286,7 @@ public class UIHelper {
             return new Pair<>(context.getString(R.string.not_encrypted_for_this_device), true);
         } else if (message.getEncryption() == Message.ENCRYPTION_AXOLOTL_FAILED) {
             return new Pair<>(context.getString(R.string.omemo_decryption_failed), true);
-        } else if (message.getType() == Message.TYPE_FILE || message.getType() == Message.TYPE_IMAGE) {
+        } else if (message.isFileOrImage()) {
             return new Pair<>(getFileDescriptionString(context, message), true);
         } else {
             final String body = MessageUtils.filterLtrRtl(message.getBody());