diff --git a/app/src/main/java/eu/faircode/email/Core.java b/app/src/main/java/eu/faircode/email/Core.java index 1d03b79162..218df05b85 100644 --- a/app/src/main/java/eu/faircode/email/Core.java +++ b/app/src/main/java/eu/faircode/email/Core.java @@ -960,7 +960,7 @@ class Core { db.folder().setFolderTotal(folder.id, count < 0 ? null : count); } - WorkerFts.init(context); + WorkerFts.init(context, false); } private static void onDelete(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException { @@ -1989,7 +1989,7 @@ class Core { db.folder().setFolderSyncState(folder.id, null); } - WorkerFts.init(context); + WorkerFts.init(context, false); } static EntityMessage synchronizeMessage( diff --git a/app/src/main/java/eu/faircode/email/DaoMessage.java b/app/src/main/java/eu/faircode/email/DaoMessage.java index fbe9f30049..1c6979ad88 100644 --- a/app/src/main/java/eu/faircode/email/DaoMessage.java +++ b/app/src/main/java/eu/faircode/email/DaoMessage.java @@ -202,6 +202,12 @@ public interface DaoMessage { " AND ui_hide") LiveData> liveHiddenThread(long account, String thread); + @Query("SELECT SUM(fts) AS fts, COUNT(*) AS total FROM message" + + " JOIN folder ON folder.id = message.folder" + + " WHERE content" + + " AND folder.type <> '" + EntityFolder.OUTBOX + "'") + LiveData liveFts(); + @Query("SELECT *" + " FROM message" + " WHERE id = :id") @@ -227,8 +233,11 @@ public interface DaoMessage { " ORDER BY message.received DESC") List getMessageIdsByFolder(Long folder); - @Query("SELECT id FROM message" + - " WHERE content AND NOT fts" + + @Query("SELECT message.id FROM message" + + " JOIN folder ON folder.id = message.folder" + + " WHERE content" + + " AND NOT fts" + + " AND folder.type <> '" + EntityFolder.OUTBOX + "'" + " ORDER BY message.received DESC") Cursor getMessageFts(); @@ -541,6 +550,9 @@ public interface DaoMessage { @Query("UPDATE message SET headers = NULL WHERE headers IS NOT NULL") int clearMessageHeaders(); + @Query("UPDATE message SET fts = 0") + int resetFts(); + @Query("DELETE FROM message WHERE id = :id") int deleteMessage(long id); diff --git a/app/src/main/java/eu/faircode/email/EntityOperation.java b/app/src/main/java/eu/faircode/email/EntityOperation.java index 3964e87ba7..18356833ad 100644 --- a/app/src/main/java/eu/faircode/email/EntityOperation.java +++ b/app/src/main/java/eu/faircode/email/EntityOperation.java @@ -100,12 +100,16 @@ public class EntityOperation { for (Object value : values) jargs.put(value); - if (ADD.equals(name) && - (EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) || - EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt))) { - EntityFolder folder = db.folder().getFolder(message.folder); - if (folder != null && EntityFolder.DRAFTS.equals(folder.type)) - return; + if (ADD.equals(name)) { + db.message().setMessageFts(message.id, false); + WorkerFts.init(context, false); + + if (EntityMessage.PGP_SIGNENCRYPT.equals(message.encrypt) || + EntityMessage.SMIME_SIGNENCRYPT.equals(message.encrypt)) { + EntityFolder folder = db.folder().getFolder(message.folder); + if (folder != null && EntityFolder.DRAFTS.equals(folder.type)) + return; + } } if (MOVE.equals(name) && @@ -199,6 +203,7 @@ public class EntityOperation { Long identity = message.identity; long uid = message.uid; int notifying = message.notifying; + boolean fts = message.fts; boolean seen = message.seen; boolean ui_seen = message.ui_seen; Boolean ui_hide = message.ui_hide; @@ -211,6 +216,7 @@ public class EntityOperation { message.identity = null; message.uid = null; message.notifying = 0; + message.fts = false; if (autoread) { message.seen = true; message.ui_seen = true; @@ -229,6 +235,7 @@ public class EntityOperation { message.identity = identity; message.uid = uid; message.notifying = notifying; + message.fts = fts; message.seen = seen; message.ui_seen = ui_seen; message.ui_hide = ui_hide; @@ -245,6 +252,8 @@ public class EntityOperation { } EntityAttachment.copy(context, message.id, tmpid); + + WorkerFts.init(context, false); } // Cross account move diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java index 6dfecf0ffe..781c82f277 100644 --- a/app/src/main/java/eu/faircode/email/FragmentMessages.java +++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java @@ -4545,12 +4545,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. // Write decrypted body Helper.copy(plain, message.getFile(context)); db.message().setMessageStored(message.id, new Date().getTime()); + db.message().setMessageFts(message.id, false); db.setTransactionSuccessful(); } finally { db.endTransaction(); } + WorkerFts.init(context, false); } else { // Decode message MessageHelper.MessageParts parts; @@ -4588,11 +4590,14 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. db.message().setMessageEncrypt(message.id, parts.getEncryption()); db.message().setMessageStored(message.id, new Date().getTime()); + db.message().setMessageFts(message.id, false); db.setTransactionSuccessful(); } finally { db.endTransaction(); } + + WorkerFts.init(context, false); } // Check signature status @@ -4857,6 +4862,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. db.message().setMessageEncrypt(message.id, parts.getEncryption()); db.message().setMessageStored(message.id, new Date().getTime()); + db.message().setMessageFts(message.id, false); if (alias != null && message.identity != null) db.identity().setIdentitySignKeyAlias(message.identity, alias); @@ -4865,6 +4871,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. } finally { db.endTransaction(); } + + WorkerFts.init(context, false); } return result; diff --git a/app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java b/app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java index bd3375fe0c..5cee090cb7 100644 --- a/app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java +++ b/app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java @@ -43,11 +43,14 @@ import androidx.annotation.Nullable; import androidx.appcompat.widget.SwitchCompat; import androidx.constraintlayout.widget.Group; import androidx.lifecycle.Lifecycle; +import androidx.lifecycle.Observer; import androidx.preference.PreferenceManager; public class FragmentOptionsMisc extends FragmentBase implements SharedPreferences.OnSharedPreferenceChangeListener { private SwitchCompat swExternalSearch; private SwitchCompat swFts; + private Button btnFtsReset; + private TextView tvFtsIndexed; private SwitchCompat swEnglish; private SwitchCompat swWatchdog; private SwitchCompat swUpdates; @@ -90,6 +93,8 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc swExternalSearch = view.findViewById(R.id.swExternalSearch); swFts = view.findViewById(R.id.swFts); + btnFtsReset = view.findViewById(R.id.btnFtsReset); + tvFtsIndexed = view.findViewById(R.id.tvFtsIndexed); swEnglish = view.findViewById(R.id.swEnglish); swWatchdog = view.findViewById(R.id.swWatchdog); swUpdates = view.findViewById(R.id.swUpdates); @@ -132,7 +137,33 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc @Override public void onCheckedChanged(CompoundButton compoundButton, boolean checked) { prefs.edit().putBoolean("fts", checked).apply(); - WorkerFts.init(getContext()); + WorkerFts.init(getContext(), true); + } + }); + + btnFtsReset.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + Bundle args = new Bundle(); + + new SimpleTask() { + @Override + protected Void onExecute(Context context, Bundle args) throws Throwable { + DB db = DB.getInstance(context); + db.message().resetFts(); + return null; + } + + @Override + protected void onExecuted(Bundle args, Void data) { + WorkerFts.init(getContext(), true); + } + + @Override + protected void onException(Bundle args, Throwable ex) { + Log.unexpectedError(getParentFragmentManager(), ex); + } + }.execute(FragmentOptionsMisc.this, args, "fts:reset"); } }); @@ -211,6 +242,19 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc } }); + tvFtsIndexed.setText(null); + + DB db = DB.getInstance(getContext()); + db.message().liveFts().observe(getViewLifecycleOwner(), new Observer() { + @Override + public void onChanged(TupleFtsStats stats) { + if (stats == null) + tvFtsIndexed.setText(null); + else + tvFtsIndexed.setText(getString(R.string.title_advanced_fts_indexed, stats.fts, stats.total)); + } + }); + setLastCleanup(prefs.getLong("last_cleanup", -1)); PreferenceManager.getDefaultSharedPreferences(getContext()).registerOnSharedPreferenceChangeListener(this); diff --git a/app/src/main/java/eu/faircode/email/FtsDbHelper.java b/app/src/main/java/eu/faircode/email/FtsDbHelper.java index 97d382539c..23187ea282 100644 --- a/app/src/main/java/eu/faircode/email/FtsDbHelper.java +++ b/app/src/main/java/eu/faircode/email/FtsDbHelper.java @@ -38,14 +38,15 @@ public class FtsDbHelper extends SQLiteOpenHelper { private static final int DATABASE_VERSION = 1; private static final String DATABASE_NAME = "fts.db"; - public FtsDbHelper(Context context) { + FtsDbHelper(Context context) { super(context, DATABASE_NAME, null, DATABASE_VERSION); } @Override public void onCreate(SQLiteDatabase db) { Log.i("FTS create"); - db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)"); + db.execSQL("CREATE VIRTUAL TABLE `message`" + + " USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)"); } @Override @@ -66,7 +67,7 @@ public class FtsDbHelper extends SQLiteOpenHelper { try { db.beginTransaction(); - db.delete("message", "docid = ?", new Object[]{message.id}); + delete(db, message.id); ContentValues cv = new ContentValues(); cv.put("docid", message.id); @@ -86,6 +87,10 @@ public class FtsDbHelper extends SQLiteOpenHelper { } } + void delete(SQLiteDatabase db, long id) { + db.delete("message", "docid = ?", new Object[]{id}); + } + List match(SQLiteDatabase db, Long folder, String search) { Log.i("FTS folder=" + folder + " search=" + search); List result = new ArrayList<>(); @@ -100,4 +105,11 @@ public class FtsDbHelper extends SQLiteOpenHelper { Log.i("FTS result=" + result.size()); return result; } + + Cursor getIds(SQLiteDatabase db) { + return db.query( + "message", new String[]{"docid"}, + null, null, + null, null, "time"); + } } diff --git a/app/src/main/java/eu/faircode/email/TupleFtsStats.java b/app/src/main/java/eu/faircode/email/TupleFtsStats.java new file mode 100644 index 0000000000..ac2cc3759c --- /dev/null +++ b/app/src/main/java/eu/faircode/email/TupleFtsStats.java @@ -0,0 +1,25 @@ +package eu.faircode.email; + +/* + This file is part of FairEmail. + + FairEmail is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + FairEmail is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with FairEmail. If not, see . + + Copyright 2018-2020 by Marcel Bokhorst (M66B) +*/ + +public class TupleFtsStats { + public long fts; + public long total; +} diff --git a/app/src/main/java/eu/faircode/email/WorkerCleanup.java b/app/src/main/java/eu/faircode/email/WorkerCleanup.java index 60836e7840..41ba9e97ff 100644 --- a/app/src/main/java/eu/faircode/email/WorkerCleanup.java +++ b/app/src/main/java/eu/faircode/email/WorkerCleanup.java @@ -21,6 +21,7 @@ package eu.faircode.email; import android.content.Context; import android.content.SharedPreferences; +import android.database.Cursor; import androidx.annotation.NonNull; import androidx.preference.PreferenceManager; @@ -39,6 +40,8 @@ import java.util.Date; import java.util.List; import java.util.concurrent.TimeUnit; +import io.requery.android.database.sqlite.SQLiteDatabase; + import static android.os.Process.THREAD_PRIORITY_BACKGROUND; public class WorkerCleanup extends Worker { @@ -190,6 +193,24 @@ public class WorkerCleanup extends Worker { } } + Log.i("Cleanup FTS"); + int fts = 0; + FtsDbHelper ftsDb = new FtsDbHelper(context); + try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) { + try (Cursor cursor = ftsDb.getIds(sdb)) { + while (cursor.moveToNext()) { + long docid = cursor.getLong(0); + EntityMessage message = db.message().getMessage(docid); + if (message == null) { + Log.i("Deleting docid" + docid); + ftsDb.delete(sdb, docid); + fts++; + } + } + } + } + Log.i("Cleanup FTS=" + fts); + Log.i("Cleanup contacts"); int contacts = db.contact().deleteContacts(now - KEEP_CONTACTS_DURATION); Log.i("Deleted contacts=" + contacts); diff --git a/app/src/main/java/eu/faircode/email/WorkerFts.java b/app/src/main/java/eu/faircode/email/WorkerFts.java index abd362ee93..9fdfea0cda 100644 --- a/app/src/main/java/eu/faircode/email/WorkerFts.java +++ b/app/src/main/java/eu/faircode/email/WorkerFts.java @@ -37,6 +37,8 @@ import java.util.concurrent.TimeUnit; import io.requery.android.database.sqlite.SQLiteDatabase; public class WorkerFts extends Worker { + private static final int INDEX_DELAY = 30; // seconds + public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) { super(context, workerParams); Log.i("Instance " + getName()); @@ -78,23 +80,23 @@ public class WorkerFts extends Worker { } } - static void init(Context context) { + static void init(Context context, boolean immediately) { try { SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); boolean fts = prefs.getBoolean("fts", true); if (fts) { Log.i("Queuing " + getName()); - OneTimeWorkRequest workRequest = - new OneTimeWorkRequest.Builder(WorkerFts.class) - .setInitialDelay(30, TimeUnit.SECONDS) - .build(); + OneTimeWorkRequest.Builder builder = new OneTimeWorkRequest.Builder(WorkerFts.class); + if (!immediately) + builder.setInitialDelay(INDEX_DELAY, TimeUnit.SECONDS); + OneTimeWorkRequest workRequest = builder.build(); WorkManager.getInstance(context) .enqueueUniqueWork(getName(), ExistingWorkPolicy.REPLACE, workRequest); Log.i("Queued " + getName()); - } else { + } else if (immediately) { Log.i("Cancelling " + getName()); WorkManager.getInstance(context).cancelUniqueWork(getName()); Log.i("Cancelled " + getName()); diff --git a/app/src/main/res/layout/fragment_options_misc.xml b/app/src/main/res/layout/fragment_options_misc.xml index a0c0513db5..6f293a07d3 100644 --- a/app/src/main/res/layout/fragment_options_misc.xml +++ b/app/src/main/res/layout/fragment_options_misc.xml @@ -47,6 +47,30 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toBottomOf="@id/swFts" /> +