diff --git a/app/src/main/java/com/topjohnwu/magisk/Const.kt b/app/src/main/java/com/topjohnwu/magisk/Const.kt index 2b8a8fede..51bcbeba3 100644 --- a/app/src/main/java/com/topjohnwu/magisk/Const.kt +++ b/app/src/main/java/com/topjohnwu/magisk/Const.kt @@ -59,6 +59,9 @@ object Const { } object Url { + @Deprecated("This shouldn't be used. There's literally no need for it") + const val REPO_URL = + "https://api.github.com/users/Magisk-Modules-Repo/repos?per_page=100&sort=pushed&page=%d" const val FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s" const val ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip" const val MODULE_INSTALLER = diff --git a/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDatabaseHelper.java b/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDatabaseHelper.java new file mode 100644 index 000000000..6e4cb35d0 --- /dev/null +++ b/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDatabaseHelper.java @@ -0,0 +1,118 @@ +package com.topjohnwu.magisk.data.database; + +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; + +import com.topjohnwu.magisk.Config; +import com.topjohnwu.magisk.model.entity.Repo; + +import java.util.HashSet; +import java.util.Set; + +@Deprecated +public class RepoDatabaseHelper extends SQLiteOpenHelper { + + private static final int DATABASE_VER = 5; + private static final String TABLE_NAME = "repos"; + + private final SQLiteDatabase mDb; + + @Deprecated + public RepoDatabaseHelper(Context context) { + super(context, "repo.db", null, DATABASE_VER); + mDb = getWritableDatabase(); + } + + @Override + public void onCreate(SQLiteDatabase db) { + onUpgrade(db, 0, DATABASE_VER); + } + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + if (oldVersion != newVersion) { + // Nuke old DB and create new table + db.execSQL("DROP TABLE IF EXISTS " + TABLE_NAME); + db.execSQL( + "CREATE TABLE IF NOT EXISTS " + TABLE_NAME + " " + + "(id TEXT, name TEXT, version TEXT, versionCode INT, " + + "author TEXT, description TEXT, last_update INT, PRIMARY KEY(id))"); + Config.remove(Config.Key.ETAG_KEY); + } + } + + @Override + public void onDowngrade(SQLiteDatabase db, int oldVersion, int newVersion) { + onUpgrade(db, 0, DATABASE_VER); + } + + @Deprecated + public void clearRepo() { + mDb.delete(TABLE_NAME, null, null); + } + + + @Deprecated + public void removeRepo(String id) { + mDb.delete(TABLE_NAME, "id=?", new String[]{id}); + } + + @Deprecated + public void removeRepo(Repo repo) { + removeRepo(repo.getId()); + } + + @Deprecated + public void removeRepo(Iterable list) { + for (String id : list) { + if (id == null) continue; + mDb.delete(TABLE_NAME, "id=?", new String[]{id}); + } + } + + @Deprecated + public void addRepo(Repo repo) { + mDb.replace(TABLE_NAME, null, repo.getContentValues()); + } + + @Deprecated + public Repo getRepo(String id) { + try (Cursor c = mDb.query(TABLE_NAME, null, "id=?", new String[]{id}, null, null, null)) { + if (c.moveToNext()) { + return new Repo(c); + } + } + return null; + } + + @Deprecated + public Cursor getRawCursor() { + return mDb.query(TABLE_NAME, null, null, null, null, null, null); + } + + @Deprecated + public Cursor getRepoCursor() { + String orderBy = null; + switch ((int) Config.get(Config.Key.REPO_ORDER)) { + case Config.Value.ORDER_NAME: + orderBy = "name COLLATE NOCASE"; + break; + case Config.Value.ORDER_DATE: + orderBy = "last_update DESC"; + } + return mDb.query(TABLE_NAME, null, null, null, null, null, orderBy); + } + + @Deprecated + public Set getRepoIDSet() { + HashSet set = new HashSet<>(300); + try (Cursor c = mDb.query(TABLE_NAME, null, null, null, null, null, null)) { + while (c.moveToNext()) { + set.add(c.getString(c.getColumnIndex("id"))); + } + } + return set; + } +} diff --git a/app/src/main/java/com/topjohnwu/magisk/di/DatabaseModule.kt b/app/src/main/java/com/topjohnwu/magisk/di/DatabaseModule.kt index 9a07cac38..218329d24 100644 --- a/app/src/main/java/com/topjohnwu/magisk/di/DatabaseModule.kt +++ b/app/src/main/java/com/topjohnwu/magisk/di/DatabaseModule.kt @@ -3,6 +3,7 @@ package com.topjohnwu.magisk.di import android.content.Context import androidx.room.Room import com.topjohnwu.magisk.data.database.* +import com.topjohnwu.magisk.tasks.UpdateRepos import org.koin.dsl.module @@ -13,6 +14,8 @@ val databaseModule = module { single { SettingsDao() } single { StringDao() } single { createRepositoryDao(get()) } + single { RepoDatabaseHelper(get()) } + single { UpdateRepos(get()) } } fun createDatabase(context: Context): AppDatabase = diff --git a/app/src/main/java/com/topjohnwu/magisk/di/ViewModelsModule.kt b/app/src/main/java/com/topjohnwu/magisk/di/ViewModelsModule.kt index 941176a48..4cfac9682 100644 --- a/app/src/main/java/com/topjohnwu/magisk/di/ViewModelsModule.kt +++ b/app/src/main/java/com/topjohnwu/magisk/di/ViewModelsModule.kt @@ -18,7 +18,7 @@ val viewModelModules = module { viewModel { HomeViewModel(get()) } viewModel { SuperuserViewModel(get(), get(), get(), get()) } viewModel { HideViewModel(get(), get()) } - viewModel { ModuleViewModel(get(), get()) } + viewModel { ModuleViewModel(get(), get(), get(), get()) } viewModel { LogViewModel(get(), get()) } viewModel { (action: String, uri: Uri?) -> FlashViewModel(action, uri, get()) } viewModel { SuRequestViewModel(get(), get(), get(SUTimeout), get()) } diff --git a/app/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java b/app/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java new file mode 100644 index 000000000..4098bb497 --- /dev/null +++ b/app/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java @@ -0,0 +1,185 @@ +package com.topjohnwu.magisk.tasks; + +import android.database.Cursor; +import android.util.Pair; + +import com.topjohnwu.magisk.App; +import com.topjohnwu.magisk.Config; +import com.topjohnwu.magisk.Const; +import com.topjohnwu.magisk.data.database.RepoDatabaseHelper; +import com.topjohnwu.magisk.model.entity.Repo; +import com.topjohnwu.magisk.utils.Logger; +import com.topjohnwu.magisk.utils.Utils; +import com.topjohnwu.net.Networking; +import com.topjohnwu.net.Request; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.HttpURLConnection; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.Locale; +import java.util.Queue; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import androidx.annotation.NonNull; +import io.reactivex.Single; + +@Deprecated +public class UpdateRepos { + private static final DateFormat DATE_FORMAT; + + static { + DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); + DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + @NonNull + private final RepoDatabaseHelper repoDB; + private Set cached; + private Queue> moduleQueue; + + public UpdateRepos(@NonNull RepoDatabaseHelper repoDatabase) { + repoDB = repoDatabase; + } + + private void runTasks(Runnable task) { + Future[] futures = new Future[App.THREAD_POOL.getMaximumPoolSize() - 1]; + for (int i = 0; i < futures.length; ++i) { + futures[i] = App.THREAD_POOL.submit(task); + } + for (Future f : futures) { + while (true) { + try { + f.get(); + } catch (InterruptedException e) { + continue; + } catch (ExecutionException ignored) { + } + break; + } + } + } + + /* We sort repos by last push, which means that we only need to check whether the + * first page is updated to determine whether the online repo database is changed + */ + private boolean parsePage(int page) { + Request req = Networking.get(Utils.fmt(Const.Url.REPO_URL, page + 1)); + if (page == 0) { + String etag = Config.get(Config.Key.ETAG_KEY); + if (etag != null) + req.addHeaders(Const.Key.IF_NONE_MATCH, etag); + } + Request.Result res = req.execForJSONArray(); + // JSON not updated + if (res.getCode() == HttpURLConnection.HTTP_NOT_MODIFIED) + return false; + // Network error + if (res.getResult() == null) { + cached.clear(); + return true; + } + // Current page is the last page + if (res.getResult().length() == 0) + return true; + + try { + for (int i = 0; i < res.getResult().length(); i++) { + JSONObject rawRepo = res.getResult().getJSONObject(i); + String id = rawRepo.getString("name"); + Date date = DATE_FORMAT.parse(rawRepo.getString("pushed_at")); + moduleQueue.offer(new Pair<>(id, date)); + } + } catch (JSONException | ParseException e) { + // Should not happen, but if exception occurs, page load fails + return false; + } + + // Update ETAG + if (page == 0) { + String etag = res.getConnection().getHeaderField(Config.Key.ETAG_KEY); + if (etag != null) { + etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1); + Config.set(Config.Key.ETAG_KEY, etag); + } + } + + String links = res.getConnection().getHeaderField(Const.Key.LINK_KEY); + return links == null || !links.contains("next") || parsePage(page + 1); + } + + private boolean loadPages() { + if (!parsePage(0)) + return false; + runTasks(() -> { + while (true) { + Pair pair = moduleQueue.poll(); + if (pair == null) + return; + Repo repo = repoDB.getRepo(pair.first); + try { + if (repo == null) + repo = new Repo(pair.first); + else + cached.remove(pair.first); + repo.update(pair.second); + repoDB.addRepo(repo); + } catch (Repo.IllegalRepoException e) { + Logger.debug(e.getMessage()); + repoDB.removeRepo(pair.first); + } + } + }); + return true; + } + + private void fullReload() { + Cursor c = repoDB.getRawCursor(); + runTasks(() -> { + while (true) { + Repo repo; + synchronized (c) { + if (!c.moveToNext()) + return; + repo = new Repo(c); + } + try { + repo.update(); + repoDB.addRepo(repo); + } catch (Repo.IllegalRepoException e) { + Logger.debug(e.getMessage()); + repoDB.removeRepo(repo); + } + } + }); + } + + public Single exec(boolean force) { + return Single.fromCallable(() -> { + cached = Collections.synchronizedSet(repoDB.getRepoIDSet()); + moduleQueue = new ConcurrentLinkedQueue<>(); + + if (loadPages()) { + // The leftover cached means they are removed from online repo + repoDB.removeRepo(cached); + } else if (force) { + fullReload(); + } + return force; // not important + }); + } + + public Single exec() { + return exec(false); + } +} diff --git a/app/src/main/java/com/topjohnwu/magisk/ui/SplashActivity.kt b/app/src/main/java/com/topjohnwu/magisk/ui/SplashActivity.kt index 1cc17e500..0d48f53fb 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ui/SplashActivity.kt +++ b/app/src/main/java/com/topjohnwu/magisk/ui/SplashActivity.kt @@ -5,11 +5,15 @@ import android.os.Bundle import android.text.TextUtils import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity +import com.skoumal.teanity.extensions.subscribeK import com.topjohnwu.magisk.* +import com.topjohnwu.magisk.tasks.UpdateRepos import com.topjohnwu.magisk.utils.Utils import com.topjohnwu.magisk.view.Notifications import com.topjohnwu.magisk.view.Shortcuts +import com.topjohnwu.net.Networking import com.topjohnwu.superuser.Shell +import org.koin.android.ext.android.get open class SplashActivity : AppCompatActivity() { @@ -62,8 +66,9 @@ open class SplashActivity : AppCompatActivity() { // Load modules //Utils.loadModules(false) // Load repos - //if (Networking.checkNetworkStatus(this)) - //UpdateRepos().exec() + if (Networking.checkNetworkStatus(this)) { + get().exec().subscribeK() + } } val intent = Intent(this, ClassMap[MainActivity::class.java]) diff --git a/app/src/main/java/com/topjohnwu/magisk/ui/base/BasePreferenceFragment.kt b/app/src/main/java/com/topjohnwu/magisk/ui/base/BasePreferenceFragment.kt index e56abb119..23b2d3b6c 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ui/base/BasePreferenceFragment.kt +++ b/app/src/main/java/com/topjohnwu/magisk/ui/base/BasePreferenceFragment.kt @@ -15,6 +15,7 @@ import com.topjohnwu.magisk.App import com.topjohnwu.magisk.Config import com.topjohnwu.magisk.KConfig import com.topjohnwu.magisk.R +import com.topjohnwu.magisk.data.database.RepoDatabaseHelper import com.topjohnwu.magisk.data.repository.ModuleRepository import com.topjohnwu.magisk.data.repository.SettingRepository import org.koin.android.ext.android.inject @@ -22,6 +23,7 @@ import org.koin.android.ext.android.inject abstract class BasePreferenceFragment : PreferenceFragmentCompat(), SharedPreferences.OnSharedPreferenceChangeListener { + protected val repoDatabase: RepoDatabaseHelper by inject() protected val prefs: SharedPreferences by inject() protected val app: App by inject() protected val settingRepo: SettingRepository by inject() diff --git a/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt b/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt index 120237215..76df5c536 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt +++ b/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt @@ -1,24 +1,30 @@ package com.topjohnwu.magisk.ui.module import android.content.res.Resources +import android.database.Cursor import com.skoumal.teanity.databinding.ComparableRvItem import com.skoumal.teanity.extensions.addOnPropertyChangedCallback +import com.skoumal.teanity.extensions.doOnSuccessUi import com.skoumal.teanity.extensions.subscribeK import com.skoumal.teanity.util.DiffObservableList import com.skoumal.teanity.util.KObservableField import com.topjohnwu.magisk.BR import com.topjohnwu.magisk.R +import com.topjohnwu.magisk.data.database.RepoDatabaseHelper import com.topjohnwu.magisk.data.repository.ModuleRepository +import com.topjohnwu.magisk.model.entity.Repo import com.topjohnwu.magisk.model.entity.recycler.ModuleRvItem import com.topjohnwu.magisk.model.entity.recycler.RepoRvItem import com.topjohnwu.magisk.model.entity.recycler.SectionRvItem import com.topjohnwu.magisk.model.events.InstallModuleEvent import com.topjohnwu.magisk.model.events.OpenChangelogEvent import com.topjohnwu.magisk.model.events.OpenFilePickerEvent +import com.topjohnwu.magisk.tasks.UpdateRepos import com.topjohnwu.magisk.ui.base.MagiskViewModel import com.topjohnwu.magisk.utils.toSingle import com.topjohnwu.magisk.utils.update import com.topjohnwu.magisk.utils.zip +import io.reactivex.Single import io.reactivex.android.schedulers.AndroidSchedulers import io.reactivex.disposables.Disposable import io.reactivex.schedulers.Schedulers @@ -26,7 +32,9 @@ import me.tatarka.bindingcollectionadapter2.OnItemBind class ModuleViewModel( private val resources: Resources, - private val moduleRepo: ModuleRepository + private val moduleRepo: ModuleRepository, + private val repoDatabase: RepoDatabaseHelper, + private val repoUpdater: UpdateRepos ) : MagiskViewModel() { val query = KObservableField("") @@ -54,7 +62,7 @@ class ModuleViewModel( fun repoPressed(item: RepoRvItem) = OpenChangelogEvent(item.item).publish() fun downloadPressed(item: RepoRvItem) = InstallModuleEvent(item.item).publish() - fun refresh(forceReload: Boolean) { + fun refreshNew(forceReload: Boolean) { val updateInstalled = moduleRepo.fetchInstalledModules() .flattenAsFlowable { it } .map { ModuleRvItem(it) } @@ -79,6 +87,24 @@ class ModuleViewModel( .subscribeK { itemsRemote.update(it.first, it.second) } } + fun refresh/*Old*/(force: Boolean) { + moduleRepo.fetchInstalledModules() + .flattenAsFlowable { it } + .map { ModuleRvItem(it) } + .toList() + .map { it to itemsInstalled.calculateDiff(it) } + .doOnSuccessUi { itemsInstalled.update(it.first, it.second) } + .flatMap { repoUpdater.exec() } + .flatMap { Single.fromCallable { repoDatabase.repoCursor.toList { Repo(it) } } } + .flattenAsFlowable { it } + .map { RepoRvItem(it) } + .toList() + .doOnSuccess { allItems.update(it) } + .flatMap { queryRaw() } + .applyViewModel(this) + .subscribeK { itemsRemote.update(it.first, it.second) } + } + private fun query() = queryRaw() .subscribeK { itemsRemote.update(it.first, it.second) } @@ -112,6 +138,12 @@ class ModuleViewModel( groupedItems.getOrElse(MODULE_REMOTE) { listOf() }.withTitle(R.string.not_installed) } + private fun Cursor.toList(transformer: (Cursor) -> Result): List { + val out = mutableListOf() + while (moveToNext()) out.add(transformer(this)) + return out + } + companion object { protected const val MODULE_INSTALLED = 0 protected const val MODULE_REMOTE = 1 diff --git a/app/src/main/java/com/topjohnwu/magisk/ui/settings/SettingsFragment.java b/app/src/main/java/com/topjohnwu/magisk/ui/settings/SettingsFragment.java index b8c092d9c..ad3753616 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ui/settings/SettingsFragment.java +++ b/app/src/main/java/com/topjohnwu/magisk/ui/settings/SettingsFragment.java @@ -107,12 +107,12 @@ public final class SettingsFragment extends BasePreferenceFragment { DownloadApp.restore(); return true; }); - Preference clear = findPreference("clear"); - clear.setOnPreferenceClickListener(pref -> { + findPreference("clear").setOnPreferenceClickListener(pref -> { getPrefs().edit().remove(Config.Key.ETAG_KEY).apply(); - getModuleRepo().deleteAllCached().subscribeOn(Schedulers.io()).subscribe(() -> { - }, throwable -> { - }); + getRepoDatabase().clearRepo(); + //getModuleRepo().deleteAllCached().subscribeOn(Schedulers.io()).subscribe(() -> { + //}, throwable -> { + //}); Utils.toast(R.string.repo_cache_cleared, Toast.LENGTH_SHORT); return true; });