Don't store multiple repo copies in memory

app/src/main/java/com/topjohnwu/magisk/MagiskManager.java

@@ -15,15 +15,14 @@ import android.widget.Toast;
 import com.topjohnwu.magisk.database.RepoDatabaseHelper;
 import com.topjohnwu.magisk.database.SuDatabaseHelper;
 import com.topjohnwu.magisk.module.Module;
-import com.topjohnwu.magisk.module.Repo;
 import com.topjohnwu.magisk.utils.CallbackEvent;
 import com.topjohnwu.magisk.utils.SafetyNetHelper;
 import com.topjohnwu.magisk.utils.Shell;
 import com.topjohnwu.magisk.utils.Utils;
-import com.topjohnwu.magisk.utils.ValueSortedMap;
 
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 
 public class MagiskManager extends Application {
 
@@ -65,8 +64,7 @@ public class MagiskManager extends Application {
     public boolean disabled;
 
     // Data
-    public ValueSortedMap<String, Repo> repoMap;
+    public Map<String, Module> moduleMap;
-    public ValueSortedMap<String, Module> moduleMap;
     public List<String> blockList;
     public List<ApplicationInfo> appList;
     public List<String> magiskHideList;

app/src/main/java/com/topjohnwu/magisk/ReposFragment.java

@@ -13,7 +13,7 @@ import android.widget.SearchView;
 import android.widget.TextView;
 
 import com.topjohnwu.magisk.adapters.ReposAdapter;
-import com.topjohnwu.magisk.asyncs.LoadRepos;
+import com.topjohnwu.magisk.asyncs.UpdateRepos;
 import com.topjohnwu.magisk.components.Fragment;
 import com.topjohnwu.magisk.utils.CallbackEvent;
 import com.topjohnwu.magisk.utils.Logger;
@@ -43,15 +43,14 @@ public class ReposFragment extends Fragment implements CallbackEvent.Listener<Vo
         View view = inflater.inflate(R.layout.fragment_repos, container, false);
         unbinder = ButterKnife.bind(this, view);
 
-        adapter = new ReposAdapter(getApplication().repoMap);
+        adapter = new ReposAdapter(getApplication().repoDB, getApplication().moduleMap);
-
         recyclerView.setAdapter(adapter);
 
         mSwipeRefreshLayout.setRefreshing(true);
 
         mSwipeRefreshLayout.setOnRefreshListener(() -> {
             recyclerView.setVisibility(View.GONE);
-            new LoadRepos(getActivity()).exec();
+            new UpdateRepos(getActivity()).exec();
         });
 
         if (getApplication().repoLoadDone.isTriggered) {
@@ -64,15 +63,10 @@ public class ReposFragment extends Fragment implements CallbackEvent.Listener<Vo
     @Override
     public void onTrigger(CallbackEvent<Void> event) {
         Logger.dev("ReposFragment: UI refresh triggered");
-        if (getApplication().repoMap.isEmpty()) {
+        mSwipeRefreshLayout.setRefreshing(false);
-            recyclerView.setVisibility(View.GONE);
+        adapter.notifyDBChanged();
-            emptyRv.setVisibility(View.VISIBLE);
+        recyclerView.setVisibility(adapter.getItemCount() == 0 ? View.GONE : View.VISIBLE);
-        } else {
+        emptyRv.setVisibility(adapter.getItemCount() == 0 ? View.VISIBLE : View.GONE);
-            adapter.filter(getApplication().moduleMap, "");
-            recyclerView.setVisibility(View.VISIBLE);
-            emptyRv.setVisibility(View.GONE);
-            mSwipeRefreshLayout.setRefreshing(false);
-        }
     }
 
     @Override
@@ -87,7 +81,7 @@ public class ReposFragment extends Fragment implements CallbackEvent.Listener<Vo
 
             @Override
             public boolean onQueryTextChange(String newText) {
-                adapter.filter(getApplication().moduleMap, newText);
+                adapter.filter(newText);
                 return false;
             }
         });

app/src/main/java/com/topjohnwu/magisk/SplashActivity.java

@@ -10,7 +10,7 @@ import android.text.TextUtils;
 
 import com.topjohnwu.magisk.asyncs.LoadApps;
 import com.topjohnwu.magisk.asyncs.LoadModules;
-import com.topjohnwu.magisk.asyncs.LoadRepos;
+import com.topjohnwu.magisk.asyncs.UpdateRepos;
 import com.topjohnwu.magisk.components.Activity;
 import com.topjohnwu.magisk.services.UpdateCheckService;
 import com.topjohnwu.magisk.utils.Utils;
@@ -48,7 +48,7 @@ public class SplashActivity extends Activity{
                 JobScheduler scheduler = (JobScheduler) getSystemService(Context.JOB_SCHEDULER_SERVICE);
                 scheduler.schedule(jobInfo);
             }
-            loadModuleTask.setCallBack(() -> new LoadRepos(this).exec());
+            loadModuleTask.setCallBack(() -> new UpdateRepos(this).exec());
         }
 
         // Now fire all async tasks

app/src/main/java/com/topjohnwu/magisk/adapters/ReposAdapter.java

@@ -2,6 +2,7 @@ package com.topjohnwu.magisk.adapters;
 
 import android.app.Activity;
 import android.content.Context;
+import android.database.Cursor;
 import android.net.Uri;
 import android.support.v7.widget.RecyclerView;
 import android.text.TextUtils;
@@ -16,14 +17,15 @@ import com.topjohnwu.magisk.R;
 import com.topjohnwu.magisk.asyncs.ProcessRepoZip;
 import com.topjohnwu.magisk.components.AlertDialogBuilder;
 import com.topjohnwu.magisk.components.MarkDownWindow;
+import com.topjohnwu.magisk.database.RepoDatabaseHelper;
 import com.topjohnwu.magisk.module.Module;
 import com.topjohnwu.magisk.module.Repo;
 import com.topjohnwu.magisk.receivers.DownloadReceiver;
 import com.topjohnwu.magisk.utils.Utils;
-import com.topjohnwu.magisk.utils.ValueSortedMap;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import butterknife.BindView;
 import butterknife.ButterKnife;
@@ -36,10 +38,13 @@ public class ReposAdapter extends RecyclerView.Adapter<RecyclerView.ViewHolder>
     private List<Repo> mUpdateRepos, mInstalledRepos, mOthersRepos;
     private int[] sectionList;
     private int size;
-    private ValueSortedMap<String, Repo> repoMap;
+    private Cursor repoCursor = null;
+    private Map<String, Module> moduleMap;
+    private RepoDatabaseHelper repoDB;
 
-    public ReposAdapter(ValueSortedMap<String, Repo> map) {
+    public ReposAdapter(RepoDatabaseHelper db, Map<String, Module> map) {
-        repoMap = map;
+        repoDB = db;
+        moduleMap = map;
         mUpdateRepos = new ArrayList<>();
         mInstalledRepos = new ArrayList<>();
         mOthersRepos = new ArrayList<>();
@@ -137,12 +142,20 @@ public class ReposAdapter extends RecyclerView.Adapter<RecyclerView.ViewHolder>
         return size;
     }
 
-    public void filter(ValueSortedMap<String, Module> moduleMap, String s) {
+    public void notifyDBChanged() {
+        if (repoCursor != null)
+            repoCursor.close();
+        repoCursor = repoDB.getRepoCursor();
+        filter("");
+    }
+
+    public void filter(String s) {
         mUpdateRepos.clear();
         mInstalledRepos.clear();
         mOthersRepos.clear();
         sectionList[0] = sectionList[1] = sectionList[2] = 0;
-        for (Repo repo : repoMap.values()) {
+        while (repoCursor.moveToNext()) {
+            Repo repo = new Repo(repoCursor);
             if (repo.getName().toLowerCase().contains(s.toLowerCase())
                     || repo.getAuthor().toLowerCase().contains(s.toLowerCase())
                     || repo.getDescription().toLowerCase().contains(s.toLowerCase())
@@ -161,6 +174,7 @@ public class ReposAdapter extends RecyclerView.Adapter<RecyclerView.ViewHolder>
                 }
             }
         }
+        repoCursor.moveToFirst();
 
         sectionList[0] = mUpdateRepos.isEmpty() ? -1 : 0;
         size = mUpdateRepos.isEmpty() ? 0 : mUpdateRepos.size() + 1;

app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java

@@ -9,7 +9,6 @@ import com.topjohnwu.magisk.database.RepoDatabaseHelper;
 import com.topjohnwu.magisk.module.BaseModule;
 import com.topjohnwu.magisk.module.Repo;
 import com.topjohnwu.magisk.utils.Logger;
-import com.topjohnwu.magisk.utils.ValueSortedMap;
 import com.topjohnwu.magisk.utils.WebService;
 
 import org.json.JSONArray;
@@ -25,7 +24,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-public class LoadRepos extends ParallelTask<Void, Void, Void> {
+public class UpdateRepos extends ParallelTask<Void, Void, Void> {
 
     public static final String ETAG_KEY = "ETag";
 
@@ -38,11 +37,11 @@ public class LoadRepos extends ParallelTask<Void, Void, Void> {
     private static final int LOAD_PREV = 2;
 
     private List<String> etags;
-    private ValueSortedMap<String, Repo> cached, fetched;
+    private List<String> cached;
     private RepoDatabaseHelper repoDB;
     private SharedPreferences prefs;
 
-    public LoadRepos(Context context) {
+    public UpdateRepos(Context context) {
         super(context);
         prefs = getMagiskManager().prefs;
         repoDB = getMagiskManager().repoDB;
@@ -68,25 +67,27 @@ public class LoadRepos extends ParallelTask<Void, Void, Void> {
             String lastUpdate = jsonobject.getString("pushed_at");
             SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
             Date updatedDate = format.parse(lastUpdate);
-            Repo repo = cached.get(id);
+            Repo repo = repoDB.getRepo(id);
             try {
+                Boolean updated;
                 if (repo == null) {
-                    Logger.dev("LoadRepos: Create new repo " + id);
+                    Logger.dev("UpdateRepos: Create new repo " + id);
                     repo = new Repo(name, updatedDate);
+                    updated = true;
                 } else {
                     // Popout from cached
                     cached.remove(id);
-                    repo.update(updatedDate);
+                    updated = repo.update(updatedDate);
                 }
-                if (repo.getId() != null) {
+                if (updated) {
-                    fetched.put(id, repo);
+                    repoDB.addRepo(repo);
                 }
             } catch (BaseModule.CacheModException ignored) {}
         }
     }
 
     private boolean loadPage(int page, String url, int mode) {
-        Logger.dev("LoadRepos: Loading page: " + (page + 1));
+        Logger.dev("UpdateRepos: Loading page: " + (page + 1));
         Map<String, String> header = new HashMap<>();
         if (mode == CHECK_ETAG && page < etags.size() && !TextUtils.isEmpty(etags.get(page))) {
             Logger.dev("ETAG: " + etags.get(page));
@@ -157,18 +158,16 @@ public class LoadRepos extends ParallelTask<Void, Void, Void> {
     protected Void doInBackground(Void... voids) {
         MagiskManager magiskManager = getMagiskManager();
         if (magiskManager == null) return null;
-        Logger.dev("LoadRepos: Loading repos");
+        Logger.dev("UpdateRepos: Loading repos");
 
-        cached = repoDB.getRepoMap(false);
+        cached = repoDB.getRepoIDList();
-        fetched = new ValueSortedMap<>();
 
         if (!loadPage(0, null, CHECK_ETAG)) {
-            magiskManager.repoMap = repoDB.getRepoMap();
+            Logger.dev("UpdateRepos: No updates, use DB");
-            Logger.dev("LoadRepos: No updates, use DB");
             return null;
         }
 
-        repoDB.addRepoMap(fetched);
+        // The leftover cached means they are removed from online repo
         repoDB.removeRepo(cached);
 
         // Update ETag
@@ -179,8 +178,7 @@ public class LoadRepos extends ParallelTask<Void, Void, Void> {
         }
         prefs.edit().putString(ETAG_KEY, etagBuilder.toString()).apply();
 
-        magiskManager.repoMap = repoDB.getRepoMap();
+        Logger.dev("UpdateRepos: Done");
-        Logger.dev("LoadRepos: Done");
         return null;
     }
 

app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java

@@ -7,9 +7,9 @@ import android.database.sqlite.SQLiteOpenHelper;
 
 import com.topjohnwu.magisk.module.Repo;
 import com.topjohnwu.magisk.utils.Logger;
-import com.topjohnwu.magisk.utils.ValueSortedMap;
 
-import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
 
 public class RepoDatabaseHelper extends SQLiteOpenHelper {
 
@@ -45,37 +45,39 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         }
     }
 
-    public void addRepoMap(ValueSortedMap<String, Repo> map) {
-        Collection<Repo> list = map.values();
-        for (Repo repo : list) {
-            Logger.dev("Add to DB: " + repo.getId());
-            mDb.replace(TABLE_NAME, null, repo.getContentValues());
-        }
-    }
-
     public void clearRepo() {
         mDb.delete(TABLE_NAME, null, null);
     }
 
-    public void removeRepo(ValueSortedMap<String, Repo> map) {
+    public void removeRepo(List<String> list) {
-        Collection<Repo> list = map.values();
+        for (String id : list) {
-        for (Repo repo : list) {
+            Logger.dev("Remove from DB: " + id);
-            Logger.dev("Remove from DB: " + repo.getId());
+            mDb.delete(TABLE_NAME, "id=?", new String[] { id });
-            mDb.delete(TABLE_NAME, "id=?", new String[] { repo.getId() });
         }
     }
 
-    public ValueSortedMap<String, Repo> getRepoMap() {
+    public void addRepo(Repo repo) {
-        return getRepoMap(true);
+        mDb.replace(TABLE_NAME, null, repo.getContentValues());
     }
 
-    public ValueSortedMap<String, Repo> getRepoMap(boolean filtered) {
+    public Repo getRepo(String id) {
-        ValueSortedMap<String, Repo> ret = new ValueSortedMap<>();
+        try (Cursor c = mDb.query(TABLE_NAME, null, "id=?", new String[] { id }, null, null, null)) {
-        Repo repo;
+            if (c.moveToNext()) {
-        try (Cursor c = mDb.query(TABLE_NAME, null, "template>=?", new String[] { filtered ? String.valueOf(MIN_TEMPLATE_VER) : "0" }, null, null, null)) {
+                return new Repo(c);
+            }
+        }
+        return null;
+    }
+
+    public Cursor getRepoCursor() {
+        return mDb.query(TABLE_NAME, null, "template>=?", new String[] { String.valueOf(MIN_TEMPLATE_VER) }, null, null, "name COLLATE NOCASE");
+    }
+
+    public List<String> getRepoIDList() {
+        LinkedList<String> ret = new LinkedList<>();
+        try (Cursor c = mDb.query(TABLE_NAME, null, null, null, null, null, null)) {
             while (c.moveToNext()) {
-                repo = new Repo(c);
+                ret.add(c.getString(c.getColumnIndex("id")));
-                ret.put(repo.getId(), repo);
             }
         }
         return ret;

app/src/main/java/com/topjohnwu/magisk/module/Repo.java

@@ -35,11 +35,13 @@ public class Repo extends BaseModule {
         Logger.dev("Repo: Fetching prop: " + getId());
     }
 
-    public void update(Date lastUpdate) throws CacheModException {
+    public boolean update(Date lastUpdate) throws CacheModException {
         if (lastUpdate.after(mLastUpdate)) {
             mLastUpdate = lastUpdate;
             update();
+            return true;
         }
+        return false;
     }
 
     public ContentValues getContentValues() {

app/src/main/java/com/topjohnwu/magisk/utils/Utils.java

@@ -27,9 +27,8 @@ import android.widget.Toast;
 import com.topjohnwu.magisk.MagiskManager;
 import com.topjohnwu.magisk.R;
 import com.topjohnwu.magisk.SplashActivity;
-import com.topjohnwu.magisk.asyncs.LoadRepos;
+import com.topjohnwu.magisk.asyncs.UpdateRepos;
 import com.topjohnwu.magisk.components.SnackbarMaker;
-import com.topjohnwu.magisk.database.RepoDatabaseHelper;
 import com.topjohnwu.magisk.receivers.DownloadReceiver;
 import com.topjohnwu.magisk.receivers.ManagerUpdate;
 
@@ -142,7 +141,7 @@ public class Utils {
 
     public static void clearRepoCache(Context context) {
         MagiskManager magiskManager = getMagiskManager(context);
-        magiskManager.prefs.edit().remove(LoadRepos.ETAG_KEY).apply();
+        magiskManager.prefs.edit().remove(UpdateRepos.ETAG_KEY).apply();
         magiskManager.repoDB.clearRepo();
         magiskManager.toast(R.string.repo_cache_cleared, Toast.LENGTH_SHORT);
     }