Optimize repo update

src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java

@@ -1,12 +1,10 @@
 package com.topjohnwu.magisk.asyncs;
 
-import android.content.SharedPreferences;
 import android.os.AsyncTask;
 
 import com.topjohnwu.magisk.MagiskManager;
 import com.topjohnwu.magisk.ReposFragment;
 import com.topjohnwu.magisk.container.Repo;
-import com.topjohnwu.magisk.database.RepoDatabaseHelper;
 import com.topjohnwu.magisk.utils.Const;
 import com.topjohnwu.magisk.utils.Logger;
 import com.topjohnwu.magisk.utils.WebService;
@@ -16,6 +14,7 @@ import org.json.JSONObject;
 
 import java.io.File;
 import java.net.HttpURLConnection;
+import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -24,55 +23,81 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public class UpdateRepos extends ParallelTask<Void, Void, Void> {
 
     private static final int CHECK_ETAG = 0;
     private static final int LOAD_NEXT = 1;
     private static final int LOAD_PREV = 2;
+    private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
 
+    private MagiskManager mm;
     private List<String> cached, etags, newEtags = new ArrayList<>();
-    private RepoDatabaseHelper repoDB;
-    private SharedPreferences prefs;
     private boolean forceUpdate;
-
+    private AtomicInteger taskCount = new AtomicInteger(0);
-    private int tasks = 0;
+    final private Object allDone = new Object();
 
     public UpdateRepos(boolean force) {
-        MagiskManager mm = MagiskManager.get();
+        mm = MagiskManager.get();
-        prefs = mm.prefs;
-        repoDB = mm.repoDB;
         mm.repoLoadDone.reset();
         // Legacy data cleanup
         File old = new File(mm.getApplicationInfo().dataDir + "/shared_prefs", "RepoMap.xml");
-        if (old.exists() || prefs.getString("repomap", null) != null) {
+        if (old.exists() || mm.prefs.getString("repomap", null) != null) {
             old.delete();
-            prefs.edit().remove("version").remove("repomap").remove(Const.Key.ETAG_KEY).apply();
+            mm.prefs.edit().remove("version").remove("repomap").remove(Const.Key.ETAG_KEY).apply();
-            repoDB.clearRepo();
+            mm.repoDB.clearRepo();
         }
         forceUpdate = force;
     }
 
+    private void queueTask(Runnable task) {
+        // Thread pool's queue has an upper bound, batch it with 64 tasks
+        while (taskCount.get() >= 64) {
+            waitTasks();
+        }
+        taskCount.incrementAndGet();
+        AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
+            task.run();
+            if (taskCount.decrementAndGet() == 0) {
+                synchronized (allDone) {
+                    allDone.notify();
+                }
+            }
+        });
+    }
+
+    private void waitTasks() {
+        if (taskCount.get() == 0)
+            return;
+        synchronized (allDone) {
+            try {
+                allDone.wait();
+            } catch (InterruptedException e) {
+                // Wait again
+                waitTasks();
+            }
+        }
+    }
+
     private void loadJSON(String jsonString) throws Exception {
         JSONArray jsonArray = new JSONArray(jsonString);
 
         // Empty page, throw error
-        if (jsonArray.length() == 0) throw new Exception();
+        if (jsonArray.length() == 0)
+            throw new Exception();
 
         for (int i = 0; i < jsonArray.length(); i++) {
-            JSONObject jsonobject = jsonArray.getJSONObject(i);
+            JSONObject rawRepo = jsonArray.getJSONObject(i);
-            String id = jsonobject.getString("description");
+            String id = rawRepo.getString("description");
-            String name = jsonobject.getString("name");
+            String name = rawRepo.getString("name");
-            String lastUpdate = jsonobject.getString("pushed_at");
+            Date date = dateFormat.parse(rawRepo.getString("pushed_at"));
-            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+            queueTask(() -> {
-            Date updatedDate = format.parse(lastUpdate);
+                Repo repo = mm.repoDB.getRepo(id);
-            ++tasks;
-            AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
-                Repo repo = repoDB.getRepo(id);
                 Boolean updated;
                 try {
                     if (repo == null) {
-                        repo = new Repo(name, updatedDate);
+                        repo = new Repo(name, date);
                         updated = true;
                     } else {
                         // Popout from cached
@@ -81,11 +106,11 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
                             repo.update();
                             updated = true;
                         } else {
-                            updated = repo.update(updatedDate);
+                            updated = repo.update(date);
                         }
                     }
                     if (updated) {
-                        repoDB.addRepo(repo);
+                        mm.repoDB.addRepo(repo);
                         publishProgress();
                     }
                     if (!id.equals(repo.getId())) {
@@ -93,9 +118,8 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
                     }
                 } catch (Repo.IllegalRepoException e) {
                     Logger.error(e.getMessage());
-                    repoDB.removeRepo(id);
+                    mm.repoDB.removeRepo(id);
                 }
-                --tasks;
             });
         }
     }
@@ -111,7 +135,8 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
         HttpURLConnection conn = WebService.request(url, header);
 
         try {
-            if (conn == null) throw new Exception();
+            if (conn == null)
+                throw new Exception();
             if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
                 newEtags.add(etag);
                 return page + 1 < etags.size() && loadPage(page + 1, CHECK_ETAG);
@@ -143,17 +168,6 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
         return true;
     }
 
-    private Void waitTasks() {
-        while (tasks > 0) {
-            try {
-                Thread.sleep(5);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-        return null;
-    }
-
     @Override
     protected void onProgressUpdate(Void... values) {
         if (ReposFragment.adapter != null)
@@ -162,55 +176,52 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
 
     @Override
     protected void onPreExecute() {
-        MagiskManager.get().repoLoadDone.setPending();
+        mm.repoLoadDone.setPending();
     }
 
     @Override
     protected Void doInBackground(Void... voids) {
-        etags = new ArrayList<>(Arrays.asList(prefs.getString(Const.Key.ETAG_KEY, "").split(",")));
+        etags = new ArrayList<>(Arrays.asList(mm.prefs.getString(Const.Key.ETAG_KEY, "").split(",")));
-        cached = repoDB.getRepoIDList();
+        cached = mm.repoDB.getRepoIDList();
 
         if (!loadPage(0, CHECK_ETAG)) {
             // Nothing changed online
             if (forceUpdate) {
                 for (String id : cached) {
                     if (id == null) continue;
-                    ++tasks;
+                    queueTask(() -> {
-                    AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
+                        Repo repo = mm.repoDB.getRepo(id);
-                        Repo repo = repoDB.getRepo(id);
                         try {
                             repo.update();
-                            repoDB.addRepo(repo);
+                            mm.repoDB.addRepo(repo);
                         } catch (Repo.IllegalRepoException e) {
                             Logger.error(e.getMessage());
-                            repoDB.removeRepo(repo);
+                            mm.repoDB.removeRepo(repo);
                         }
-                        --tasks;
                     });
                 }
             }
-            return waitTasks();
+            waitTasks();
+        } else {
+            waitTasks();
+
+            // The leftover cached means they are removed from online repo
+            mm.repoDB.removeRepo(cached);
+
+            // Update ETag
+            StringBuilder etagBuilder = new StringBuilder();
+            for (int i = 0; i < newEtags.size(); ++i) {
+                if (i != 0) etagBuilder.append(",");
+                etagBuilder.append(newEtags.get(i));
+            }
+            mm.prefs.edit().putString(Const.Key.ETAG_KEY, etagBuilder.toString()).apply();
         }
-
-        // Wait till all tasks are done
-        waitTasks();
-
-        // The leftover cached means they are removed from online repo
-        repoDB.removeRepo(cached);
-
-        // Update ETag
-        StringBuilder etagBuilder = new StringBuilder();
-        for (int i = 0; i < newEtags.size(); ++i) {
-            if (i != 0) etagBuilder.append(",");
-            etagBuilder.append(newEtags.get(i));
-        }
-        prefs.edit().putString(Const.Key.ETAG_KEY, etagBuilder.toString()).apply();
         return null;
     }
 
     @Override
     protected void onPostExecute(Void v) {
-        MagiskManager.get().repoLoadDone.publish();
+        mm.repoLoadDone.publish();
         super.onPostExecute(v);
     }
 }