From 684c5d225a28de616cdffa327af2ac222d708523 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Sat, 7 Apr 2018 04:47:22 +0800 Subject: [PATCH] Optimize repo update --- .../topjohnwu/magisk/asyncs/UpdateRepos.java | 139 ++++++++++-------- 1 file changed, 75 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java b/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java index 2ec6b37d0..a42bc1f6f 100644 --- a/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java +++ b/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java @@ -1,12 +1,10 @@ package com.topjohnwu.magisk.asyncs; -import android.content.SharedPreferences; import android.os.AsyncTask; import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.ReposFragment; import com.topjohnwu.magisk.container.Repo; -import com.topjohnwu.magisk.database.RepoDatabaseHelper; import com.topjohnwu.magisk.utils.Const; import com.topjohnwu.magisk.utils.Logger; import com.topjohnwu.magisk.utils.WebService; @@ -16,6 +14,7 @@ import org.json.JSONObject; import java.io.File; import java.net.HttpURLConnection; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -24,55 +23,81 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; public class UpdateRepos extends ParallelTask { private static final int CHECK_ETAG = 0; private static final int LOAD_NEXT = 1; private static final int LOAD_PREV = 2; + private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); + private MagiskManager mm; private List cached, etags, newEtags = new ArrayList<>(); - private RepoDatabaseHelper repoDB; - private SharedPreferences prefs; private boolean forceUpdate; - - private int tasks = 0; + private AtomicInteger taskCount = new AtomicInteger(0); + final private Object allDone = new Object(); public UpdateRepos(boolean force) { - MagiskManager mm = MagiskManager.get(); - prefs = mm.prefs; - repoDB = mm.repoDB; + mm = MagiskManager.get(); mm.repoLoadDone.reset(); // Legacy data cleanup File old = new File(mm.getApplicationInfo().dataDir + "/shared_prefs", "RepoMap.xml"); - if (old.exists() || prefs.getString("repomap", null) != null) { + if (old.exists() || mm.prefs.getString("repomap", null) != null) { old.delete(); - prefs.edit().remove("version").remove("repomap").remove(Const.Key.ETAG_KEY).apply(); - repoDB.clearRepo(); + mm.prefs.edit().remove("version").remove("repomap").remove(Const.Key.ETAG_KEY).apply(); + mm.repoDB.clearRepo(); } forceUpdate = force; } + private void queueTask(Runnable task) { + // Thread pool's queue has an upper bound, batch it with 64 tasks + while (taskCount.get() >= 64) { + waitTasks(); + } + taskCount.incrementAndGet(); + AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { + task.run(); + if (taskCount.decrementAndGet() == 0) { + synchronized (allDone) { + allDone.notify(); + } + } + }); + } + + private void waitTasks() { + if (taskCount.get() == 0) + return; + synchronized (allDone) { + try { + allDone.wait(); + } catch (InterruptedException e) { + // Wait again + waitTasks(); + } + } + } + private void loadJSON(String jsonString) throws Exception { JSONArray jsonArray = new JSONArray(jsonString); // Empty page, throw error - if (jsonArray.length() == 0) throw new Exception(); + if (jsonArray.length() == 0) + throw new Exception(); for (int i = 0; i < jsonArray.length(); i++) { - JSONObject jsonobject = jsonArray.getJSONObject(i); - String id = jsonobject.getString("description"); - String name = jsonobject.getString("name"); - String lastUpdate = jsonobject.getString("pushed_at"); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); - Date updatedDate = format.parse(lastUpdate); - ++tasks; - AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { - Repo repo = repoDB.getRepo(id); + JSONObject rawRepo = jsonArray.getJSONObject(i); + String id = rawRepo.getString("description"); + String name = rawRepo.getString("name"); + Date date = dateFormat.parse(rawRepo.getString("pushed_at")); + queueTask(() -> { + Repo repo = mm.repoDB.getRepo(id); Boolean updated; try { if (repo == null) { - repo = new Repo(name, updatedDate); + repo = new Repo(name, date); updated = true; } else { // Popout from cached @@ -81,11 +106,11 @@ public class UpdateRepos extends ParallelTask { repo.update(); updated = true; } else { - updated = repo.update(updatedDate); + updated = repo.update(date); } } if (updated) { - repoDB.addRepo(repo); + mm.repoDB.addRepo(repo); publishProgress(); } if (!id.equals(repo.getId())) { @@ -93,9 +118,8 @@ public class UpdateRepos extends ParallelTask { } } catch (Repo.IllegalRepoException e) { Logger.error(e.getMessage()); - repoDB.removeRepo(id); + mm.repoDB.removeRepo(id); } - --tasks; }); } } @@ -111,7 +135,8 @@ public class UpdateRepos extends ParallelTask { HttpURLConnection conn = WebService.request(url, header); try { - if (conn == null) throw new Exception(); + if (conn == null) + throw new Exception(); if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { newEtags.add(etag); return page + 1 < etags.size() && loadPage(page + 1, CHECK_ETAG); @@ -143,17 +168,6 @@ public class UpdateRepos extends ParallelTask { return true; } - private Void waitTasks() { - while (tasks > 0) { - try { - Thread.sleep(5); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - return null; - } - @Override protected void onProgressUpdate(Void... values) { if (ReposFragment.adapter != null) @@ -162,55 +176,52 @@ public class UpdateRepos extends ParallelTask { @Override protected void onPreExecute() { - MagiskManager.get().repoLoadDone.setPending(); + mm.repoLoadDone.setPending(); } @Override protected Void doInBackground(Void... voids) { - etags = new ArrayList<>(Arrays.asList(prefs.getString(Const.Key.ETAG_KEY, "").split(","))); - cached = repoDB.getRepoIDList(); + etags = new ArrayList<>(Arrays.asList(mm.prefs.getString(Const.Key.ETAG_KEY, "").split(","))); + cached = mm.repoDB.getRepoIDList(); if (!loadPage(0, CHECK_ETAG)) { // Nothing changed online if (forceUpdate) { for (String id : cached) { if (id == null) continue; - ++tasks; - AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { - Repo repo = repoDB.getRepo(id); + queueTask(() -> { + Repo repo = mm.repoDB.getRepo(id); try { repo.update(); - repoDB.addRepo(repo); + mm.repoDB.addRepo(repo); } catch (Repo.IllegalRepoException e) { Logger.error(e.getMessage()); - repoDB.removeRepo(repo); + mm.repoDB.removeRepo(repo); } - --tasks; }); } } - return waitTasks(); + waitTasks(); + } else { + waitTasks(); + + // The leftover cached means they are removed from online repo + mm.repoDB.removeRepo(cached); + + // Update ETag + StringBuilder etagBuilder = new StringBuilder(); + for (int i = 0; i < newEtags.size(); ++i) { + if (i != 0) etagBuilder.append(","); + etagBuilder.append(newEtags.get(i)); + } + mm.prefs.edit().putString(Const.Key.ETAG_KEY, etagBuilder.toString()).apply(); } - - // Wait till all tasks are done - waitTasks(); - - // The leftover cached means they are removed from online repo - repoDB.removeRepo(cached); - - // Update ETag - StringBuilder etagBuilder = new StringBuilder(); - for (int i = 0; i < newEtags.size(); ++i) { - if (i != 0) etagBuilder.append(","); - etagBuilder.append(newEtags.get(i)); - } - prefs.edit().putString(Const.Key.ETAG_KEY, etagBuilder.toString()).apply(); return null; } @Override protected void onPostExecute(Void v) { - MagiskManager.get().repoLoadDone.publish(); + mm.repoLoadDone.publish(); super.onPostExecute(v); } }