From 8fd03f743455219c67089b262bf7d1fc3dcd38dd Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Sun, 6 May 2018 02:51:23 +0800 Subject: [PATCH] Optimize repo updates --- .../topjohnwu/magisk/asyncs/UpdateRepos.java | 32 ++++++++++++------- .../topjohnwu/magisk/container/Module.java | 2 +- .../com/topjohnwu/magisk/container/Repo.java | 9 +++--- .../magisk/database/RepoDatabaseHelper.java | 4 +-- .../com/topjohnwu/magisk/utils/Const.java | 3 +- .../com/topjohnwu/magisk/utils/Utils.java | 9 ++++++ .../topjohnwu/magisk/utils/WebService.java | 11 ++++--- 7 files changed, 46 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java b/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java index a42bc1f6f..101462e5f 100644 --- a/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java +++ b/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java @@ -7,6 +7,7 @@ import com.topjohnwu.magisk.ReposFragment; import com.topjohnwu.magisk.container.Repo; import com.topjohnwu.magisk.utils.Const; import com.topjohnwu.magisk.utils.Logger; +import com.topjohnwu.magisk.utils.Utils; import com.topjohnwu.magisk.utils.WebService; import org.json.JSONArray; @@ -20,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -33,7 +35,7 @@ public class UpdateRepos extends ParallelTask { private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); private MagiskManager mm; - private List cached, etags, newEtags = new ArrayList<>(); + private List cached, etags, newEtags = new LinkedList<>(); private boolean forceUpdate; private AtomicInteger taskCount = new AtomicInteger(0); final private Object allDone = new Object(); @@ -92,6 +94,7 @@ public class UpdateRepos extends ParallelTask { String id = rawRepo.getString("description"); String name = rawRepo.getString("name"); Date date = dateFormat.parse(rawRepo.getString("pushed_at")); + final List c = cached; queueTask(() -> { Repo repo = mm.repoDB.getRepo(id); Boolean updated; @@ -101,7 +104,9 @@ public class UpdateRepos extends ParallelTask { updated = true; } else { // Popout from cached - cached.remove(id); + synchronized (c) { + c.remove(id); + } if (forceUpdate) { repo.update(); updated = true; @@ -114,7 +119,7 @@ public class UpdateRepos extends ParallelTask { publishProgress(); } if (!id.equals(repo.getId())) { - Logger.error("Repo [" + name + "] id=[" + repo.getId() + "] has illegal repo id"); + Logger.error("Repo [" + name + "] rid=[" + id + "] id=[" + repo.getId() + "] mismatch"); } } catch (Repo.IllegalRepoException e) { Logger.error(e.getMessage()); @@ -126,19 +131,16 @@ public class UpdateRepos extends ParallelTask { private boolean loadPage(int page, int mode) { Map header = new HashMap<>(); - String etag = ""; - if (mode == CHECK_ETAG && page < etags.size()) { - etag = etags.get(page); - } + String etag = (mode == CHECK_ETAG && page < etags.size()) ? etags.get(page) : ""; header.put(Const.Key.IF_NONE_MATCH, etag); - String url = String.format(Locale.US, Const.Url.REPO_URL, page + 1); + String url = Utils.fmt(Const.Url.REPO_URL, page + 1); HttpURLConnection conn = WebService.request(url, header); try { if (conn == null) throw new Exception(); if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { - newEtags.add(etag); + // Current page is not updated, check the next page return page + 1 < etags.size() && loadPage(page + 1, CHECK_ETAG); } loadJSON(WebService.getString(conn)); @@ -148,10 +150,17 @@ public class UpdateRepos extends ParallelTask { return true; } + /* If one page is updated, we force update all pages */ + // Update ETAG etag = header.get(Const.Key.ETAG_KEY); etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1); - newEtags.add(etag); + if (mode == LOAD_PREV) { + // We are loading a previous page, push the new tag to the front + newEtags.add(0, etag); + } else { + newEtags.add(etag); + } String links = header.get(Const.Key.LINK_KEY); if (links != null) { @@ -159,7 +168,8 @@ public class UpdateRepos extends ParallelTask { if (mode != LOAD_PREV && s.contains("next")) { // Force load all next pages loadPage(page + 1, LOAD_NEXT); - } else if (mode != LOAD_NEXT && s.contains("prev")) { + } + if (mode != LOAD_NEXT && s.contains("prev")) { // Back propagation loadPage(page - 1, LOAD_PREV); } diff --git a/src/main/java/com/topjohnwu/magisk/container/Module.java b/src/main/java/com/topjohnwu/magisk/container/Module.java index 59524bfdb..8caeac17c 100644 --- a/src/main/java/com/topjohnwu/magisk/container/Module.java +++ b/src/main/java/com/topjohnwu/magisk/container/Module.java @@ -13,7 +13,7 @@ public class Module extends BaseModule { public Module(String path) { try { - parseProps(Shell.Sync.su("cat " + path + "/module.prop")); + parseProps(Shell.Sync.su("dos2unix < " + path + "/module.prop")); } catch (NumberFormatException ignored) {} mRemoveFile = new SuFile(path + "/remove", true); diff --git a/src/main/java/com/topjohnwu/magisk/container/Repo.java b/src/main/java/com/topjohnwu/magisk/container/Repo.java index 734c4235e..41e41fa33 100644 --- a/src/main/java/com/topjohnwu/magisk/container/Repo.java +++ b/src/main/java/com/topjohnwu/magisk/container/Repo.java @@ -5,6 +5,8 @@ import android.database.Cursor; import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.utils.Const; +import com.topjohnwu.magisk.utils.Logger; +import com.topjohnwu.magisk.utils.Utils; import com.topjohnwu.magisk.utils.WebService; import java.text.DateFormat; @@ -28,10 +30,9 @@ public class Repo extends BaseModule { } public void update() throws IllegalRepoException { - String props = WebService.getString(getManifestUrl()); - String lines[] = props.split("\\n"); + String props[] = Utils.dos2unix(WebService.getString(getManifestUrl())).split("\\n"); try { - parseProps(lines); + parseProps(props); } catch (NumberFormatException e) { throw new IllegalRepoException("Repo [" + repoName + "] parse error: " + e.getMessage()); } @@ -43,7 +44,7 @@ public class Repo extends BaseModule { throw new IllegalRepoException("Repo [" + repoName + "] does not contain versionCode"); } if (getMinMagiskVersion() < Const.MIN_MODULE_VER()) { - throw new IllegalRepoException("Repo [" + repoName + "] is outdated"); + Logger.error("Repo [" + repoName + "] is outdated"); } } diff --git a/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java b/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java index 71795706c..442526d84 100644 --- a/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java +++ b/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java @@ -103,8 +103,8 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { case Const.Value.ORDER_DATE: orderBy = "last_update DESC"; } - return mDb.query(TABLE_NAME, null, "minMagisk<=?", - new String[] { String.valueOf(mm.magiskVersionCode) }, + return mDb.query(TABLE_NAME, null, "minMagisk<=? AND minMagisk>=?", + new String[] { String.valueOf(mm.magiskVersionCode), String.valueOf(Const.MIN_MODULE_VER()) }, null, null, orderBy); } diff --git a/src/main/java/com/topjohnwu/magisk/utils/Const.java b/src/main/java/com/topjohnwu/magisk/utils/Const.java index ec713c4b4..5393c96e1 100644 --- a/src/main/java/com/topjohnwu/magisk/utils/Const.java +++ b/src/main/java/com/topjohnwu/magisk/utils/Const.java @@ -39,7 +39,7 @@ public class Const { public static final int SNET_VER = 7; public static int MIN_MODULE_VER() { - return MagiskManager.get().magiskVersionCode >= 1630 ? 1500 : 1400; + return MagiskManager.get().magiskVersionCode >= MAGISK_VER.REMOVE_LEGACY_LINK ? 1500 : 1400; } public synchronized static SuFile MAGISK_PATH() { @@ -78,6 +78,7 @@ public class Const { public static final int DTBO_SUPPORT = 1446; public static final int LEGACY_GLOBAL_DB = 1450; public static final int HIDDEN_PATH = 1460; + public static final int REMOVE_LEGACY_LINK = 1630; public static final int SEPOL_REFACTOR = 1640; } diff --git a/src/main/java/com/topjohnwu/magisk/utils/Utils.java b/src/main/java/com/topjohnwu/magisk/utils/Utils.java index c0884a098..50b77b462 100644 --- a/src/main/java/com/topjohnwu/magisk/utils/Utils.java +++ b/src/main/java/com/topjohnwu/magisk/utils/Utils.java @@ -288,4 +288,13 @@ public class Utils { public static String fmt(String fmt, Object... args) { return String.format(Locale.US, fmt, args); } + + public static String dos2unix(String s) { + String newString = s.replace("\r\n", "\n"); + if(!newString.endsWith("\n")) { + return newString + "\n"; + } else { + return newString; + } + } } \ No newline at end of file diff --git a/src/main/java/com/topjohnwu/magisk/utils/WebService.java b/src/main/java/com/topjohnwu/magisk/utils/WebService.java index db29f44af..bc4073a34 100644 --- a/src/main/java/com/topjohnwu/magisk/utils/WebService.java +++ b/src/main/java/com/topjohnwu/magisk/utils/WebService.java @@ -24,11 +24,12 @@ public class WebService { try { StringBuilder builder = new StringBuilder(); if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) { - BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream())); - int len; - char buf[] = new char[4096]; - while ((len = br.read(buf)) != -1) { - builder.append(buf, 0, len); + try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()))) { + int len; + char buf[] = new char[4096]; + while ((len = br.read(buf)) != -1) { + builder.append(buf, 0, len); + } } } conn.disconnect();