Optimize repo updates

src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java

@@ -7,6 +7,7 @@ import com.topjohnwu.magisk.ReposFragment;
 import com.topjohnwu.magisk.container.Repo;
 import com.topjohnwu.magisk.utils.Const;
 import com.topjohnwu.magisk.utils.Logger;
+import com.topjohnwu.magisk.utils.Utils;
 import com.topjohnwu.magisk.utils.WebService;
 
 import org.json.JSONArray;
@@ -20,6 +21,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -33,7 +35,7 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
     private static final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
 
     private MagiskManager mm;
-    private List<String> cached, etags, newEtags = new ArrayList<>();
+    private List<String> cached, etags, newEtags = new LinkedList<>();
     private boolean forceUpdate;
     private AtomicInteger taskCount = new AtomicInteger(0);
     final private Object allDone = new Object();
@@ -92,6 +94,7 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
             String id = rawRepo.getString("description");
             String name = rawRepo.getString("name");
             Date date = dateFormat.parse(rawRepo.getString("pushed_at"));
+            final List<String> c = cached;
             queueTask(() -> {
                 Repo repo = mm.repoDB.getRepo(id);
                 Boolean updated;
@@ -101,7 +104,9 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
                         updated = true;
                     } else {
                         // Popout from cached
-                        cached.remove(id);
+                        synchronized (c) {
+                            c.remove(id);
+                        }
                         if (forceUpdate) {
                             repo.update();
                             updated = true;
@@ -114,7 +119,7 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
                         publishProgress();
                     }
                     if (!id.equals(repo.getId())) {
-                        Logger.error("Repo [" + name + "] id=[" + repo.getId() + "] has illegal repo id");
+                        Logger.error("Repo [" + name + "] rid=[" + id + "] id=[" + repo.getId() + "] mismatch");
                     }
                 } catch (Repo.IllegalRepoException e) {
                     Logger.error(e.getMessage());
@@ -126,19 +131,16 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
 
     private boolean loadPage(int page, int mode) {
         Map<String, String> header = new HashMap<>();
-        String etag = "";
+        String etag = (mode == CHECK_ETAG && page < etags.size()) ? etags.get(page) : "";
-        if (mode == CHECK_ETAG && page < etags.size()) {
-            etag = etags.get(page);
-        }
         header.put(Const.Key.IF_NONE_MATCH, etag);
-        String url = String.format(Locale.US, Const.Url.REPO_URL, page + 1);
+        String url = Utils.fmt(Const.Url.REPO_URL, page + 1);
         HttpURLConnection conn = WebService.request(url, header);
 
         try {
             if (conn == null)
                 throw new Exception();
             if (conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
-                newEtags.add(etag);
+                // Current page is not updated, check the next page
                 return page + 1 < etags.size() && loadPage(page + 1, CHECK_ETAG);
             }
             loadJSON(WebService.getString(conn));
@@ -148,10 +150,17 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
             return true;
         }
 
+        /* If one page is updated, we force update all pages */
+
         // Update ETAG
         etag = header.get(Const.Key.ETAG_KEY);
         etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1);
+        if (mode == LOAD_PREV) {
+            // We are loading a previous page, push the new tag to the front
+            newEtags.add(0, etag);
+        } else {
             newEtags.add(etag);
+        }
 
         String links = header.get(Const.Key.LINK_KEY);
         if (links != null) {
@@ -159,7 +168,8 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
                 if (mode != LOAD_PREV && s.contains("next")) {
                     // Force load all next pages
                     loadPage(page + 1, LOAD_NEXT);
-                } else if (mode != LOAD_NEXT && s.contains("prev")) {
+                }
+                if (mode != LOAD_NEXT && s.contains("prev")) {
                     // Back propagation
                     loadPage(page - 1, LOAD_PREV);
                 }

src/main/java/com/topjohnwu/magisk/container/Module.java

@@ -13,7 +13,7 @@ public class Module extends BaseModule {
     public Module(String path) {
 
         try {
-            parseProps(Shell.Sync.su("cat " + path + "/module.prop"));
+            parseProps(Shell.Sync.su("dos2unix <  " + path + "/module.prop"));
         } catch (NumberFormatException ignored) {}
 
         mRemoveFile = new SuFile(path + "/remove", true);

src/main/java/com/topjohnwu/magisk/container/Repo.java

@@ -5,6 +5,8 @@ import android.database.Cursor;
 
 import com.topjohnwu.magisk.MagiskManager;
 import com.topjohnwu.magisk.utils.Const;
+import com.topjohnwu.magisk.utils.Logger;
+import com.topjohnwu.magisk.utils.Utils;
 import com.topjohnwu.magisk.utils.WebService;
 
 import java.text.DateFormat;
@@ -28,10 +30,9 @@ public class Repo extends BaseModule {
     }
 
     public void update() throws IllegalRepoException {
-        String props = WebService.getString(getManifestUrl());
+        String props[] = Utils.dos2unix(WebService.getString(getManifestUrl())).split("\\n");
-        String lines[] = props.split("\\n");
         try {
-            parseProps(lines);
+            parseProps(props);
         } catch (NumberFormatException e) {
             throw new IllegalRepoException("Repo [" + repoName + "] parse error: " + e.getMessage());
         }
@@ -43,7 +44,7 @@ public class Repo extends BaseModule {
             throw new IllegalRepoException("Repo [" + repoName + "] does not contain versionCode");
         }
         if (getMinMagiskVersion() < Const.MIN_MODULE_VER()) {
-            throw new IllegalRepoException("Repo [" + repoName + "] is outdated");
+            Logger.error("Repo [" + repoName + "] is outdated");
         }
     }
 

src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java

@@ -103,8 +103,8 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
             case Const.Value.ORDER_DATE:
                 orderBy = "last_update DESC";
         }
-        return mDb.query(TABLE_NAME, null, "minMagisk<=?",
+        return mDb.query(TABLE_NAME, null, "minMagisk<=? AND minMagisk>=?",
-                new String[] { String.valueOf(mm.magiskVersionCode) },
+                new String[] { String.valueOf(mm.magiskVersionCode), String.valueOf(Const.MIN_MODULE_VER()) },
                 null, null, orderBy);
     }
 

src/main/java/com/topjohnwu/magisk/utils/Const.java

@@ -39,7 +39,7 @@ public class Const {
     public static final int SNET_VER = 7;
 
     public static int MIN_MODULE_VER() {
-        return MagiskManager.get().magiskVersionCode >= 1630 ? 1500 : 1400;
+        return MagiskManager.get().magiskVersionCode >= MAGISK_VER.REMOVE_LEGACY_LINK ? 1500 : 1400;
     }
 
     public synchronized static SuFile MAGISK_PATH() {
@@ -78,6 +78,7 @@ public class Const {
         public static final int DTBO_SUPPORT = 1446;
         public static final int LEGACY_GLOBAL_DB = 1450;
         public static final int HIDDEN_PATH = 1460;
+        public static final int REMOVE_LEGACY_LINK = 1630;
         public static final int SEPOL_REFACTOR = 1640;
     }
 

src/main/java/com/topjohnwu/magisk/utils/Utils.java

@@ -288,4 +288,13 @@ public class Utils {
     public static String fmt(String fmt, Object... args) {
         return String.format(Locale.US, fmt, args);
     }
+
+    public static String dos2unix(String s) {
+        String newString = s.replace("\r\n", "\n");
+        if(!newString.endsWith("\n")) {
+            return newString + "\n";
+        } else {
+            return newString;
+        }
+    }
 }

src/main/java/com/topjohnwu/magisk/utils/WebService.java

@@ -24,13 +24,14 @@ public class WebService {
         try {
             StringBuilder builder = new StringBuilder();
             if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
-                BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+                try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
                     int len;
                     char buf[] = new char[4096];
                     while ((len = br.read(buf)) != -1) {
                         builder.append(buf, 0, len);
                     }
                 }
+            }
             conn.disconnect();
             return builder.toString();
         } catch (IOException e) {