Support unlimited amount of repos

app/src/main/java/com/topjohnwu/magisk/SettingsActivity.java

@@ -7,7 +7,6 @@ import android.preference.PreferenceCategory;
 import android.preference.PreferenceFragment;
 import android.preference.PreferenceManager;
 import android.preference.PreferenceScreen;
-import android.preference.SwitchPreference;
 import android.support.v7.app.ActionBar;
 import android.support.v7.widget.Toolbar;
 import android.view.WindowManager;

app/src/main/java/com/topjohnwu/magisk/asyncs/LoadRepos.java

@@ -12,14 +12,15 @@ import com.topjohnwu.magisk.utils.ValueSortedMap;
 import com.topjohnwu.magisk.utils.WebService;
 
 import org.json.JSONArray;
-import org.json.JSONException;
 import org.json.JSONObject;
 
 import java.io.File;
-import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
@@ -27,94 +28,156 @@ public class LoadRepos extends ParallelTask<Void, Void, Void> {
 
     public static final String ETAG_KEY = "ETag";
 
-    private static final String REPO_URL = "https://api.github.com/orgs/Magisk-Modules-Repo/repos";
+    private static final String REPO_URL = "https://api.github.com/users/Magisk-Modules-Repo/repos?per_page=100&page=%d";
+    private static final String IF_NONE_MATCH = "If-None-Match";
+    private static final String LINK_KEY = "Link";
 
-    private String prefsPath;
+    private static final int CHECK_ETAG = 0;
+    private static final int LOAD_NEXT = 1;
+    private static final int LOAD_PREV = 2;
+
+    private List<String> etags;
+    private ValueSortedMap<String, Repo> cached, fetched;
+    private RepoDatabaseHelper repoDB;
+    private SharedPreferences prefs;
 
     public LoadRepos(Activity context) {
         super(context);
-        prefsPath = context.getApplicationInfo().dataDir + "/shared_prefs";
+        prefs = magiskManager.prefs;
+        String prefsPath = context.getApplicationInfo().dataDir + "/shared_prefs";
+        repoDB = new RepoDatabaseHelper(magiskManager);
+        // Legacy data cleanup
+        File old = new File(prefsPath, "RepoMap.xml");
+        if (old.exists() || !prefs.getString("repomap", "empty").equals("empty")) {
+            old.delete();
+            prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply();
+            repoDB.clearRepo();
+        }
+        etags = new ArrayList<>(
+                Arrays.asList(magiskManager.prefs.getString(ETAG_KEY, "").split(",")));
+    }
+
+    private void loadJSON(String jsonString) throws Exception {
+        JSONArray jsonArray = new JSONArray(jsonString);
+
+        for (int i = 0; i < jsonArray.length(); i++) {
+            JSONObject jsonobject = jsonArray.getJSONObject(i);
+            String id = jsonobject.getString("description");
+            String name = jsonobject.getString("name");
+            String lastUpdate = jsonobject.getString("pushed_at");
+            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+            Date updatedDate = format.parse(lastUpdate);
+            Repo repo = cached.get(id);
+            try {
+                if (repo == null) {
+                    Logger.dev("LoadRepos: Create new repo " + id);
+                    repo = new Repo(name, updatedDate);
+                } else {
+                    // Popout from cached
+                    cached.remove(id);
+                    repo.update(updatedDate);
+                }
+                if (repo.getId() != null) {
+                    fetched.put(id, repo);
+                }
+            } catch (BaseModule.CacheModException ignored) {}
+        }
+    }
+
+    private boolean loadPage(int page, String url, int mode) {
+        Logger.dev("LoadRepos: Loading page: " + (page + 1));
+        Map<String, String> header = new HashMap<>();
+        if (mode == CHECK_ETAG && page < etags.size() && !TextUtils.isEmpty(etags.get(page))) {
+            Logger.dev("ETAG: " + etags.get(page));
+            header.put(IF_NONE_MATCH, etags.get(page));
+        }
+        if (url == null) {
+            url = String.format(Locale.US, REPO_URL, page + 1);
+        }
+        String jsonString = WebService.request(url, WebService.GET, header, true);
+        if (TextUtils.isEmpty(jsonString)) {
+            // At least check the pages we know
+            return page + 1 < etags.size() && loadPage(page + 1, null, CHECK_ETAG);
+        }
+
+        // The request succeed, parse the new stuffs
+        try {
+            loadJSON(jsonString);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+
+        // Update the ETAG
+        String newEtag = header.get(ETAG_KEY);
+        newEtag = newEtag.substring(newEtag.indexOf('\"'), newEtag.lastIndexOf('\"') + 1);
+        Logger.dev("New ETAG: " + newEtag);
+        if (page < etags.size()) {
+            etags.set(page, newEtag);
+        } else {
+            etags.add(newEtag);
+        }
+
+        String links = header.get(LINK_KEY);
+        if (links != null) {
+            if (mode == CHECK_ETAG || mode == LOAD_NEXT) {
+                // Try to check next page URL
+                url = null;
+                for (String s : links.split(", ")) {
+                    if (s.contains("next")) {
+                        url = s.substring(s.indexOf("<") + 1, s.indexOf(">; "));
+                        break;
+                    }
+                }
+                if (url != null) {
+                    loadPage(page + 1, url, LOAD_NEXT);
+                }
+            }
+
+            if (mode == CHECK_ETAG || mode == LOAD_PREV) {
+                // Try to check prev page URL
+                url = null;
+                for (String s : links.split(", ")) {
+                    if (s.contains("prev")) {
+                        url = s.substring(s.indexOf("<") + 1, s.indexOf(">; "));
+                        break;
+                    }
+                }
+                if (url != null) {
+                    loadPage(page - 1, url, LOAD_PREV);
+                }
+            }
+        }
+
+        return true;
     }
 
     @Override
     protected Void doInBackground(Void... voids) {
         Logger.dev("LoadRepos: Loading repos");
 
-        SharedPreferences prefs = magiskManager.prefs;
+        cached = repoDB.getRepoMap(false);
+        fetched = new ValueSortedMap<>();
 
-        RepoDatabaseHelper dbHelper = new RepoDatabaseHelper(magiskManager);
+        if (!loadPage(0, null, CHECK_ETAG)) {
-
+            magiskManager.repoMap = repoDB.getRepoMap();
-        // Legacy data cleanup
+            Logger.dev("LoadRepos: No updates, use DB");
-        File old = new File(prefsPath, "RepoMap.xml");
+            return null;
-        if (old.exists() || !prefs.getString("repomap", "empty").equals("empty")) {
-            old.delete();
-            prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply();
-            dbHelper.clearRepo();
         }
 
-        Map<String, String> header = new HashMap<>();
+        repoDB.addRepoMap(fetched);
-        // Get cached ETag to add in the request header
+        repoDB.removeRepo(cached);
-        String etag = prefs.getString(ETAG_KEY, "");
-        header.put("If-None-Match", etag);
 
-        // Make a request to main URL for repo info
+        // Update ETag
-        String jsonString = WebService.request(REPO_URL, WebService.GET, null, header, false);
+        StringBuilder etagBuilder = new StringBuilder();
-
+        for (int i = 0; i < etags.size(); ++i) {
-        ValueSortedMap<String, Repo> cached = dbHelper.getRepoMap(false), fetched = new ValueSortedMap<>();
+            if (i != 0) etagBuilder.append(",");
-
+            etagBuilder.append(etags.get(i));
-        if (!TextUtils.isEmpty(jsonString)) {
-            try {
-                JSONArray jsonArray = new JSONArray(jsonString);
-                // If it gets to this point, the response is valid, update ETag
-                etag = WebService.getLastResponseHeader().get(ETAG_KEY).get(0);
-                // Maybe bug in Android build tools, sometimes the ETag has crap in it...
-                etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1);
-
-                // Update repo info
-                for (int i = 0; i < jsonArray.length(); i++) {
-                    JSONObject jsonobject = jsonArray.getJSONObject(i);
-                    String id = jsonobject.getString("description");
-                    String name = jsonobject.getString("name");
-                    String lastUpdate = jsonobject.getString("pushed_at");
-                    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
-                    Date updatedDate;
-                    try {
-                        updatedDate = format.parse(lastUpdate);
-                    } catch (ParseException e) {
-                        continue;
-                    }
-                    Repo repo = cached.get(id);
-                    try {
-                        if (repo == null) {
-                            Logger.dev("LoadRepos: Create new repo " + id);
-                            repo = new Repo(name, updatedDate);
-                        } else {
-                            // Popout from cached
-                            cached.remove(id);
-                            Logger.dev("LoadRepos: Update cached repo " + id);
-                            repo.update(updatedDate);
-                        }
-                        if (repo.getId() != null) {
-                            fetched.put(id, repo);
-                        }
-                    } catch (BaseModule.CacheModException ignored) {}
-
-                    // Update the database
-                    dbHelper.addRepoMap(fetched);
-                    // The leftover cached are those removed remote, cleanup db
-                    dbHelper.removeRepo(cached);
-                    // Update ETag
-                    prefs.edit().putString(ETAG_KEY, etag).apply();
-                }
-
-            } catch (JSONException e) {
-                e.printStackTrace();
-            }
         }
+        prefs.edit().putString(ETAG_KEY, etagBuilder.toString()).apply();
 
-        magiskManager.repoMap = dbHelper.getRepoMap();
+        magiskManager.repoMap = repoDB.getRepoMap();
-
+        Logger.dev("LoadRepos: Done");
-        Logger.dev("LoadRepos: Repo load done");
         return null;
     }
 

app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java

@@ -15,6 +15,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
 
     private static final int DATABASE_VER = 2;
     private static final String TABLE_NAME = "repos";
+    private static final int MIN_TEMPLATE_VER = 3;
 
     public RepoDatabaseHelper(Context context) {
         super(context, "repo.db", null, DATABASE_VER);
@@ -45,6 +46,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         SQLiteDatabase db = getWritableDatabase();
         Collection<Repo> list = map.values();
         for (Repo repo : list) {
+            Logger.dev("Add to DB: " + repo.getId());
             db.replace(TABLE_NAME, null, repo.getContentValues());
         }
         db.close();
@@ -60,6 +62,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         SQLiteDatabase db = getWritableDatabase();
         Collection<Repo> list = map.values();
         for (Repo repo : list) {
+            Logger.dev("Remove from DB: " + repo.getId());
             db.delete(TABLE_NAME, "id=?", new String[] { repo.getId() });
         }
         db.close();
@@ -76,12 +79,12 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
         try (Cursor c = db.query(TABLE_NAME, null, null, null, null, null, null)) {
             while (c.moveToNext()) {
                 repo = new Repo(c);
-                Logger.dev("Load from cache: " + repo.getId());
+                if (repo.getTemplateVersion() < MIN_TEMPLATE_VER && filtered) {
-                if (repo.getTemplateVersion() < 3 && filtered) {
                     Logger.dev("Outdated repo: " + repo.getId());
-                    continue;
+                } else {
+                    // Logger.dev("Load from DB: " + repo.getId());
+                    ret.put(repo.getId(), repo);
                 }
-                ret.put(repo.getId(), repo);
             }
         }
         db.close();

app/src/main/java/com/topjohnwu/magisk/module/Repo.java

@@ -29,14 +29,13 @@ public class Repo extends BaseModule {
     }
 
     public void update() throws CacheModException {
-        Logger.dev("Repo: Re-fetch prop");
+        String props = WebService.request(getManifestUrl(), WebService.GET);
-        String props = WebService.request(getManifestUrl(), WebService.GET, true);
         String lines[] = props.split("\\n");
         parseProps(lines);
+        Logger.dev("Repo: Fetching prop: " + getId());
     }
 
     public void update(Date lastUpdate) throws CacheModException {
-        Logger.dev("Repo: Local: " + mLastUpdate + " Remote: " + lastUpdate);
         if (lastUpdate.after(mLastUpdate)) {
             mLastUpdate = lastUpdate;
             update();

app/src/main/java/com/topjohnwu/magisk/utils/WebService.java

@@ -1,13 +1,9 @@
 package com.topjohnwu.magisk.utils;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.net.HttpURLConnection;
 import java.net.URL;
-import java.net.URLEncoder;
 import java.util.List;
 import java.util.Map;
 
@@ -18,8 +14,6 @@ public class WebService {
     public final static int GET = 1;
     public final static int POST = 2;
 
-    private static Map<String, List<String>> responseHeader;
-
     /**
      * Making web service call
      *
@@ -27,11 +21,11 @@ public class WebService {
      * @requestmethod - http request method
      */
     public static String request(String url, int method) {
-        return request(url, method, null, null, false);
+        return request(url, method, null, true);
     }
 
     public static String request(String url, int method, boolean newline) {
-        return request(url, method, null, null, newline);
+        return request(url, method, null, newline);
     }
 
     /**
@@ -44,8 +38,7 @@ public class WebService {
      * @newline - true to append a newline each line
      */
     public static String request(String urlAddress, int method,
-                                 Map<String, String> params, Map<String, String> header,
+                                 Map<String, String> header, boolean newline) {
-                                 boolean newline) {
         Logger.dev("WebService: Service call " + urlAddress);
         URL url;
         StringBuilder response = new StringBuilder();
@@ -69,32 +62,6 @@ public class WebService {
                 }
             }
 
-            if (params != null) {
-                OutputStream os = conn.getOutputStream();
-                BufferedWriter writer = new BufferedWriter(
-                        new OutputStreamWriter(os, "UTF-8"));
-
-                StringBuilder result = new StringBuilder();
-                boolean first = true;
-                for (Map.Entry<String, String> entry : params.entrySet()) {
-                    if (first) {
-                        first = false;
-                    } else {
-                        result.append("&");
-                    }
-
-                    result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
-                    result.append("=");
-                    result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
-                }
-
-                writer.write(result.toString());
-
-                writer.flush();
-                writer.close();
-                os.close();
-            }
-
             int responseCode = conn.getResponseCode();
 
             if (responseCode == HttpsURLConnection.HTTP_OK) {
@@ -107,9 +74,13 @@ public class WebService {
                         response.append(line);
                     }
                 }
-                responseHeader = conn.getHeaderFields();
+                if (header != null) {
-            } else {
+                    header.clear();
-                responseHeader = null;
+                    for (Map.Entry<String, List<String>> entry : conn.getHeaderFields().entrySet()) {
+                        List<String> l = entry.getValue();
+                        header.put(entry.getKey(), l.get(l.size() - 1));
+                    }
+                }
             }
         } catch (Exception e) {
             e.printStackTrace();
@@ -118,8 +89,4 @@ public class WebService {
         return response.toString();
     }
 
-    public static Map<String, List<String>> getLastResponseHeader() {
-        return responseHeader;
-    }
-
 }

app/src/main/jni/zipadjust.c

@@ -1,7 +1,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <zlib.h>
-#include <string.h>
 #include "zipadjust.h"
 
 size_t insize = 0, outsize = 0, alloc = 0;