From 6a9e39c470bbdae4f5de65663880c61b4ea9eb35 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Wed, 26 Apr 2017 00:14:01 +0800 Subject: [PATCH] Support unlimited amount of repos --- .../topjohnwu/magisk/SettingsActivity.java | 1 - .../topjohnwu/magisk/asyncs/LoadRepos.java | 211 ++++++++++++------ .../magisk/database/RepoDatabaseHelper.java | 11 +- .../com/topjohnwu/magisk/module/Repo.java | 5 +- .../topjohnwu/magisk/utils/WebService.java | 53 +---- app/src/main/jni/zipadjust.c | 1 - 6 files changed, 156 insertions(+), 126 deletions(-) diff --git a/app/src/main/java/com/topjohnwu/magisk/SettingsActivity.java b/app/src/main/java/com/topjohnwu/magisk/SettingsActivity.java index 542c482ce..bd4b0a1ff 100644 --- a/app/src/main/java/com/topjohnwu/magisk/SettingsActivity.java +++ b/app/src/main/java/com/topjohnwu/magisk/SettingsActivity.java @@ -7,7 +7,6 @@ import android.preference.PreferenceCategory; import android.preference.PreferenceFragment; import android.preference.PreferenceManager; import android.preference.PreferenceScreen; -import android.preference.SwitchPreference; import android.support.v7.app.ActionBar; import android.support.v7.widget.Toolbar; import android.view.WindowManager; diff --git a/app/src/main/java/com/topjohnwu/magisk/asyncs/LoadRepos.java b/app/src/main/java/com/topjohnwu/magisk/asyncs/LoadRepos.java index d15721269..827f4e289 100644 --- a/app/src/main/java/com/topjohnwu/magisk/asyncs/LoadRepos.java +++ b/app/src/main/java/com/topjohnwu/magisk/asyncs/LoadRepos.java @@ -12,14 +12,15 @@ import com.topjohnwu.magisk.utils.ValueSortedMap; import com.topjohnwu.magisk.utils.WebService; import org.json.JSONArray; -import org.json.JSONException; import org.json.JSONObject; import java.io.File; -import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,94 +28,156 @@ public class LoadRepos extends ParallelTask { public static final String ETAG_KEY = "ETag"; - private static final String REPO_URL = "https://api.github.com/orgs/Magisk-Modules-Repo/repos"; + private static final String REPO_URL = "https://api.github.com/users/Magisk-Modules-Repo/repos?per_page=100&page=%d"; + private static final String IF_NONE_MATCH = "If-None-Match"; + private static final String LINK_KEY = "Link"; - private String prefsPath; + private static final int CHECK_ETAG = 0; + private static final int LOAD_NEXT = 1; + private static final int LOAD_PREV = 2; + + private List etags; + private ValueSortedMap cached, fetched; + private RepoDatabaseHelper repoDB; + private SharedPreferences prefs; public LoadRepos(Activity context) { super(context); - prefsPath = context.getApplicationInfo().dataDir + "/shared_prefs"; + prefs = magiskManager.prefs; + String prefsPath = context.getApplicationInfo().dataDir + "/shared_prefs"; + repoDB = new RepoDatabaseHelper(magiskManager); + // Legacy data cleanup + File old = new File(prefsPath, "RepoMap.xml"); + if (old.exists() || !prefs.getString("repomap", "empty").equals("empty")) { + old.delete(); + prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply(); + repoDB.clearRepo(); + } + etags = new ArrayList<>( + Arrays.asList(magiskManager.prefs.getString(ETAG_KEY, "").split(","))); + } + + private void loadJSON(String jsonString) throws Exception { + JSONArray jsonArray = new JSONArray(jsonString); + + for (int i = 0; i < jsonArray.length(); i++) { + JSONObject jsonobject = jsonArray.getJSONObject(i); + String id = jsonobject.getString("description"); + String name = jsonobject.getString("name"); + String lastUpdate = jsonobject.getString("pushed_at"); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); + Date updatedDate = format.parse(lastUpdate); + Repo repo = cached.get(id); + try { + if (repo == null) { + Logger.dev("LoadRepos: Create new repo " + id); + repo = new Repo(name, updatedDate); + } else { + // Popout from cached + cached.remove(id); + repo.update(updatedDate); + } + if (repo.getId() != null) { + fetched.put(id, repo); + } + } catch (BaseModule.CacheModException ignored) {} + } + } + + private boolean loadPage(int page, String url, int mode) { + Logger.dev("LoadRepos: Loading page: " + (page + 1)); + Map header = new HashMap<>(); + if (mode == CHECK_ETAG && page < etags.size() && !TextUtils.isEmpty(etags.get(page))) { + Logger.dev("ETAG: " + etags.get(page)); + header.put(IF_NONE_MATCH, etags.get(page)); + } + if (url == null) { + url = String.format(Locale.US, REPO_URL, page + 1); + } + String jsonString = WebService.request(url, WebService.GET, header, true); + if (TextUtils.isEmpty(jsonString)) { + // At least check the pages we know + return page + 1 < etags.size() && loadPage(page + 1, null, CHECK_ETAG); + } + + // The request succeed, parse the new stuffs + try { + loadJSON(jsonString); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + + // Update the ETAG + String newEtag = header.get(ETAG_KEY); + newEtag = newEtag.substring(newEtag.indexOf('\"'), newEtag.lastIndexOf('\"') + 1); + Logger.dev("New ETAG: " + newEtag); + if (page < etags.size()) { + etags.set(page, newEtag); + } else { + etags.add(newEtag); + } + + String links = header.get(LINK_KEY); + if (links != null) { + if (mode == CHECK_ETAG || mode == LOAD_NEXT) { + // Try to check next page URL + url = null; + for (String s : links.split(", ")) { + if (s.contains("next")) { + url = s.substring(s.indexOf("<") + 1, s.indexOf(">; ")); + break; + } + } + if (url != null) { + loadPage(page + 1, url, LOAD_NEXT); + } + } + + if (mode == CHECK_ETAG || mode == LOAD_PREV) { + // Try to check prev page URL + url = null; + for (String s : links.split(", ")) { + if (s.contains("prev")) { + url = s.substring(s.indexOf("<") + 1, s.indexOf(">; ")); + break; + } + } + if (url != null) { + loadPage(page - 1, url, LOAD_PREV); + } + } + } + + return true; } @Override protected Void doInBackground(Void... voids) { Logger.dev("LoadRepos: Loading repos"); - SharedPreferences prefs = magiskManager.prefs; + cached = repoDB.getRepoMap(false); + fetched = new ValueSortedMap<>(); - RepoDatabaseHelper dbHelper = new RepoDatabaseHelper(magiskManager); - - // Legacy data cleanup - File old = new File(prefsPath, "RepoMap.xml"); - if (old.exists() || !prefs.getString("repomap", "empty").equals("empty")) { - old.delete(); - prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply(); - dbHelper.clearRepo(); + if (!loadPage(0, null, CHECK_ETAG)) { + magiskManager.repoMap = repoDB.getRepoMap(); + Logger.dev("LoadRepos: No updates, use DB"); + return null; } - Map header = new HashMap<>(); - // Get cached ETag to add in the request header - String etag = prefs.getString(ETAG_KEY, ""); - header.put("If-None-Match", etag); + repoDB.addRepoMap(fetched); + repoDB.removeRepo(cached); - // Make a request to main URL for repo info - String jsonString = WebService.request(REPO_URL, WebService.GET, null, header, false); - - ValueSortedMap cached = dbHelper.getRepoMap(false), fetched = new ValueSortedMap<>(); - - if (!TextUtils.isEmpty(jsonString)) { - try { - JSONArray jsonArray = new JSONArray(jsonString); - // If it gets to this point, the response is valid, update ETag - etag = WebService.getLastResponseHeader().get(ETAG_KEY).get(0); - // Maybe bug in Android build tools, sometimes the ETag has crap in it... - etag = etag.substring(etag.indexOf('\"'), etag.lastIndexOf('\"') + 1); - - // Update repo info - for (int i = 0; i < jsonArray.length(); i++) { - JSONObject jsonobject = jsonArray.getJSONObject(i); - String id = jsonobject.getString("description"); - String name = jsonobject.getString("name"); - String lastUpdate = jsonobject.getString("pushed_at"); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); - Date updatedDate; - try { - updatedDate = format.parse(lastUpdate); - } catch (ParseException e) { - continue; - } - Repo repo = cached.get(id); - try { - if (repo == null) { - Logger.dev("LoadRepos: Create new repo " + id); - repo = new Repo(name, updatedDate); - } else { - // Popout from cached - cached.remove(id); - Logger.dev("LoadRepos: Update cached repo " + id); - repo.update(updatedDate); - } - if (repo.getId() != null) { - fetched.put(id, repo); - } - } catch (BaseModule.CacheModException ignored) {} - - // Update the database - dbHelper.addRepoMap(fetched); - // The leftover cached are those removed remote, cleanup db - dbHelper.removeRepo(cached); - // Update ETag - prefs.edit().putString(ETAG_KEY, etag).apply(); - } - - } catch (JSONException e) { - e.printStackTrace(); - } + // Update ETag + StringBuilder etagBuilder = new StringBuilder(); + for (int i = 0; i < etags.size(); ++i) { + if (i != 0) etagBuilder.append(","); + etagBuilder.append(etags.get(i)); } + prefs.edit().putString(ETAG_KEY, etagBuilder.toString()).apply(); - magiskManager.repoMap = dbHelper.getRepoMap(); - - Logger.dev("LoadRepos: Repo load done"); + magiskManager.repoMap = repoDB.getRepoMap(); + Logger.dev("LoadRepos: Done"); return null; } diff --git a/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java b/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java index d30d88be6..cdfc18bd0 100644 --- a/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java +++ b/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java @@ -15,6 +15,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { private static final int DATABASE_VER = 2; private static final String TABLE_NAME = "repos"; + private static final int MIN_TEMPLATE_VER = 3; public RepoDatabaseHelper(Context context) { super(context, "repo.db", null, DATABASE_VER); @@ -45,6 +46,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { SQLiteDatabase db = getWritableDatabase(); Collection list = map.values(); for (Repo repo : list) { + Logger.dev("Add to DB: " + repo.getId()); db.replace(TABLE_NAME, null, repo.getContentValues()); } db.close(); @@ -60,6 +62,7 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { SQLiteDatabase db = getWritableDatabase(); Collection list = map.values(); for (Repo repo : list) { + Logger.dev("Remove from DB: " + repo.getId()); db.delete(TABLE_NAME, "id=?", new String[] { repo.getId() }); } db.close(); @@ -76,12 +79,12 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { try (Cursor c = db.query(TABLE_NAME, null, null, null, null, null, null)) { while (c.moveToNext()) { repo = new Repo(c); - Logger.dev("Load from cache: " + repo.getId()); - if (repo.getTemplateVersion() < 3 && filtered) { + if (repo.getTemplateVersion() < MIN_TEMPLATE_VER && filtered) { Logger.dev("Outdated repo: " + repo.getId()); - continue; + } else { + // Logger.dev("Load from DB: " + repo.getId()); + ret.put(repo.getId(), repo); } - ret.put(repo.getId(), repo); } } db.close(); diff --git a/app/src/main/java/com/topjohnwu/magisk/module/Repo.java b/app/src/main/java/com/topjohnwu/magisk/module/Repo.java index 48ec896f1..6ab5012ca 100644 --- a/app/src/main/java/com/topjohnwu/magisk/module/Repo.java +++ b/app/src/main/java/com/topjohnwu/magisk/module/Repo.java @@ -29,14 +29,13 @@ public class Repo extends BaseModule { } public void update() throws CacheModException { - Logger.dev("Repo: Re-fetch prop"); - String props = WebService.request(getManifestUrl(), WebService.GET, true); + String props = WebService.request(getManifestUrl(), WebService.GET); String lines[] = props.split("\\n"); parseProps(lines); + Logger.dev("Repo: Fetching prop: " + getId()); } public void update(Date lastUpdate) throws CacheModException { - Logger.dev("Repo: Local: " + mLastUpdate + " Remote: " + lastUpdate); if (lastUpdate.after(mLastUpdate)) { mLastUpdate = lastUpdate; update(); diff --git a/app/src/main/java/com/topjohnwu/magisk/utils/WebService.java b/app/src/main/java/com/topjohnwu/magisk/utils/WebService.java index 99afd5536..20a4dd110 100644 --- a/app/src/main/java/com/topjohnwu/magisk/utils/WebService.java +++ b/app/src/main/java/com/topjohnwu/magisk/utils/WebService.java @@ -1,13 +1,9 @@ package com.topjohnwu.magisk.utils; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLEncoder; import java.util.List; import java.util.Map; @@ -18,8 +14,6 @@ public class WebService { public final static int GET = 1; public final static int POST = 2; - private static Map> responseHeader; - /** * Making web service call * @@ -27,11 +21,11 @@ public class WebService { * @requestmethod - http request method */ public static String request(String url, int method) { - return request(url, method, null, null, false); + return request(url, method, null, true); } public static String request(String url, int method, boolean newline) { - return request(url, method, null, null, newline); + return request(url, method, null, newline); } /** @@ -44,8 +38,7 @@ public class WebService { * @newline - true to append a newline each line */ public static String request(String urlAddress, int method, - Map params, Map header, - boolean newline) { + Map header, boolean newline) { Logger.dev("WebService: Service call " + urlAddress); URL url; StringBuilder response = new StringBuilder(); @@ -69,32 +62,6 @@ public class WebService { } } - if (params != null) { - OutputStream os = conn.getOutputStream(); - BufferedWriter writer = new BufferedWriter( - new OutputStreamWriter(os, "UTF-8")); - - StringBuilder result = new StringBuilder(); - boolean first = true; - for (Map.Entry entry : params.entrySet()) { - if (first) { - first = false; - } else { - result.append("&"); - } - - result.append(URLEncoder.encode(entry.getKey(), "UTF-8")); - result.append("="); - result.append(URLEncoder.encode(entry.getValue(), "UTF-8")); - } - - writer.write(result.toString()); - - writer.flush(); - writer.close(); - os.close(); - } - int responseCode = conn.getResponseCode(); if (responseCode == HttpsURLConnection.HTTP_OK) { @@ -107,9 +74,13 @@ public class WebService { response.append(line); } } - responseHeader = conn.getHeaderFields(); - } else { - responseHeader = null; + if (header != null) { + header.clear(); + for (Map.Entry> entry : conn.getHeaderFields().entrySet()) { + List l = entry.getValue(); + header.put(entry.getKey(), l.get(l.size() - 1)); + } + } } } catch (Exception e) { e.printStackTrace(); @@ -118,8 +89,4 @@ public class WebService { return response.toString(); } - public static Map> getLastResponseHeader() { - return responseHeader; - } - } \ No newline at end of file diff --git a/app/src/main/jni/zipadjust.c b/app/src/main/jni/zipadjust.c index 90478fe4c..7ea20985b 100644 --- a/app/src/main/jni/zipadjust.c +++ b/app/src/main/jni/zipadjust.c @@ -1,7 +1,6 @@ #include #include #include -#include #include "zipadjust.h" size_t insize = 0, outsize = 0, alloc = 0;