From bcd92499f29fecf0cb0ed332c3ee09fbdb8936be Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Sat, 14 Oct 2017 02:51:42 +0800 Subject: [PATCH] Massive improvement on Online Repo fetching --- .../com/topjohnwu/magisk/MagiskManager.java | 2 +- .../com/topjohnwu/magisk/ReposFragment.java | 2 +- .../topjohnwu/magisk/asyncs/UpdateRepos.java | 88 +++++++++++++++---- .../magisk/container/BaseModule.java | 14 ++- .../topjohnwu/magisk/container/Module.java | 4 +- .../com/topjohnwu/magisk/container/Repo.java | 34 ++++++- .../magisk/database/RepoDatabaseHelper.java | 21 ++++- 7 files changed, 128 insertions(+), 37 deletions(-) diff --git a/app/src/main/java/com/topjohnwu/magisk/MagiskManager.java b/app/src/main/java/com/topjohnwu/magisk/MagiskManager.java index 349e8d0b2..6e437fe55 100644 --- a/app/src/main/java/com/topjohnwu/magisk/MagiskManager.java +++ b/app/src/main/java/com/topjohnwu/magisk/MagiskManager.java @@ -284,7 +284,7 @@ public class MagiskManager extends Application { .setPeriodic(8 * 60 * 60 * 1000) .build(); ((JobScheduler) getSystemService(Context.JOB_SCHEDULER_SERVICE)).schedule(jobInfo); - loadModuleTask.setCallBack(() -> new UpdateRepos(this).exec()); + loadModuleTask.setCallBack(() -> new UpdateRepos(this, false).exec()); } // Fire asynctasks loadModuleTask.exec(); diff --git a/app/src/main/java/com/topjohnwu/magisk/ReposFragment.java b/app/src/main/java/com/topjohnwu/magisk/ReposFragment.java index fca67b71c..4d144439d 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ReposFragment.java +++ b/app/src/main/java/com/topjohnwu/magisk/ReposFragment.java @@ -47,7 +47,7 @@ public class ReposFragment extends Fragment implements Topic.Subscriber { mSwipeRefreshLayout.setOnRefreshListener(() -> { recyclerView.setVisibility(View.VISIBLE); emptyRv.setVisibility(View.GONE); - new UpdateRepos(getActivity()).exec(); + new UpdateRepos(getActivity(), true).exec(); }); getActivity().setTitle(R.string.downloads); diff --git a/app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java b/app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java index 3afc8ac77..17f3f8ab7 100644 --- a/app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java +++ b/app/src/main/java/com/topjohnwu/magisk/asyncs/UpdateRepos.java @@ -2,11 +2,13 @@ package com.topjohnwu.magisk.asyncs; import android.content.Context; import android.content.SharedPreferences; +import android.os.AsyncTask; import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.ReposFragment; import com.topjohnwu.magisk.container.Repo; import com.topjohnwu.magisk.database.RepoDatabaseHelper; +import com.topjohnwu.magisk.utils.Logger; import com.topjohnwu.magisk.utils.WebService; import org.json.JSONArray; @@ -38,8 +40,11 @@ public class UpdateRepos extends ParallelTask { private List cached, etags, newEtags = new ArrayList<>(); private RepoDatabaseHelper repoDB; private SharedPreferences prefs; + private boolean forceUpdate; - public UpdateRepos(Context context) { + private int tasks = 0; + + public UpdateRepos(Context context, boolean force) { super(context); prefs = getMagiskManager().prefs; repoDB = getMagiskManager().repoDB; @@ -51,6 +56,7 @@ public class UpdateRepos extends ParallelTask { prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply(); repoDB.clearRepo(); } + forceUpdate = force; } private void loadJSON(String jsonString) throws Exception { @@ -66,20 +72,37 @@ public class UpdateRepos extends ParallelTask { String lastUpdate = jsonobject.getString("pushed_at"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); Date updatedDate = format.parse(lastUpdate); - Repo repo = repoDB.getRepo(id); - Boolean updated; - if (repo == null) { - repo = new Repo(name, updatedDate); - updated = true; - } else { - // Popout from cached - cached.remove(id); - updated = repo.update(updatedDate); - } - if (updated) { - repoDB.addRepo(repo); - publishProgress(); - } + ++tasks; + AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { + Repo repo = repoDB.getRepo(id); + Boolean updated; + try { + if (repo == null) { + repo = new Repo(name, updatedDate); + updated = true; + } else { + // Popout from cached + cached.remove(id); + if (forceUpdate) { + repo.update(); + updated = true; + } else { + updated = repo.update(updatedDate); + } + } + if (updated) { + repoDB.addRepo(repo); + publishProgress(); + } + if (!id.equals(repo.getId())) { + Logger.error("Repo [" + name + "] id=[" + repo.getId() + "] has illegal repo id"); + } + } catch (Repo.IllegalRepoException e) { + Logger.error(e.getMessage()); + repoDB.removeRepo(id); + } + --tasks; + }); } } @@ -126,6 +149,17 @@ public class UpdateRepos extends ParallelTask { return true; } + private Void waitTasks() { + while (tasks > 0) { + try { + Thread.sleep(5); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + return null; + } + @Override protected void onProgressUpdate(Void... values) { if (ReposFragment.adapter != null) @@ -138,10 +172,30 @@ public class UpdateRepos extends ParallelTask { cached = repoDB.getRepoIDList(); if (!loadPage(0, CHECK_ETAG)) { - // Nothing changed - return null; + // Nothing changed online + if (forceUpdate) { + for (String id : cached) { + if (id == null) continue; + ++tasks; + AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> { + Repo repo = repoDB.getRepo(id); + try { + repo.update(); + repoDB.addRepo(repo); + } catch (Repo.IllegalRepoException e) { + Logger.error(e.getMessage()); + repoDB.removeRepo(repo); + } + --tasks; + }); + } + } + return waitTasks(); } + // Wait till all tasks are done + waitTasks(); + // The leftover cached means they are removed from online repo repoDB.removeRepo(cached); diff --git a/app/src/main/java/com/topjohnwu/magisk/container/BaseModule.java b/app/src/main/java/com/topjohnwu/magisk/container/BaseModule.java index f4e2a811a..c161a7d74 100644 --- a/app/src/main/java/com/topjohnwu/magisk/container/BaseModule.java +++ b/app/src/main/java/com/topjohnwu/magisk/container/BaseModule.java @@ -9,8 +9,8 @@ import java.util.List; public abstract class BaseModule implements Comparable { - private String mId, mName, mVersion, mAuthor, mDescription; - private int mVersionCode = 0, templateVersion = 0; + private String mId = null, mName, mVersion, mAuthor, mDescription; + private int mVersionCode = -1, templateVersion = -1; protected BaseModule() {} @@ -38,7 +38,7 @@ public abstract class BaseModule implements Comparable { protected void parseProps(List props) { parseProps(props.toArray(new String[0])); } - protected void parseProps(String[] props) { + protected void parseProps(String[] props) throws NumberFormatException { for (String line : props) { String[] prop = line.split("=", 2); if (prop.length != 2) @@ -59,9 +59,7 @@ public abstract class BaseModule implements Comparable { mVersion = prop[1]; break; case "versionCode": - try { - mVersionCode = Integer.parseInt(prop[1]); - } catch (NumberFormatException ignored) {} + mVersionCode = Integer.parseInt(prop[1]); break; case "author": mAuthor = prop[1]; @@ -70,9 +68,7 @@ public abstract class BaseModule implements Comparable { mDescription = prop[1]; break; case "template": - try { - templateVersion = Integer.parseInt(prop[1]); - } catch (NumberFormatException ignored) {} + templateVersion = Integer.parseInt(prop[1]); break; default: break; diff --git a/app/src/main/java/com/topjohnwu/magisk/container/Module.java b/app/src/main/java/com/topjohnwu/magisk/container/Module.java index cbb71a478..b171964cb 100644 --- a/app/src/main/java/com/topjohnwu/magisk/container/Module.java +++ b/app/src/main/java/com/topjohnwu/magisk/container/Module.java @@ -10,7 +10,9 @@ public class Module extends BaseModule { public Module(Shell shell, String path) { - parseProps(Utils.readFile(shell, path + "/module.prop")); + try { + parseProps(Utils.readFile(shell, path + "/module.prop")); + } catch (NumberFormatException ignored) {} mRemoveFile = path + "/remove"; mDisableFile = path + "/disable"; diff --git a/app/src/main/java/com/topjohnwu/magisk/container/Repo.java b/app/src/main/java/com/topjohnwu/magisk/container/Repo.java index 2dd460127..4e5a62200 100644 --- a/app/src/main/java/com/topjohnwu/magisk/container/Repo.java +++ b/app/src/main/java/com/topjohnwu/magisk/container/Repo.java @@ -9,13 +9,15 @@ import java.util.Date; public class Repo extends BaseModule { + public static final int MIN_TEMPLATE_VER = 4; + private static final String FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s"; private static final String ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip"; private String repoName; private Date mLastUpdate; - public Repo(String name, Date lastUpdate) { + public Repo(String name, Date lastUpdate) throws IllegalRepoException { mLastUpdate = lastUpdate; repoName = name; update(); @@ -27,13 +29,27 @@ public class Repo extends BaseModule { mLastUpdate = new Date(c.getLong(c.getColumnIndex("last_update"))); } - public void update() { + public void update() throws IllegalRepoException { String props = WebService.getString(getManifestUrl()); String lines[] = props.split("\\n"); - parseProps(lines); + try { + parseProps(lines); + } catch (NumberFormatException e) { + throw new IllegalRepoException("Repo [" + repoName + "] parse error: " + e.getMessage()); + } + + if (getId() == null) { + throw new IllegalRepoException("Repo [" + repoName + "] does not contain id"); + } + if (getVersionCode() < 0) { + throw new IllegalRepoException("Repo [" + repoName + "] does not contain versionCode"); + } + if (getTemplateVersion() < MIN_TEMPLATE_VER) { + throw new IllegalRepoException("Repo [" + repoName + "] is outdated"); + } } - public boolean update(Date lastUpdate) { + public boolean update(Date lastUpdate) throws IllegalRepoException { if (lastUpdate.after(mLastUpdate)) { mLastUpdate = lastUpdate; update(); @@ -49,6 +65,10 @@ public class Repo extends BaseModule { return values; } + public String getRepoName() { + return repoName; + } + public String getZipUrl() { return String.format(ZIP_URL, repoName); } @@ -64,4 +84,10 @@ public class Repo extends BaseModule { public Date getLastUpdate() { return mLastUpdate; } + + public class IllegalRepoException extends Exception { + IllegalRepoException(String message) { + super(message); + } + } } diff --git a/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java b/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java index c845f5849..98a4ea90d 100644 --- a/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java +++ b/app/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java @@ -7,7 +7,6 @@ import android.database.sqlite.SQLiteOpenHelper; import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.container.Repo; -import com.topjohnwu.magisk.utils.Logger; import com.topjohnwu.magisk.utils.Utils; import java.util.LinkedList; @@ -17,7 +16,6 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { private static final int DATABASE_VER = 2; private static final String TABLE_NAME = "repos"; - private static final int MIN_TEMPLATE_VER = 3; private SQLiteDatabase mDb; private MagiskManager mm; @@ -26,6 +24,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { super(context, "repo.db", null, DATABASE_VER); mDb = getWritableDatabase(); mm = Utils.getMagiskManager(context); + + // Clear bad repos + mDb.delete(TABLE_NAME, "template list) { for (String id : list) { - Logger.dev("Remove from DB: " + id); + if (id == null) continue; mDb.delete(TABLE_NAME, "id=?", new String[] { id }); } } @@ -74,7 +85,9 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper { } public Cursor getRepoCursor() { - return mDb.query(TABLE_NAME, null, "template>=? AND template<=?", new String[] { String.valueOf(MIN_TEMPLATE_VER), String.valueOf(mm.magiskVersionCode) }, null, null, "name COLLATE NOCASE"); + return mDb.query(TABLE_NAME, null, "template<=?", + new String[] { String.valueOf(mm.magiskVersionCode) }, + null, null, "name COLLATE NOCASE"); } public List getRepoIDList() {