Massive improvement on Online Repo fetching

This commit is contained in:
topjohnwu 2017-10-14 02:51:42 +08:00
parent b2bb0d4f72
commit bcd92499f2
7 changed files with 128 additions and 37 deletions

View File

@ -284,7 +284,7 @@ public class MagiskManager extends Application {
.setPeriodic(8 * 60 * 60 * 1000) .setPeriodic(8 * 60 * 60 * 1000)
.build(); .build();
((JobScheduler) getSystemService(Context.JOB_SCHEDULER_SERVICE)).schedule(jobInfo); ((JobScheduler) getSystemService(Context.JOB_SCHEDULER_SERVICE)).schedule(jobInfo);
loadModuleTask.setCallBack(() -> new UpdateRepos(this).exec()); loadModuleTask.setCallBack(() -> new UpdateRepos(this, false).exec());
} }
// Fire asynctasks // Fire asynctasks
loadModuleTask.exec(); loadModuleTask.exec();

View File

@ -47,7 +47,7 @@ public class ReposFragment extends Fragment implements Topic.Subscriber {
mSwipeRefreshLayout.setOnRefreshListener(() -> { mSwipeRefreshLayout.setOnRefreshListener(() -> {
recyclerView.setVisibility(View.VISIBLE); recyclerView.setVisibility(View.VISIBLE);
emptyRv.setVisibility(View.GONE); emptyRv.setVisibility(View.GONE);
new UpdateRepos(getActivity()).exec(); new UpdateRepos(getActivity(), true).exec();
}); });
getActivity().setTitle(R.string.downloads); getActivity().setTitle(R.string.downloads);

View File

@ -2,11 +2,13 @@ package com.topjohnwu.magisk.asyncs;
import android.content.Context; import android.content.Context;
import android.content.SharedPreferences; import android.content.SharedPreferences;
import android.os.AsyncTask;
import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.MagiskManager;
import com.topjohnwu.magisk.ReposFragment; import com.topjohnwu.magisk.ReposFragment;
import com.topjohnwu.magisk.container.Repo; import com.topjohnwu.magisk.container.Repo;
import com.topjohnwu.magisk.database.RepoDatabaseHelper; import com.topjohnwu.magisk.database.RepoDatabaseHelper;
import com.topjohnwu.magisk.utils.Logger;
import com.topjohnwu.magisk.utils.WebService; import com.topjohnwu.magisk.utils.WebService;
import org.json.JSONArray; import org.json.JSONArray;
@ -38,8 +40,11 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
private List<String> cached, etags, newEtags = new ArrayList<>(); private List<String> cached, etags, newEtags = new ArrayList<>();
private RepoDatabaseHelper repoDB; private RepoDatabaseHelper repoDB;
private SharedPreferences prefs; private SharedPreferences prefs;
private boolean forceUpdate;
public UpdateRepos(Context context) { private int tasks = 0;
public UpdateRepos(Context context, boolean force) {
super(context); super(context);
prefs = getMagiskManager().prefs; prefs = getMagiskManager().prefs;
repoDB = getMagiskManager().repoDB; repoDB = getMagiskManager().repoDB;
@ -51,6 +56,7 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply(); prefs.edit().remove("version").remove("repomap").remove(ETAG_KEY).apply();
repoDB.clearRepo(); repoDB.clearRepo();
} }
forceUpdate = force;
} }
private void loadJSON(String jsonString) throws Exception { private void loadJSON(String jsonString) throws Exception {
@ -66,20 +72,37 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
String lastUpdate = jsonobject.getString("pushed_at"); String lastUpdate = jsonobject.getString("pushed_at");
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
Date updatedDate = format.parse(lastUpdate); Date updatedDate = format.parse(lastUpdate);
++tasks;
AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
Repo repo = repoDB.getRepo(id); Repo repo = repoDB.getRepo(id);
Boolean updated; Boolean updated;
try {
if (repo == null) { if (repo == null) {
repo = new Repo(name, updatedDate); repo = new Repo(name, updatedDate);
updated = true; updated = true;
} else { } else {
// Popout from cached // Popout from cached
cached.remove(id); cached.remove(id);
if (forceUpdate) {
repo.update();
updated = true;
} else {
updated = repo.update(updatedDate); updated = repo.update(updatedDate);
} }
}
if (updated) { if (updated) {
repoDB.addRepo(repo); repoDB.addRepo(repo);
publishProgress(); publishProgress();
} }
if (!id.equals(repo.getId())) {
Logger.error("Repo [" + name + "] id=[" + repo.getId() + "] has illegal repo id");
}
} catch (Repo.IllegalRepoException e) {
Logger.error(e.getMessage());
repoDB.removeRepo(id);
}
--tasks;
});
} }
} }
@ -126,6 +149,17 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
return true; return true;
} }
private Void waitTasks() {
while (tasks > 0) {
try {
Thread.sleep(5);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return null;
}
@Override @Override
protected void onProgressUpdate(Void... values) { protected void onProgressUpdate(Void... values) {
if (ReposFragment.adapter != null) if (ReposFragment.adapter != null)
@ -138,9 +172,29 @@ public class UpdateRepos extends ParallelTask<Void, Void, Void> {
cached = repoDB.getRepoIDList(); cached = repoDB.getRepoIDList();
if (!loadPage(0, CHECK_ETAG)) { if (!loadPage(0, CHECK_ETAG)) {
// Nothing changed // Nothing changed online
return null; if (forceUpdate) {
for (String id : cached) {
if (id == null) continue;
++tasks;
AsyncTask.THREAD_POOL_EXECUTOR.execute(() -> {
Repo repo = repoDB.getRepo(id);
try {
repo.update();
repoDB.addRepo(repo);
} catch (Repo.IllegalRepoException e) {
Logger.error(e.getMessage());
repoDB.removeRepo(repo);
} }
--tasks;
});
}
}
return waitTasks();
}
// Wait till all tasks are done
waitTasks();
// The leftover cached means they are removed from online repo // The leftover cached means they are removed from online repo
repoDB.removeRepo(cached); repoDB.removeRepo(cached);

View File

@ -9,8 +9,8 @@ import java.util.List;
public abstract class BaseModule implements Comparable<BaseModule> { public abstract class BaseModule implements Comparable<BaseModule> {
private String mId, mName, mVersion, mAuthor, mDescription; private String mId = null, mName, mVersion, mAuthor, mDescription;
private int mVersionCode = 0, templateVersion = 0; private int mVersionCode = -1, templateVersion = -1;
protected BaseModule() {} protected BaseModule() {}
@ -38,7 +38,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
protected void parseProps(List<String> props) { parseProps(props.toArray(new String[0])); } protected void parseProps(List<String> props) { parseProps(props.toArray(new String[0])); }
protected void parseProps(String[] props) { protected void parseProps(String[] props) throws NumberFormatException {
for (String line : props) { for (String line : props) {
String[] prop = line.split("=", 2); String[] prop = line.split("=", 2);
if (prop.length != 2) if (prop.length != 2)
@ -59,9 +59,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
mVersion = prop[1]; mVersion = prop[1];
break; break;
case "versionCode": case "versionCode":
try {
mVersionCode = Integer.parseInt(prop[1]); mVersionCode = Integer.parseInt(prop[1]);
} catch (NumberFormatException ignored) {}
break; break;
case "author": case "author":
mAuthor = prop[1]; mAuthor = prop[1];
@ -70,9 +68,7 @@ public abstract class BaseModule implements Comparable<BaseModule> {
mDescription = prop[1]; mDescription = prop[1];
break; break;
case "template": case "template":
try {
templateVersion = Integer.parseInt(prop[1]); templateVersion = Integer.parseInt(prop[1]);
} catch (NumberFormatException ignored) {}
break; break;
default: default:
break; break;

View File

@ -10,7 +10,9 @@ public class Module extends BaseModule {
public Module(Shell shell, String path) { public Module(Shell shell, String path) {
try {
parseProps(Utils.readFile(shell, path + "/module.prop")); parseProps(Utils.readFile(shell, path + "/module.prop"));
} catch (NumberFormatException ignored) {}
mRemoveFile = path + "/remove"; mRemoveFile = path + "/remove";
mDisableFile = path + "/disable"; mDisableFile = path + "/disable";

View File

@ -9,13 +9,15 @@ import java.util.Date;
public class Repo extends BaseModule { public class Repo extends BaseModule {
public static final int MIN_TEMPLATE_VER = 4;
private static final String FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s"; private static final String FILE_URL = "https://raw.githubusercontent.com/Magisk-Modules-Repo/%s/master/%s";
private static final String ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip"; private static final String ZIP_URL = "https://github.com/Magisk-Modules-Repo/%s/archive/master.zip";
private String repoName; private String repoName;
private Date mLastUpdate; private Date mLastUpdate;
public Repo(String name, Date lastUpdate) { public Repo(String name, Date lastUpdate) throws IllegalRepoException {
mLastUpdate = lastUpdate; mLastUpdate = lastUpdate;
repoName = name; repoName = name;
update(); update();
@ -27,13 +29,27 @@ public class Repo extends BaseModule {
mLastUpdate = new Date(c.getLong(c.getColumnIndex("last_update"))); mLastUpdate = new Date(c.getLong(c.getColumnIndex("last_update")));
} }
public void update() { public void update() throws IllegalRepoException {
String props = WebService.getString(getManifestUrl()); String props = WebService.getString(getManifestUrl());
String lines[] = props.split("\\n"); String lines[] = props.split("\\n");
try {
parseProps(lines); parseProps(lines);
} catch (NumberFormatException e) {
throw new IllegalRepoException("Repo [" + repoName + "] parse error: " + e.getMessage());
} }
public boolean update(Date lastUpdate) { if (getId() == null) {
throw new IllegalRepoException("Repo [" + repoName + "] does not contain id");
}
if (getVersionCode() < 0) {
throw new IllegalRepoException("Repo [" + repoName + "] does not contain versionCode");
}
if (getTemplateVersion() < MIN_TEMPLATE_VER) {
throw new IllegalRepoException("Repo [" + repoName + "] is outdated");
}
}
public boolean update(Date lastUpdate) throws IllegalRepoException {
if (lastUpdate.after(mLastUpdate)) { if (lastUpdate.after(mLastUpdate)) {
mLastUpdate = lastUpdate; mLastUpdate = lastUpdate;
update(); update();
@ -49,6 +65,10 @@ public class Repo extends BaseModule {
return values; return values;
} }
public String getRepoName() {
return repoName;
}
public String getZipUrl() { public String getZipUrl() {
return String.format(ZIP_URL, repoName); return String.format(ZIP_URL, repoName);
} }
@ -64,4 +84,10 @@ public class Repo extends BaseModule {
public Date getLastUpdate() { public Date getLastUpdate() {
return mLastUpdate; return mLastUpdate;
} }
public class IllegalRepoException extends Exception {
IllegalRepoException(String message) {
super(message);
}
}
} }

View File

@ -7,7 +7,6 @@ import android.database.sqlite.SQLiteOpenHelper;
import com.topjohnwu.magisk.MagiskManager; import com.topjohnwu.magisk.MagiskManager;
import com.topjohnwu.magisk.container.Repo; import com.topjohnwu.magisk.container.Repo;
import com.topjohnwu.magisk.utils.Logger;
import com.topjohnwu.magisk.utils.Utils; import com.topjohnwu.magisk.utils.Utils;
import java.util.LinkedList; import java.util.LinkedList;
@ -17,7 +16,6 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
private static final int DATABASE_VER = 2; private static final int DATABASE_VER = 2;
private static final String TABLE_NAME = "repos"; private static final String TABLE_NAME = "repos";
private static final int MIN_TEMPLATE_VER = 3;
private SQLiteDatabase mDb; private SQLiteDatabase mDb;
private MagiskManager mm; private MagiskManager mm;
@ -26,6 +24,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
super(context, "repo.db", null, DATABASE_VER); super(context, "repo.db", null, DATABASE_VER);
mDb = getWritableDatabase(); mDb = getWritableDatabase();
mm = Utils.getMagiskManager(context); mm = Utils.getMagiskManager(context);
// Clear bad repos
mDb.delete(TABLE_NAME, "template<?",
new String[] { String.valueOf(Repo.MIN_TEMPLATE_VER) });
} }
@Override @Override
@ -53,9 +55,18 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
mDb.delete(TABLE_NAME, null, null); mDb.delete(TABLE_NAME, null, null);
} }
public void removeRepo(String id) {
mDb.delete(TABLE_NAME, "id=?", new String[] { id });
}
public void removeRepo(Repo repo) {
mDb.delete(TABLE_NAME, "repo_name=?", new String[] { repo.getRepoName() });
}
public void removeRepo(List<String> list) { public void removeRepo(List<String> list) {
for (String id : list) { for (String id : list) {
Logger.dev("Remove from DB: " + id); if (id == null) continue;
mDb.delete(TABLE_NAME, "id=?", new String[] { id }); mDb.delete(TABLE_NAME, "id=?", new String[] { id });
} }
} }
@ -74,7 +85,9 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
} }
public Cursor getRepoCursor() { public Cursor getRepoCursor() {
return mDb.query(TABLE_NAME, null, "template>=? AND template<=?", new String[] { String.valueOf(MIN_TEMPLATE_VER), String.valueOf(mm.magiskVersionCode) }, null, null, "name COLLATE NOCASE"); return mDb.query(TABLE_NAME, null, "template<=?",
new String[] { String.valueOf(mm.magiskVersionCode) },
null, null, "name COLLATE NOCASE");
} }
public List<String> getRepoIDList() { public List<String> getRepoIDList() {