MarcoBuster/Magisk
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Optimize repo list fetching

Browse Source
This commit is contained in:
topjohnwu 2019-02-02 04:15:30 -05:00
parent 699debdaca
commit bd3e0b9336
5 changed files with 81 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app-core/src/main/java/com/topjohnwu/magisk/database/RepoDatabaseHelper.java
View File

@ -19,7 +19,6 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
private static final String TABLE_NAME = "repos";
private SQLiteDatabase mDb;
private Runnable adapterCb;
public RepoDatabaseHelper(Context context) {
super(context, "repo.db", null, DATABASE_VER);
@ -54,13 +53,11 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
public void clearRepo() {
mDb.delete(TABLE_NAME, null, null);
notifyAdapter();
}
public void removeRepo(String id) {
mDb.delete(TABLE_NAME, "id=?", new String[] { id });
notifyAdapter();
}
public void removeRepo(Repo repo) {
@ -72,12 +69,10 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
if (id == null) continue;
mDb.delete(TABLE_NAME, "id=?", new String[] { id });
}
notifyAdapter();
}
public void addRepo(Repo repo) {
mDb.replace(TABLE_NAME, null, repo.getContentValues());
notifyAdapter();
}
public Repo getRepo(String id) {
@ -116,18 +111,4 @@ public class RepoDatabaseHelper extends SQLiteOpenHelper {
}
return set;
}
public void registerAdapterCallback(Runnable cb) {
adapterCb = cb;
}
public void unregisterAdapterCallback() {
adapterCb = null;
}
private void notifyAdapter() {
if (adapterCb != null) {
UiThreadHandler.run(adapterCb);
}
}
}

110
app-core/src/main/java/com/topjohnwu/magisk/tasks/UpdateRepos.java
View File

@ -1,6 +1,7 @@
package com.topjohnwu.magisk.tasks;
import android.database.Cursor;
import android.util.Pair;
import com.topjohnwu.magisk.App;
import com.topjohnwu.magisk.Config;
@ -23,62 +24,46 @@ import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.Locale;
import java.util.Queue;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
public class UpdateRepos {
private static final int CPU_COUNT = Runtime.getRuntime().availableProcessors();
private static final int CORE_POOL_SIZE = Math.max(2, CPU_COUNT - 1);
private static final DateFormat dateFormat;
private static final DateFormat DATE_FORMAT;
private App app = App.self;
private Set<String> cached;
private ExecutorService threadPool;
private Queue<Pair<String, Date>> moduleQueue;
static {
dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
}
private void waitTasks() {
threadPool.shutdown();
while (true) {
try {
if (threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS))
break;
} catch (InterruptedException ignored) {}
private void runTasks(Runnable task) {
Future[] futures = new Future[App.THREAD_POOL.getMaximumPoolSize() - 1];
for (int i = 0; i < futures.length; ++i) {
futures[i] = App.THREAD_POOL.submit(task);
}
}
private void loadJSON(JSONArray array) throws JSONException, ParseException {
for (int i = 0; i < array.length(); i++) {
JSONObject rawRepo = array.getJSONObject(i);
String id = rawRepo.getString("name");
Date date = dateFormat.parse(rawRepo.getString("pushed_at"));
threadPool.execute(() -> {
Repo repo = app.repoDB.getRepo(id);
for (Future f : futures) {
while (true) {
try {
if (repo == null)
repo = new Repo(id);
else
cached.remove(id);
repo.update(date);
app.repoDB.addRepo(repo);
} catch (Repo.IllegalRepoException e) {
Logger.debug(e.getMessage());
app.repoDB.removeRepo(id);
}
});
f.get();
} catch (InterruptedException e) {
continue;
} catch (ExecutionException ignored) {}
break;
}
}
}
/* We sort repos by last push, which means that we only need to check whether the
* first page is updated to determine whether the online repo database is changed
*/
private boolean loadPage(int page) {
private boolean parsePage(int page) {
Request req = Networking.get(Utils.fmt(Const.Url.REPO_URL, page + 1));
if (page == 0) {
String etag = Config.get(Config.Key.ETAG_KEY);
@ -99,7 +84,12 @@ public class UpdateRepos {
return true;
try {
loadJSON(res.getResult());
for (int i = 0; i < res.getResult().length(); i++) {
JSONObject rawRepo = res.getResult().getJSONObject(i);
String id = rawRepo.getString("name");
Date date = DATE_FORMAT.parse(rawRepo.getString("pushed_at"));
moduleQueue.offer(new Pair<>(id, date));
}
} catch (JSONException | ParseException e) {
// Should not happen, but if exception occurs, page load fails
return false;
@ -115,18 +105,44 @@ public class UpdateRepos {
}
String links = res.getConnection().getHeaderField(Const.Key.LINK_KEY);
return links == null || !links.contains("next") || loadPage(page + 1);
return links == null || !links.contains("next") || parsePage(page + 1);
}
private boolean loadPages() {
return loadPage(0);
if (!parsePage(0))
return false;
runTasks(() -> {
while (true) {
Pair<String, Date> pair = moduleQueue.poll();
if (pair == null)
return;
Repo repo = app.repoDB.getRepo(pair.first);
try {
if (repo == null)
repo = new Repo(pair.first);
else
cached.remove(pair.first);
repo.update(pair.second);
app.repoDB.addRepo(repo);
} catch (Repo.IllegalRepoException e) {
Logger.debug(e.getMessage());
app.repoDB.removeRepo(pair.first);
}
}
});
return true;
}
private void fullReload() {
Cursor c = app.repoDB.getRawCursor();
while (c.moveToNext()) {
Repo repo = new Repo(c);
threadPool.execute(() -> {
runTasks(() -> {
while (true) {
Repo repo;
synchronized (c) {
if (!c.moveToNext())
return;
repo = new Repo(c);
}
try {
repo.update();
app.repoDB.addRepo(repo);
@ -134,19 +150,17 @@ public class UpdateRepos {
Logger.debug(e.getMessage());
app.repoDB.removeRepo(repo);
}
});
}
waitTasks();
}
});
}
public void exec(boolean force) {
Topic.reset(Topic.REPO_LOAD_DONE);
App.THREAD_POOL.execute(() -> {
cached = Collections.synchronizedSet(app.repoDB.getRepoIDSet());
threadPool = Executors.newFixedThreadPool(CORE_POOL_SIZE);
moduleQueue = new ConcurrentLinkedQueue<>();
if (loadPages()) {
waitTasks();
// The leftover cached means they are removed from online repo
app.repoDB.removeRepo(cached);
} else if (force) {

4
app-core/src/main/java/com/topjohnwu/magisk/utils/Topic.java
View File

@ -87,6 +87,10 @@ public class Topic {
return true;
}
public static boolean isPublished(AutoSubscriber sub) {
return isPublished(sub.getSubscribedTopics());
}
private static class Store {
boolean published = false;
Set<Subscriber> subscribers = new HashSet<>();

34
app/src/full/java/com/topjohnwu/magisk/fragments/ReposFragment.java
View File

@ -50,11 +50,7 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
mSwipeRefreshLayout.setRefreshing(true);
recyclerView.setVisibility(View.GONE);
mSwipeRefreshLayout.setOnRefreshListener(() -> {
recyclerView.setVisibility(View.VISIBLE);
emptyRv.setVisibility(View.GONE);
new UpdateRepos().exec(true);
});
mSwipeRefreshLayout.setOnRefreshListener(() -> new UpdateRepos().exec(true));
requireActivity().setTitle(R.string.downloads);
@ -68,17 +64,21 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
@Override
public void onPublish(int topic, Object[] result) {
if (topic == Topic.MODULE_LOAD_DONE) {
adapter = new ReposAdapter(app.repoDB, (Map<String, Module>) result[0]);
app.repoDB.registerAdapterCallback(adapter::notifyDBChanged);
recyclerView.setAdapter(adapter);
recyclerView.setVisibility(View.VISIBLE);
emptyRv.setVisibility(View.GONE);
switch (topic) {
case Topic.MODULE_LOAD_DONE:
adapter = new ReposAdapter(app.repoDB, (Map<String, Module>) result[0]);
recyclerView.setAdapter(adapter);
break;
case Topic.REPO_LOAD_DONE:
if (adapter != null)
adapter.notifyDBChanged();
break;
}
if (Topic.isPublished(getSubscribedTopics())) {
if (Topic.isPublished(this)) {
mSwipeRefreshLayout.setRefreshing(false);
recyclerView.setVisibility(adapter.getItemCount() == 0 ? View.GONE : View.VISIBLE);
emptyRv.setVisibility(adapter.getItemCount() == 0 ? View.VISIBLE : View.GONE);
boolean empty = adapter.getItemCount() == 0;
recyclerView.setVisibility(empty ? View.GONE : View.VISIBLE);
emptyRv.setVisibility(empty ? View.VISIBLE : View.GONE);
}
}
@ -114,10 +114,4 @@ public class ReposFragment extends BaseFragment implements Topic.Subscriber {
}
return true;
}
@Override
public void onDestroyView() {
super.onDestroyView();
app.repoDB.unregisterAdapterCallback();
}
}

3
app/src/full/res/layout/fragment_repos.xml
View File

@ -20,8 +20,7 @@
android:gravity="center"
android:text="@string/no_modules_found"
android:textSize="20sp"
android:textStyle="italic"
android:visibility="gone" />
android:textStyle="italic" />
<androidx.recyclerview.widget.RecyclerView
android:id="@+id/recyclerView"