From 311c1f0dfdb29c92a6c8b5bd51d5b5f18f19370b Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Sat, 10 Oct 2020 14:31:30 -0700 Subject: [PATCH] Switch to new repo format --- .../java/com/topjohnwu/magisk/core/Const.kt | 1 + .../topjohnwu/magisk/core/SplashActivity.kt | 4 +- .../topjohnwu/magisk/core/download/Subject.kt | 2 +- .../topjohnwu/magisk/core/model/UpdateInfo.kt | 16 +++ .../magisk/core/model/module/Repo.kt | 53 ++++---- .../magisk/core/tasks/RepoUpdater.kt | 124 ++++-------------- .../topjohnwu/magisk/data/database/RepoDao.kt | 30 +---- .../magisk/data/network/NetworkServices.kt | 22 +--- .../magisk/data/repository/NetworkService.kt | 14 +- .../topjohnwu/magisk/di/NetworkingModule.kt | 4 +- .../com/topjohnwu/magisk/events/ViewEvents.kt | 4 +- .../magisk/ui/module/ModuleViewModel.kt | 6 +- build.gradle.kts | 2 +- 13 files changed, 93 insertions(+), 189 deletions(-) diff --git a/app/src/main/java/com/topjohnwu/magisk/core/Const.kt b/app/src/main/java/com/topjohnwu/magisk/core/Const.kt index c4cab04a6..03bb644ab 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/Const.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/Const.kt @@ -54,6 +54,7 @@ object Const { const val GITHUB_API_URL = "https://api.github.com/" const val GITHUB_PAGE_URL = "https://topjohnwu.github.io/magisk_files/" const val JS_DELIVR_URL = "https://cdn.jsdelivr.net/gh/" + const val OFFICIAL_REPO = "https://magisk-modules-repo.github.io/submission/modules.json" } object Key { diff --git a/app/src/main/java/com/topjohnwu/magisk/core/SplashActivity.kt b/app/src/main/java/com/topjohnwu/magisk/core/SplashActivity.kt index bb48ddc85..e9c9cb0cb 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/SplashActivity.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/SplashActivity.kt @@ -5,7 +5,7 @@ import android.content.Context import android.os.Bundle import com.topjohnwu.magisk.BuildConfig import com.topjohnwu.magisk.R -import com.topjohnwu.magisk.data.network.GithubRawServices +import com.topjohnwu.magisk.data.network.RawServices import com.topjohnwu.magisk.ktx.get import com.topjohnwu.magisk.ui.MainActivity import com.topjohnwu.magisk.view.Notifications @@ -55,7 +55,7 @@ open class SplashActivity : Activity() { Shortcuts.setupDynamic(this) // Pre-fetch network stuffs - get() + get() DONE = true diff --git a/app/src/main/java/com/topjohnwu/magisk/core/download/Subject.kt b/app/src/main/java/com/topjohnwu/magisk/core/download/Subject.kt index 6e61a10a5..332605987 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/download/Subject.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/download/Subject.kt @@ -29,7 +29,7 @@ sealed class Subject : Parcelable { val module: Repo, override val action: Action ) : Subject() { - override val url: String get() = module.zipUrl + override val url: String get() = module.zip_url override val title: String get() = module.downloadFilename @IgnoredOnParcel diff --git a/app/src/main/java/com/topjohnwu/magisk/core/model/UpdateInfo.kt b/app/src/main/java/com/topjohnwu/magisk/core/model/UpdateInfo.kt index f058fafe6..51c88f842 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/model/UpdateInfo.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/model/UpdateInfo.kt @@ -42,6 +42,22 @@ data class StubJson( val link: String = "" ) : Parcelable +@JsonClass(generateAdapter = true) +data class ModuleJson( + val id: String, + val last_update: Long, + val prop_url: String, + val zip_url: String, + val notes_url: String +) + +@JsonClass(generateAdapter = true) +data class RepoJson( + val name: String, + val last_update: Long, + val modules: List +) + @JsonClass(generateAdapter = true) data class CommitInfo( val sha: String diff --git a/app/src/main/java/com/topjohnwu/magisk/core/model/module/Repo.kt b/app/src/main/java/com/topjohnwu/magisk/core/model/module/Repo.kt index 0e7c249ef..605cc9475 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/model/module/Repo.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/model/module/Repo.kt @@ -3,7 +3,7 @@ package com.topjohnwu.magisk.core.model.module import android.os.Parcelable import androidx.room.Entity import androidx.room.PrimaryKey -import com.topjohnwu.magisk.core.Const +import com.topjohnwu.magisk.core.model.ModuleJson import com.topjohnwu.magisk.data.repository.NetworkService import com.topjohnwu.magisk.ktx.get import com.topjohnwu.magisk.ktx.legalFilename @@ -15,34 +15,40 @@ import java.util.* @Parcelize data class Repo( @PrimaryKey override var id: String, - override var name: String, - override var author: String, - override var version: String, - override var versionCode: Int, - override var description: String, - var last_update: Long + override var name: String = "", + override var author: String = "", + override var version: String = "", + override var versionCode: Int = -1, + override var description: String = "", + val last_update: Long, + val prop_url: String, + val zip_url: String, + val notes_url: String ) : BaseModule(), Parcelable { private val svc: NetworkService get() = get() + constructor(info: ModuleJson) : this( + id = info.id, + last_update = info.last_update, + prop_url = info.prop_url, + zip_url = info.zip_url, + notes_url = info.notes_url + ) + val lastUpdate get() = Date(last_update) + val lastUpdateString get() = DATE_FORMAT.format(lastUpdate) + val downloadFilename get() = "$name-$version($versionCode).zip".legalFilename() - val lastUpdateString: String get() = dateFormat.format(lastUpdate) - - val downloadFilename: String get() = "$name-$version($versionCode).zip".legalFilename() - - suspend fun readme() = svc.fetchReadme(this) - - val zipUrl: String get() = Const.Url.ZIP_URL.format(id) - - constructor(id: String) : this(id, "", "", "", -1, "", 0) + suspend fun notes() = svc.fetchString(notes_url) @Throws(IllegalRepoException::class) - private fun loadProps(props: String) { + suspend fun load() { + val props = svc.fetchString(prop_url) props.split("\\n".toRegex()).dropLastWhile { it.isEmpty() }.runCatching { parseProps(this) }.onFailure { - throw IllegalRepoException("Repo [$id] parse error: " + it.message) + throw IllegalRepoException("Repo [$id] parse error: ", it) } if (versionCode < 0) { @@ -50,15 +56,10 @@ data class Repo( } } - @Throws(IllegalRepoException::class) - suspend fun update(lastUpdate: Date? = null) { - lastUpdate?.let { last_update = it.time } - loadProps(svc.fetchMetadata(this)) - } - - class IllegalRepoException(message: String) : Exception(message) + class IllegalRepoException(msg: String, cause: Throwable? = null) : Exception(msg, cause) companion object { - val dateFormat = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM) + private val DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM) } + } diff --git a/app/src/main/java/com/topjohnwu/magisk/core/tasks/RepoUpdater.kt b/app/src/main/java/com/topjohnwu/magisk/core/tasks/RepoUpdater.kt index e6d41337d..4cf564168 100644 --- a/app/src/main/java/com/topjohnwu/magisk/core/tasks/RepoUpdater.kt +++ b/app/src/main/java/com/topjohnwu/magisk/core/tasks/RepoUpdater.kt @@ -1,119 +1,41 @@ package com.topjohnwu.magisk.core.tasks -import com.squareup.moshi.JsonClass -import com.topjohnwu.magisk.core.Const import com.topjohnwu.magisk.core.model.module.Repo import com.topjohnwu.magisk.data.database.RepoDao import com.topjohnwu.magisk.data.repository.NetworkService import com.topjohnwu.magisk.ktx.synchronized -import kotlinx.coroutines.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import timber.log.Timber -import java.net.HttpURLConnection -import java.text.SimpleDateFormat import java.util.* -import kotlin.collections.HashSet class RepoUpdater( private val svc: NetworkService, private val repoDB: RepoDao ) { - private fun String.trimEtag() = substring(indexOf('\"'), lastIndexOf('\"') + 1) - - private suspend fun forcedReload(cached: MutableSet) = coroutineScope { - cached.forEach { - launch { - val repo = repoDB.getRepo(it)!! - try { - repo.update() - repoDB.addRepo(repo) - } catch (e: Repo.IllegalRepoException) { - Timber.e(e) - } - } - } - } - - private suspend fun loadRepos( - repos: List, - cached: MutableSet - ) = coroutineScope { - repos.forEach { - // Skip submission - if (it.id == "submission") - return@forEach - launch { - val repo = repoDB.getRepo(it.id)?.apply { cached.remove(it.id) } ?: Repo(it.id) - try { - repo.update(it.pushDate) - repoDB.addRepo(repo) - } catch (e: Repo.IllegalRepoException) { - Timber.e(e) - } - } - } - } - - private enum class PageResult { - SUCCESS, - CACHED, - ERROR - } - - private suspend fun loadPage( - cached: MutableSet, - page: Int = 1, - etag: String = "" - ): PageResult = coroutineScope { - runCatching { - val result = svc.fetchRepos(page, etag) - result.run { - if (code() == HttpURLConnection.HTTP_NOT_MODIFIED) - return@coroutineScope PageResult.CACHED - - if (!isSuccessful) - return@coroutineScope PageResult.ERROR - - if (page == 1) - repoDB.etagKey = headers()[Const.Key.ETAG_KEY].orEmpty().trimEtag() - - val repoLoad = async { loadRepos(body()!!, cached) } - val next = if (headers()[Const.Key.LINK_KEY].orEmpty().contains("next")) { - async { loadPage(cached, page + 1) } - } else { - async { PageResult.SUCCESS } - } - repoLoad.await() - return@coroutineScope next.await() - } - }.getOrElse { - Timber.e(it) - PageResult.ERROR - } - } - suspend fun run(forced: Boolean) = withContext(Dispatchers.IO) { - val cached = HashSet(repoDB.repoIDList).synchronized() - when (loadPage(cached, etag = repoDB.etagKey)) { - PageResult.CACHED -> if (forced) forcedReload(cached) - PageResult.SUCCESS -> repoDB.removeRepos(cached) - PageResult.ERROR -> Unit + val cachedMap = HashMap().also { map -> + repoDB.getRepoStubs().forEach { map[it.id] = Date(it.last_update) } + }.synchronized() + val info = svc.fetchRepoInfo() + coroutineScope { + info.modules.forEach { + launch { + val lastUpdated = cachedMap.remove(it.id) + if (forced || lastUpdated?.before(Date(it.last_update)) != false) { + try { + val repo = Repo(it).apply { load() } + repoDB.addRepo(repo) + } catch (e: Repo.IllegalRepoException) { + Timber.e(e) + } + } + } + } } + repoDB.removeRepos(cachedMap.keys) } } - -private val dateFormat: SimpleDateFormat = - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - -@JsonClass(generateAdapter = true) -data class GithubRepoInfo( - val name: String, - val pushed_at: String -) { - val id get() = name - - @Transient - val pushDate = dateFormat.parse(pushed_at)!! -} diff --git a/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDao.kt b/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDao.kt index 23953ad9b..f6e7b917e 100644 --- a/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDao.kt +++ b/app/src/main/java/com/topjohnwu/magisk/data/database/RepoDao.kt @@ -6,7 +6,7 @@ import com.topjohnwu.magisk.core.model.module.Repo import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext -@Database(version = 6, entities = [Repo::class, RepoEtag::class], exportSchema = false) +@Database(version = 7, entities = [Repo::class], exportSchema = false) abstract class RepoDatabase : RoomDatabase() { abstract fun repoDao() : RepoDao @@ -17,17 +17,11 @@ abstract class RepoDatabase : RoomDatabase() { @Dao abstract class RepoDao(private val db: RepoDatabase) { - val repoIDList get() = getRepoID().map { it.id } - val repos: List get() = when (Config.repoOrder) { Config.Value.ORDER_NAME -> getReposNameOrder() else -> getReposDateOrder() } - var etagKey: String - set(value) = addEtagRaw(RepoEtag(0, value)) - get() = etagRaw()?.key.orEmpty() - suspend fun clear() = withContext(Dispatchers.IO) { db.clearAllTables() } @Query("SELECT * FROM repos ORDER BY last_update DESC") @@ -42,8 +36,8 @@ abstract class RepoDao(private val db: RepoDatabase) { @Query("SELECT * FROM repos WHERE id = :id") abstract fun getRepo(id: String): Repo? - @Query("SELECT id FROM repos") - protected abstract fun getRepoID(): List + @Query("SELECT id, last_update FROM repos") + abstract fun getRepoStubs(): List @Delete abstract fun removeRepo(repo: Repo) @@ -53,21 +47,9 @@ abstract class RepoDao(private val db: RepoDatabase) { @Query("DELETE FROM repos WHERE id IN (:idList)") abstract fun removeRepos(idList: Collection) - - @Query("SELECT * FROM etag") - protected abstract fun etagRaw(): RepoEtag? - - @Insert(onConflict = OnConflictStrategy.REPLACE) - protected abstract fun addEtagRaw(etag: RepoEtag) } -data class RepoID( - @PrimaryKey val id: String +data class RepoStub( + @PrimaryKey val id: String, + val last_update: Long ) - -@Entity(tableName = "etag") -data class RepoEtag( - @PrimaryKey val id: Int, - val key: String -) - diff --git a/app/src/main/java/com/topjohnwu/magisk/data/network/NetworkServices.kt b/app/src/main/java/com/topjohnwu/magisk/data/network/NetworkServices.kt index 92fa202b3..63c85f3c9 100644 --- a/app/src/main/java/com/topjohnwu/magisk/data/network/NetworkServices.kt +++ b/app/src/main/java/com/topjohnwu/magisk/data/network/NetworkServices.kt @@ -2,22 +2,17 @@ package com.topjohnwu.magisk.data.network import com.topjohnwu.magisk.core.Const import com.topjohnwu.magisk.core.model.BranchInfo +import com.topjohnwu.magisk.core.model.RepoJson import com.topjohnwu.magisk.core.model.UpdateInfo -import com.topjohnwu.magisk.core.tasks.GithubRepoInfo import okhttp3.ResponseBody -import retrofit2.Response import retrofit2.http.* private const val REVISION = "revision" -private const val MODULE = "module" -private const val FILE = "file" -private const val IF_NONE_MATCH = "If-None-Match" private const val BRANCH = "branch" private const val REPO = "repo" const val MAGISK_FILES = "topjohnwu/magisk_files" const val MAGISK_MAIN = "topjohnwu/Magisk" -private const val MAGISK_MODULES = "Magisk-Modules-Repo" interface GithubPageServices { @@ -46,13 +41,13 @@ interface JSDelivrServices { suspend fun fetchInstaller(@Path(REVISION) revision: String): ResponseBody } -interface GithubRawServices { +interface RawServices { @GET suspend fun fetchCustomUpdate(@Url url: String): UpdateInfo - @GET("$MAGISK_MODULES/{$MODULE}/master/{$FILE}") - suspend fun fetchModuleFile(@Path(MODULE) id: String, @Path(FILE) file: String): String + @GET + suspend fun fetchRepoInfo(@Url url: String): RepoJson @GET @Streaming @@ -65,15 +60,6 @@ interface GithubRawServices { interface GithubApiServices { - @GET("users/$MAGISK_MODULES/repos") - @Headers("Accept: application/vnd.github.v3+json") - suspend fun fetchRepos( - @Query("page") page: Int, - @Header(IF_NONE_MATCH) etag: String, - @Query("sort") sort: String = "pushed", - @Query("per_page") count: Int = 100 - ): Response> - @GET("repos/{$REPO}/branches/{$BRANCH}") @Headers("Accept: application/vnd.github.v3+json") suspend fun fetchBranch( diff --git a/app/src/main/java/com/topjohnwu/magisk/data/repository/NetworkService.kt b/app/src/main/java/com/topjohnwu/magisk/data/repository/NetworkService.kt index a72f93d35..4c28d0ebc 100644 --- a/app/src/main/java/com/topjohnwu/magisk/data/repository/NetworkService.kt +++ b/app/src/main/java/com/topjohnwu/magisk/data/repository/NetworkService.kt @@ -9,7 +9,6 @@ import com.topjohnwu.magisk.core.Config.Value.STABLE_CHANNEL import com.topjohnwu.magisk.core.Const import com.topjohnwu.magisk.core.Info import com.topjohnwu.magisk.core.model.* -import com.topjohnwu.magisk.core.model.module.Repo import com.topjohnwu.magisk.data.network.* import okhttp3.ResponseBody import retrofit2.HttpException @@ -18,7 +17,7 @@ import java.io.IOException class NetworkService( private val pages: GithubPageServices, - private val raw: GithubRawServices, + private val raw: RawServices, private val jsd: JSDelivrServices, private val api: GithubApiServices ) { @@ -68,7 +67,10 @@ class NetworkService( ) } - // Byte streams + // Modules related + suspend fun fetchRepoInfo(url: String = Const.Url.OFFICIAL_REPO) = raw.fetchRepoInfo(url) + + // Fetch files suspend fun fetchSafetynet() = jsd.fetchSafetynet() suspend fun fetchBootctl() = jsd.fetchBootctl() suspend fun fetchInstaller(): ResponseBody { @@ -76,14 +78,8 @@ class NetworkService( return jsd.fetchInstaller(sha) } suspend fun fetchFile(url: String) = raw.fetchFile(url) - - // Strings - suspend fun fetchMetadata(repo: Repo) = raw.fetchModuleFile(repo.id, "module.prop") - suspend fun fetchReadme(repo: Repo) = raw.fetchModuleFile(repo.id, "README.md") suspend fun fetchString(url: String) = raw.fetchString(url) - // API calls - suspend fun fetchRepos(page: Int, etag: String) = api.fetchRepos(page, etag) private suspend fun fetchCanaryVersion() = api.fetchBranch(MAGISK_FILES, "canary").commit.sha private suspend fun fetchMainVersion() = api.fetchBranch(MAGISK_MAIN, "master").commit.sha } diff --git a/app/src/main/java/com/topjohnwu/magisk/di/NetworkingModule.kt b/app/src/main/java/com/topjohnwu/magisk/di/NetworkingModule.kt index 2301e3a8d..b3711dc8c 100644 --- a/app/src/main/java/com/topjohnwu/magisk/di/NetworkingModule.kt +++ b/app/src/main/java/com/topjohnwu/magisk/di/NetworkingModule.kt @@ -8,8 +8,8 @@ import com.topjohnwu.magisk.core.Const import com.topjohnwu.magisk.core.Info import com.topjohnwu.magisk.data.network.GithubApiServices import com.topjohnwu.magisk.data.network.GithubPageServices -import com.topjohnwu.magisk.data.network.GithubRawServices import com.topjohnwu.magisk.data.network.JSDelivrServices +import com.topjohnwu.magisk.data.network.RawServices import com.topjohnwu.magisk.ktx.precomputedText import com.topjohnwu.magisk.net.Networking import com.topjohnwu.magisk.net.NoSSLv3SocketFactory @@ -30,7 +30,7 @@ import java.net.UnknownHostException val networkingModule = module { single { createOkHttpClient(get()) } single { createRetrofit(get()) } - single { createApiService(get(), Const.Url.GITHUB_RAW_URL) } + single { createApiService(get(), Const.Url.GITHUB_RAW_URL) } single { createApiService(get(), Const.Url.GITHUB_API_URL) } single { createApiService(get(), Const.Url.GITHUB_PAGE_URL) } single { createApiService(get(), Const.Url.JS_DELIVR_URL) } diff --git a/app/src/main/java/com/topjohnwu/magisk/events/ViewEvents.kt b/app/src/main/java/com/topjohnwu/magisk/events/ViewEvents.kt index 40835b2fe..6d9e38017 100644 --- a/app/src/main/java/com/topjohnwu/magisk/events/ViewEvents.kt +++ b/app/src/main/java/com/topjohnwu/magisk/events/ViewEvents.kt @@ -22,10 +22,10 @@ class ViewActionEvent(val action: BaseActivity.() -> Unit) : ViewEvent(), Activi override fun invoke(activity: BaseUIActivity<*, *>) = action(activity) } -class OpenChangelogEvent(val item: Repo) : ViewEventWithScope(), ContextExecutor { +class OpenReadmeEvent(val item: Repo) : ViewEventWithScope(), ContextExecutor { override fun invoke(context: Context) { scope.launch { - MarkDownWindow.show(context, null, item::readme) + MarkDownWindow.show(context, null, item::notes) } } } diff --git a/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt b/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt index 2d056eb5f..15333d08e 100644 --- a/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt +++ b/app/src/main/java/com/topjohnwu/magisk/ui/module/ModuleViewModel.kt @@ -14,8 +14,8 @@ import com.topjohnwu.magisk.core.tasks.RepoUpdater import com.topjohnwu.magisk.data.database.RepoByNameDao import com.topjohnwu.magisk.data.database.RepoByUpdatedDao import com.topjohnwu.magisk.databinding.RvItem +import com.topjohnwu.magisk.events.OpenReadmeEvent import com.topjohnwu.magisk.events.SelectModuleEvent -import com.topjohnwu.magisk.events.OpenChangelogEvent import com.topjohnwu.magisk.events.SnackbarEvent import com.topjohnwu.magisk.events.dialog.ModuleInstallDialog import com.topjohnwu.magisk.ktx.addOnListChangedCallback @@ -315,14 +315,14 @@ class ModuleViewModel( } fun infoPressed(item: RepoItem) = - if (isConnected.get()) OpenChangelogEvent(item.item).publish() + if (isConnected.get()) OpenReadmeEvent(item.item).publish() else SnackbarEvent(R.string.no_connection).publish() fun infoPressed(item: ModuleItem) { item.repo?.also { if (isConnected.get()) - OpenChangelogEvent(it).publish() + OpenReadmeEvent(it).publish() else SnackbarEvent(R.string.no_connection).publish() } ?: return diff --git a/build.gradle.kts b/build.gradle.kts index 02049a102..2606ee406 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -18,7 +18,7 @@ buildscript { extra["vNav"] = vNav dependencies { - classpath("com.android.tools.build:gradle:4.0.1") + classpath("com.android.tools.build:gradle:4.0.2") classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.4.10") classpath("androidx.navigation:navigation-safe-args-gradle-plugin:${vNav}")