From 4486ff52efe3f94794f203c7837f0cbe37b5a756 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Wed, 7 Oct 2020 23:16:53 +0200 Subject: [PATCH] EN: Improve some database bits --- .../exposurenotification/CleanupService.kt | 9 ++- .../exposurenotification/ExposureDatabase.kt | 59 ++++++++++++------- .../exposurenotification/MeasuredExposure.kt | 5 +- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/CleanupService.kt b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/CleanupService.kt index fad33772..bcc2bf58 100644 --- a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/CleanupService.kt +++ b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/CleanupService.kt @@ -12,6 +12,7 @@ import android.content.Intent import androidx.lifecycle.LifecycleService import androidx.lifecycle.lifecycleScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.launch import org.microg.gms.common.ForegroundServiceContext @@ -22,8 +23,12 @@ class CleanupService : LifecycleService() { if (isNeeded(this)) { lifecycleScope.launchWhenStarted { launch(Dispatchers.IO) { - ExposureDatabase.with(this@CleanupService) { - it.dailyCleanup() + var workPending = true + while (workPending) { + ExposureDatabase.with(this@CleanupService) { + workPending = !it.dailyCleanup() + } + if (workPending) delay(5000L) } ExposurePreferences(this@CleanupService).lastCleanup = System.currentTimeMillis() } diff --git a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/ExposureDatabase.kt b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/ExposureDatabase.kt index 0790776a..a21fecf8 100644 --- a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/ExposureDatabase.kt +++ b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/ExposureDatabase.kt @@ -79,28 +79,35 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit Log.d(TAG, "Finished database upgrade") } - fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray): Int = - compileStatement("DELETE FROM $table WHERE $whereClause").use { + fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray, limit: Int): Int = + compileStatement("DELETE FROM $table WHERE $whereClause LIMIT $limit").use { args.forEachIndexed { idx, l -> it.bindLong(idx + 1, l) } it.executeUpdateDelete() } - fun dailyCleanup() = writableDatabase.run { + fun dailyCleanup(): Boolean = writableDatabase.run { val rollingStartTime = currentRollingStartNumber * ROLLING_WINDOW_LENGTH * 1000 - TimeUnit.DAYS.toMillis(KEEP_DAYS.toLong()) - val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime)) + val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $advertisements adv") - val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime)) + if (advertisements == DELETE_LIMIT) return@run false + val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $appLogEntries applogs") - val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS)) + if (appLogEntries == DELETE_LIMIT) return@run false + val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $temporaryExposureKeys teks") - val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD)) + if (temporaryExposureKeys == DELETE_LIMIT) return@run false + val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $singleCheckedTemporaryExposureKeys tcss") - val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime)) + if (singleCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false + val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $fileCheckedTemporaryExposureKeys tcfs") - val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY)) + if (fileCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false + val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY), DELETE_LIMIT) Log.d(TAG, "Deleted on daily cleanup: $appPerms perms") + if (appPerms == DELETE_LIMIT) return@run false execSQL("VACUUM;") Log.d(TAG, "Done vacuuming") + return@run true } fun grantPermission(packageName: String, signatureDigest: String, permission: String, timestamp: Long = System.currentTimeMillis()) = writableDatabase.run { @@ -205,7 +212,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit } fun batchStoreSingleDiagnosisKey(tid: Long, keys: List, database: SQLiteDatabase = writableDatabase) = database.run { - beginTransaction() + beginTransactionNonExclusive() try { keys.forEach { storeSingleDiagnosisKey(tid, it, database) } setTransactionSuccessful() @@ -356,7 +363,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit } fun finishFileMatching(tid: Long, hash: ByteArray, endTimestamp: Long, keys: List, updates: List, database: SQLiteDatabase = writableDatabase) = database.run { - beginTransaction() + beginTransactionNonExclusive() try { insert(TABLE_TEK_CHECK_FILE, "NULL", ContentValues().apply { put("hash", ByteString.of(*hash).hex()) @@ -367,27 +374,36 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit val workQueue = LinkedBlockingQueue() val poolSize = Runtime.getRuntime().availableProcessors() val executor = ThreadPoolExecutor(poolSize, poolSize, 1, TimeUnit.SECONDS, workQueue) - val futures = arrayListOf>() + val futures = arrayListOf>() val oldestRpi = oldestRpi var ignored = 0 var processed = 0 var found = 0 + var riskLogged = 0 for (key in keys) { + if (key.transmissionRiskLevel > riskLogged) { + riskLogged = key.transmissionRiskLevel + Log.d(TAG, "First key with risk ${key.transmissionRiskLevel}: ${ByteString.of(*key.keyData).hex()} starts ${key.rollingStartIntervalNumber}") + } if ((key.rollingStartIntervalNumber + key.rollingPeriod) * ROLLING_WINDOW_LENGTH_MS + ALLOWED_KEY_OFFSET_MS < oldestRpi) { // Early ignore because key is older than since we started scanning. ignored++; } else { - futures.add(executor.submit { - if (findMeasuredExposures(key).isNotEmpty()) { - applyDiagnosisFileKeySearchResult(tcfid, key, this) - found++; - } + futures.add(executor.submit(Callable { processed++ - }) + if (findMeasuredExposures(key).isNotEmpty()) { + key + } else { + null + } + })) } } for (future in futures) { - future.get() + future.get()?.let { + applyDiagnosisFileKeySearchResult(tcfid, it, this) + found++ + } } Log.d(TAG, "Processed $processed keys, found $found matches, ignored $ignored keys that are older than our scanning efforts ($oldestRpi)") executor.shutdown() @@ -585,7 +601,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit get() = readableDatabase.run { query(TABLE_ADVERTISEMENTS, arrayOf("MIN(timestamp)"), null, null, null, null, null).use { cursor -> if (cursor.moveToNext()) { - cursor.getLong(0) + cursor.getLong(0).let { if (it == 0L) System.currentTimeMillis() else it } } else { System.currentTimeMillis() } @@ -634,7 +650,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit } private fun ensureTemporaryExposureKey(): TemporaryExposureKey = writableDatabase.let { database -> - database.beginTransaction() + database.beginTransactionNonExclusive() try { var key = findOwnKeyAt(currentRollingStartNumber.toInt(), database) if (key == null) { @@ -688,6 +704,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit companion object { private const val DB_NAME = "exposure.db" private const val DB_VERSION = 5 + private const val DELETE_LIMIT = 1000 private const val TABLE_ADVERTISEMENTS = "advertisements" private const val TABLE_APP_LOG = "app_log" private const val TABLE_TEK = "tek" diff --git a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/MeasuredExposure.kt b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/MeasuredExposure.kt index 18113f1f..1ef24849 100644 --- a/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/MeasuredExposure.kt +++ b/play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/MeasuredExposure.kt @@ -5,6 +5,7 @@ package org.microg.gms.nearby.exposurenotification +import android.util.Log import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration import com.google.android.gms.nearby.exposurenotification.ExposureInformation import com.google.android.gms.nearby.exposurenotification.RiskLevel @@ -111,7 +112,9 @@ data class MergedExposure internal constructor(val key: TemporaryExposureKey, va } fun getRiskScore(configuration: ExposureConfiguration): Int { - return getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration) + val risk = getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration) + Log.d(TAG, "Risk score calculation: ${getAttenuationRiskScore(configuration)} * ${getDaysSinceLastExposureRiskScore(configuration)} * ${getDurationRiskScore(configuration)} * ${getTransmissionRiskScore(configuration)} = $risk") + return risk } fun getAttenuationDurations(configuration: ExposureConfiguration): IntArray {