EN: Improve some database bits

play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/CleanupService.kt

@@ -12,6 +12,7 @@ import android.content.Intent
 import androidx.lifecycle.LifecycleService
 import androidx.lifecycle.lifecycleScope
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import org.microg.gms.common.ForegroundServiceContext
 
@@ -22,8 +23,12 @@ class CleanupService : LifecycleService() {
         if (isNeeded(this)) {
             lifecycleScope.launchWhenStarted {
                 launch(Dispatchers.IO) {
-                    ExposureDatabase.with(this@CleanupService) {
+                    var workPending = true
-                        it.dailyCleanup()
+                    while (workPending) {
+                        ExposureDatabase.with(this@CleanupService) {
+                            workPending = !it.dailyCleanup()
+                        }
+                        if (workPending) delay(5000L)
                     }
                     ExposurePreferences(this@CleanupService).lastCleanup = System.currentTimeMillis()
                 }

play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/ExposureDatabase.kt

@@ -79,28 +79,35 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
         Log.d(TAG, "Finished database upgrade")
     }
 
-    fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray): Int =
+    fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray, limit: Int): Int =
-            compileStatement("DELETE FROM $table WHERE $whereClause").use {
+            compileStatement("DELETE FROM $table WHERE $whereClause LIMIT $limit").use {
                 args.forEachIndexed { idx, l -> it.bindLong(idx + 1, l) }
                 it.executeUpdateDelete()
             }
 
-    fun dailyCleanup() = writableDatabase.run {
+    fun dailyCleanup(): Boolean = writableDatabase.run {
         val rollingStartTime = currentRollingStartNumber * ROLLING_WINDOW_LENGTH * 1000 - TimeUnit.DAYS.toMillis(KEEP_DAYS.toLong())
-        val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime))
+        val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $advertisements adv")
-        val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime))
+        if (advertisements == DELETE_LIMIT) return@run false
+        val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $appLogEntries applogs")
-        val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS))
+        if (appLogEntries == DELETE_LIMIT) return@run false
+        val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $temporaryExposureKeys teks")
-        val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD))
+        if (temporaryExposureKeys == DELETE_LIMIT) return@run false
+        val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $singleCheckedTemporaryExposureKeys tcss")
-        val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime))
+        if (singleCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false
+        val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $fileCheckedTemporaryExposureKeys tcfs")
-        val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY))
+        if (fileCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false
+        val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY), DELETE_LIMIT)
         Log.d(TAG, "Deleted on daily cleanup: $appPerms perms")
+        if (appPerms == DELETE_LIMIT) return@run false
         execSQL("VACUUM;")
         Log.d(TAG, "Done vacuuming")
+        return@run true
     }
 
     fun grantPermission(packageName: String, signatureDigest: String, permission: String, timestamp: Long = System.currentTimeMillis()) = writableDatabase.run {
@@ -205,7 +212,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
     }
 
     fun batchStoreSingleDiagnosisKey(tid: Long, keys: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run {
-        beginTransaction()
+        beginTransactionNonExclusive()
         try {
             keys.forEach { storeSingleDiagnosisKey(tid, it, database) }
             setTransactionSuccessful()
@@ -356,7 +363,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
     }
 
     fun finishFileMatching(tid: Long, hash: ByteArray, endTimestamp: Long, keys: List<TemporaryExposureKey>, updates: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run {
-        beginTransaction()
+        beginTransactionNonExclusive()
         try {
             insert(TABLE_TEK_CHECK_FILE, "NULL", ContentValues().apply {
                 put("hash", ByteString.of(*hash).hex())
@@ -367,27 +374,36 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
             val workQueue = LinkedBlockingQueue<Runnable>()
             val poolSize = Runtime.getRuntime().availableProcessors()
             val executor = ThreadPoolExecutor(poolSize, poolSize, 1, TimeUnit.SECONDS, workQueue)
-            val futures = arrayListOf<Future<*>>()
+            val futures = arrayListOf<Future<TemporaryExposureKey?>>()
             val oldestRpi = oldestRpi
             var ignored = 0
             var processed = 0
             var found = 0
+            var riskLogged = 0
             for (key in keys) {
+                if (key.transmissionRiskLevel > riskLogged) {
+                    riskLogged = key.transmissionRiskLevel
+                    Log.d(TAG, "First key with risk ${key.transmissionRiskLevel}: ${ByteString.of(*key.keyData).hex()} starts ${key.rollingStartIntervalNumber}")
+                }
                 if ((key.rollingStartIntervalNumber + key.rollingPeriod) * ROLLING_WINDOW_LENGTH_MS + ALLOWED_KEY_OFFSET_MS < oldestRpi) {
                     // Early ignore because key is older than since we started scanning.
                     ignored++;
                 } else {
-                    futures.add(executor.submit {
+                    futures.add(executor.submit(Callable {
-                        if (findMeasuredExposures(key).isNotEmpty()) {
-                            applyDiagnosisFileKeySearchResult(tcfid, key, this)
-                            found++;
-                        }
                         processed++
-                    })
+                        if (findMeasuredExposures(key).isNotEmpty()) {
+                            key
+                        } else {
+                            null
+                        }
+                    }))
                 }
             }
             for (future in futures) {
-                future.get()
+                future.get()?.let {
+                    applyDiagnosisFileKeySearchResult(tcfid, it, this)
+                    found++
+                }
             }
             Log.d(TAG,  "Processed $processed keys, found $found matches, ignored $ignored keys that are older than our scanning efforts ($oldestRpi)")
             executor.shutdown()
@@ -585,7 +601,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
         get() = readableDatabase.run {
             query(TABLE_ADVERTISEMENTS, arrayOf("MIN(timestamp)"), null, null, null, null, null).use { cursor ->
                 if (cursor.moveToNext()) {
-                    cursor.getLong(0)
+                    cursor.getLong(0).let { if (it == 0L) System.currentTimeMillis() else it }
                 } else {
                     System.currentTimeMillis()
                 }
@@ -634,7 +650,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
     }
 
     private fun ensureTemporaryExposureKey(): TemporaryExposureKey = writableDatabase.let { database ->
-        database.beginTransaction()
+        database.beginTransactionNonExclusive()
         try {
             var key = findOwnKeyAt(currentRollingStartNumber.toInt(), database)
             if (key == null) {
@@ -688,6 +704,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
     companion object {
         private const val DB_NAME = "exposure.db"
         private const val DB_VERSION = 5
+        private const val DELETE_LIMIT = 1000
         private const val TABLE_ADVERTISEMENTS = "advertisements"
         private const val TABLE_APP_LOG = "app_log"
         private const val TABLE_TEK = "tek"

play-services-nearby-core/src/main/kotlin/org/microg/gms/nearby/exposurenotification/MeasuredExposure.kt

@@ -5,6 +5,7 @@
 
 package org.microg.gms.nearby.exposurenotification
 
+import android.util.Log
 import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration
 import com.google.android.gms.nearby.exposurenotification.ExposureInformation
 import com.google.android.gms.nearby.exposurenotification.RiskLevel
@@ -111,7 +112,9 @@ data class MergedExposure internal constructor(val key: TemporaryExposureKey, va
     }
 
     fun getRiskScore(configuration: ExposureConfiguration): Int {
-        return getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration)
+        val risk = getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration)
+        Log.d(TAG, "Risk score calculation: ${getAttenuationRiskScore(configuration)} * ${getDaysSinceLastExposureRiskScore(configuration)} * ${getDurationRiskScore(configuration)} * ${getTransmissionRiskScore(configuration)} = $risk")
+        return risk
     }
 
     fun getAttenuationDurations(configuration: ExposureConfiguration): IntArray {