EN: Improve some database bits

This commit is contained in:
Marvin W 2020-10-07 23:16:53 +02:00
parent 7601b37ee4
commit 4486ff52ef
No known key found for this signature in database
GPG Key ID: 072E9235DB996F2A
3 changed files with 49 additions and 24 deletions

View File

@ -12,6 +12,7 @@ import android.content.Intent
import androidx.lifecycle.LifecycleService import androidx.lifecycle.LifecycleService
import androidx.lifecycle.lifecycleScope import androidx.lifecycle.lifecycleScope
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import org.microg.gms.common.ForegroundServiceContext import org.microg.gms.common.ForegroundServiceContext
@ -22,8 +23,12 @@ class CleanupService : LifecycleService() {
if (isNeeded(this)) { if (isNeeded(this)) {
lifecycleScope.launchWhenStarted { lifecycleScope.launchWhenStarted {
launch(Dispatchers.IO) { launch(Dispatchers.IO) {
ExposureDatabase.with(this@CleanupService) { var workPending = true
it.dailyCleanup() while (workPending) {
ExposureDatabase.with(this@CleanupService) {
workPending = !it.dailyCleanup()
}
if (workPending) delay(5000L)
} }
ExposurePreferences(this@CleanupService).lastCleanup = System.currentTimeMillis() ExposurePreferences(this@CleanupService).lastCleanup = System.currentTimeMillis()
} }

View File

@ -79,28 +79,35 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
Log.d(TAG, "Finished database upgrade") Log.d(TAG, "Finished database upgrade")
} }
fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray): Int = fun SQLiteDatabase.delete(table: String, whereClause: String, args: LongArray, limit: Int): Int =
compileStatement("DELETE FROM $table WHERE $whereClause").use { compileStatement("DELETE FROM $table WHERE $whereClause LIMIT $limit").use {
args.forEachIndexed { idx, l -> it.bindLong(idx + 1, l) } args.forEachIndexed { idx, l -> it.bindLong(idx + 1, l) }
it.executeUpdateDelete() it.executeUpdateDelete()
} }
fun dailyCleanup() = writableDatabase.run { fun dailyCleanup(): Boolean = writableDatabase.run {
val rollingStartTime = currentRollingStartNumber * ROLLING_WINDOW_LENGTH * 1000 - TimeUnit.DAYS.toMillis(KEEP_DAYS.toLong()) val rollingStartTime = currentRollingStartNumber * ROLLING_WINDOW_LENGTH * 1000 - TimeUnit.DAYS.toMillis(KEEP_DAYS.toLong())
val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime)) val advertisements = delete(TABLE_ADVERTISEMENTS, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $advertisements adv") Log.d(TAG, "Deleted on daily cleanup: $advertisements adv")
val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime)) if (advertisements == DELETE_LIMIT) return@run false
val appLogEntries = delete(TABLE_APP_LOG, "timestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $appLogEntries applogs") Log.d(TAG, "Deleted on daily cleanup: $appLogEntries applogs")
val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS)) if (appLogEntries == DELETE_LIMIT) return@run false
val temporaryExposureKeys = delete(TABLE_TEK, "(rollingStartNumber + rollingPeriod) < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $temporaryExposureKeys teks") Log.d(TAG, "Deleted on daily cleanup: $temporaryExposureKeys teks")
val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD)) if (temporaryExposureKeys == DELETE_LIMIT) return@run false
val singleCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_SINGLE, "rollingStartNumber < ?", longArrayOf(rollingStartTime / ROLLING_WINDOW_LENGTH_MS - ROLLING_PERIOD), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $singleCheckedTemporaryExposureKeys tcss") Log.d(TAG, "Deleted on daily cleanup: $singleCheckedTemporaryExposureKeys tcss")
val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime)) if (singleCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false
val fileCheckedTemporaryExposureKeys = delete(TABLE_TEK_CHECK_FILE, "endTimestamp < ?", longArrayOf(rollingStartTime), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $fileCheckedTemporaryExposureKeys tcfs") Log.d(TAG, "Deleted on daily cleanup: $fileCheckedTemporaryExposureKeys tcfs")
val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY)) if (fileCheckedTemporaryExposureKeys == DELETE_LIMIT) return@run false
val appPerms = delete(TABLE_APP_PERMS, "timestamp < ?", longArrayOf(System.currentTimeMillis() - CONFIRM_PERMISSION_VALIDITY), DELETE_LIMIT)
Log.d(TAG, "Deleted on daily cleanup: $appPerms perms") Log.d(TAG, "Deleted on daily cleanup: $appPerms perms")
if (appPerms == DELETE_LIMIT) return@run false
execSQL("VACUUM;") execSQL("VACUUM;")
Log.d(TAG, "Done vacuuming") Log.d(TAG, "Done vacuuming")
return@run true
} }
fun grantPermission(packageName: String, signatureDigest: String, permission: String, timestamp: Long = System.currentTimeMillis()) = writableDatabase.run { fun grantPermission(packageName: String, signatureDigest: String, permission: String, timestamp: Long = System.currentTimeMillis()) = writableDatabase.run {
@ -205,7 +212,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
} }
fun batchStoreSingleDiagnosisKey(tid: Long, keys: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run { fun batchStoreSingleDiagnosisKey(tid: Long, keys: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run {
beginTransaction() beginTransactionNonExclusive()
try { try {
keys.forEach { storeSingleDiagnosisKey(tid, it, database) } keys.forEach { storeSingleDiagnosisKey(tid, it, database) }
setTransactionSuccessful() setTransactionSuccessful()
@ -356,7 +363,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
} }
fun finishFileMatching(tid: Long, hash: ByteArray, endTimestamp: Long, keys: List<TemporaryExposureKey>, updates: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run { fun finishFileMatching(tid: Long, hash: ByteArray, endTimestamp: Long, keys: List<TemporaryExposureKey>, updates: List<TemporaryExposureKey>, database: SQLiteDatabase = writableDatabase) = database.run {
beginTransaction() beginTransactionNonExclusive()
try { try {
insert(TABLE_TEK_CHECK_FILE, "NULL", ContentValues().apply { insert(TABLE_TEK_CHECK_FILE, "NULL", ContentValues().apply {
put("hash", ByteString.of(*hash).hex()) put("hash", ByteString.of(*hash).hex())
@ -367,27 +374,36 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
val workQueue = LinkedBlockingQueue<Runnable>() val workQueue = LinkedBlockingQueue<Runnable>()
val poolSize = Runtime.getRuntime().availableProcessors() val poolSize = Runtime.getRuntime().availableProcessors()
val executor = ThreadPoolExecutor(poolSize, poolSize, 1, TimeUnit.SECONDS, workQueue) val executor = ThreadPoolExecutor(poolSize, poolSize, 1, TimeUnit.SECONDS, workQueue)
val futures = arrayListOf<Future<*>>() val futures = arrayListOf<Future<TemporaryExposureKey?>>()
val oldestRpi = oldestRpi val oldestRpi = oldestRpi
var ignored = 0 var ignored = 0
var processed = 0 var processed = 0
var found = 0 var found = 0
var riskLogged = 0
for (key in keys) { for (key in keys) {
if (key.transmissionRiskLevel > riskLogged) {
riskLogged = key.transmissionRiskLevel
Log.d(TAG, "First key with risk ${key.transmissionRiskLevel}: ${ByteString.of(*key.keyData).hex()} starts ${key.rollingStartIntervalNumber}")
}
if ((key.rollingStartIntervalNumber + key.rollingPeriod) * ROLLING_WINDOW_LENGTH_MS + ALLOWED_KEY_OFFSET_MS < oldestRpi) { if ((key.rollingStartIntervalNumber + key.rollingPeriod) * ROLLING_WINDOW_LENGTH_MS + ALLOWED_KEY_OFFSET_MS < oldestRpi) {
// Early ignore because key is older than since we started scanning. // Early ignore because key is older than since we started scanning.
ignored++; ignored++;
} else { } else {
futures.add(executor.submit { futures.add(executor.submit(Callable {
if (findMeasuredExposures(key).isNotEmpty()) {
applyDiagnosisFileKeySearchResult(tcfid, key, this)
found++;
}
processed++ processed++
}) if (findMeasuredExposures(key).isNotEmpty()) {
key
} else {
null
}
}))
} }
} }
for (future in futures) { for (future in futures) {
future.get() future.get()?.let {
applyDiagnosisFileKeySearchResult(tcfid, it, this)
found++
}
} }
Log.d(TAG, "Processed $processed keys, found $found matches, ignored $ignored keys that are older than our scanning efforts ($oldestRpi)") Log.d(TAG, "Processed $processed keys, found $found matches, ignored $ignored keys that are older than our scanning efforts ($oldestRpi)")
executor.shutdown() executor.shutdown()
@ -585,7 +601,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
get() = readableDatabase.run { get() = readableDatabase.run {
query(TABLE_ADVERTISEMENTS, arrayOf("MIN(timestamp)"), null, null, null, null, null).use { cursor -> query(TABLE_ADVERTISEMENTS, arrayOf("MIN(timestamp)"), null, null, null, null, null).use { cursor ->
if (cursor.moveToNext()) { if (cursor.moveToNext()) {
cursor.getLong(0) cursor.getLong(0).let { if (it == 0L) System.currentTimeMillis() else it }
} else { } else {
System.currentTimeMillis() System.currentTimeMillis()
} }
@ -634,7 +650,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
} }
private fun ensureTemporaryExposureKey(): TemporaryExposureKey = writableDatabase.let { database -> private fun ensureTemporaryExposureKey(): TemporaryExposureKey = writableDatabase.let { database ->
database.beginTransaction() database.beginTransactionNonExclusive()
try { try {
var key = findOwnKeyAt(currentRollingStartNumber.toInt(), database) var key = findOwnKeyAt(currentRollingStartNumber.toInt(), database)
if (key == null) { if (key == null) {
@ -688,6 +704,7 @@ class ExposureDatabase private constructor(private val context: Context) : SQLit
companion object { companion object {
private const val DB_NAME = "exposure.db" private const val DB_NAME = "exposure.db"
private const val DB_VERSION = 5 private const val DB_VERSION = 5
private const val DELETE_LIMIT = 1000
private const val TABLE_ADVERTISEMENTS = "advertisements" private const val TABLE_ADVERTISEMENTS = "advertisements"
private const val TABLE_APP_LOG = "app_log" private const val TABLE_APP_LOG = "app_log"
private const val TABLE_TEK = "tek" private const val TABLE_TEK = "tek"

View File

@ -5,6 +5,7 @@
package org.microg.gms.nearby.exposurenotification package org.microg.gms.nearby.exposurenotification
import android.util.Log
import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration
import com.google.android.gms.nearby.exposurenotification.ExposureInformation import com.google.android.gms.nearby.exposurenotification.ExposureInformation
import com.google.android.gms.nearby.exposurenotification.RiskLevel import com.google.android.gms.nearby.exposurenotification.RiskLevel
@ -111,7 +112,9 @@ data class MergedExposure internal constructor(val key: TemporaryExposureKey, va
} }
fun getRiskScore(configuration: ExposureConfiguration): Int { fun getRiskScore(configuration: ExposureConfiguration): Int {
return getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration) val risk = getAttenuationRiskScore(configuration) * getDaysSinceLastExposureRiskScore(configuration) * getDurationRiskScore(configuration) * getTransmissionRiskScore(configuration)
Log.d(TAG, "Risk score calculation: ${getAttenuationRiskScore(configuration)} * ${getDaysSinceLastExposureRiskScore(configuration)} * ${getDurationRiskScore(configuration)} * ${getTransmissionRiskScore(configuration)} = $risk")
return risk
} }
fun getAttenuationDurations(configuration: ExposureConfiguration): IntArray { fun getAttenuationDurations(configuration: ExposureConfiguration): IntArray {