MetaData first pass

.github/workflows/ci.yml

@@ -22,6 +22,9 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
 
+      - name: Verify SqlDelight Migration
+        run: ./gradlew verifySqlDelightMigration
+
       - name: Build and publish
         run: ./gradlew jvmTest
 

gradle.properties

@@ -7,7 +7,7 @@ org.gradle.parallel=true
 
 # Maven
 GROUP=com.mercury.sqkon
-VERSION_NAME=1.0.0-alpha01
+VERSION_NAME=1.0.0-alpha02
 POM_NAME=Sqkon
 POM_INCEPTION_YEAR=2024
 POM_URL=https://github.com/MercuryTechnologies/sqkon/
@@ -27,6 +27,3 @@ kotlin.daemon.jvmargs=-Xmx4G
 #Android
 android.useAndroidX=true
 android.nonTransitiveRClass=true
-
-# KMP
-kotlin.mpp.androidGradlePluginCompatibility.nowarn=true

library/build.gradle.kts

@@ -1,3 +1,4 @@
+import app.cash.sqldelight.VERSION
 import com.android.build.api.variant.HasUnitTestBuilder
 import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSetTree
@@ -80,6 +81,8 @@ sqldelight {
             generateAsync = true
             packageName.set("com.mercury.sqkon.db")
             schemaOutputDirectory.set(file("src/commonMain/sqldelight/databases"))
+            // We're technically using 3.45.0, but 3.38 is the latest supported version
+            dialect("app.cash.sqldelight:sqlite-3-38-dialect:$VERSION")
         }
     }
 }

library/src/androidInstrumentedTest/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriverTest.android.kt

@@ -2,11 +2,9 @@ package com.mercury.sqkon.db
 
 import androidx.test.platform.app.InstrumentationRegistry
 
-actual fun createEntityQueries(): EntityQueries {
-    return createEntityQueries(
-        DriverFactory(
-            context = InstrumentationRegistry.getInstrumentation().targetContext,
-            name = null // in-memory database
-        )
+internal actual fun driverFactory(): DriverFactory {
+    return DriverFactory(
+        context = InstrumentationRegistry.getInstrumentation().targetContext,
+        name = null // in-memory database
     )
-}
+}

library/src/androidMain/kotlin/com/mercury/sqkon/db/Sqkon.android.kt

@@ -16,6 +16,8 @@ fun Sqkon(
     config: KeyValueStorage.Config = KeyValueStorage.Config(),
 ): Sqkon {
     val factory = DriverFactory(context, if (inMemory) null else "sqkon.db")
-    val entities = createEntityQueries(factory)
-    return Sqkon(entities, scope, json, config)
+    val driver = factory.createDriver()
+    val metadataQueries = MetadataQueries(driver)
+    val entityQueries = EntityQueries(driver)
+    return Sqkon(entityQueries, metadataQueries, scope, json, config)
 }

library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt

@@ -9,8 +9,8 @@ import kotlinx.coroutines.delay
 import org.jetbrains.annotations.VisibleForTesting
 
 class EntityQueries(
-    driver: SqlDriver,
-) : SuspendingTransacterImpl(driver) {
+    internal val sqlDriver: SqlDriver,
+) : SuspendingTransacterImpl(sqlDriver) {
 
     // Used to slow down insert/updates for testing
     @VisibleForTesting

library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt

@@ -2,22 +2,29 @@ package com.mercury.sqkon.db
 
 import app.cash.paging.PagingSource
 import app.cash.sqldelight.SuspendingTransacter
+import app.cash.sqldelight.TransactionCallbacks
 import app.cash.sqldelight.coroutines.asFlow
 import app.cash.sqldelight.coroutines.mapToList
 import app.cash.sqldelight.coroutines.mapToOne
+import app.cash.sqldelight.coroutines.mapToOneNotNull
 import com.mercury.sqkon.db.KeyValueStorage.Config.DeserializePolicy
 import com.mercury.sqkon.db.paging.OffsetQueryPagingSource
 import com.mercury.sqkon.db.serialization.KotlinSqkonSerializer
 import com.mercury.sqkon.db.serialization.SqkonJson
 import com.mercury.sqkon.db.serialization.SqkonSerializer
+import com.mercury.sqkon.db.utils.RequestHash
 import com.mercury.sqkon.db.utils.nowMillis
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.delay
+import kotlinx.coroutines.currentCoroutineContext
 import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.distinctUntilChanged
 import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import kotlinx.datetime.Clock
 import kotlin.reflect.KType
 import kotlin.reflect.typeOf
 
@@ -30,6 +37,7 @@ import kotlin.reflect.typeOf
 open class KeyValueStorage<T : Any>(
     protected val entityName: String,
     protected val entityQueries: EntityQueries,
+    protected val metadataQueries: MetadataQueries,
     protected val scope: CoroutineScope,
     protected val type: KType,
     protected val serializer: SqkonSerializer = KotlinSqkonSerializer(),
@@ -46,7 +54,7 @@ open class KeyValueStorage<T : Any>(
      *  @see update
      *  @see upsert
      */
-    suspend fun insert(key: String, value: T, ignoreIfExists: Boolean = false) {
+    suspend fun insert(key: String, value: T, ignoreIfExists: Boolean = false) = transaction {
         val now = nowMillis()
         val entity = Entity(
             entity_name = entityName,
@@ -57,6 +65,7 @@ open class KeyValueStorage<T : Any>(
             value_ = serializer.serialize(type, value) ?: error("Failed to serialize value")
         )
         entityQueries.insertEntity(entity, ignoreIfExists)
+        updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: entity.hashCode())
     }
 
     /**
@@ -67,7 +76,9 @@ open class KeyValueStorage<T : Any>(
      * @see updateAll
      * @see upsertAll
      */
-    suspend fun insertAll(values: Map<String, T>, ignoreIfExists: Boolean = false) {
+    suspend fun insertAll(
+        values: Map<String, T>, ignoreIfExists: Boolean = false
+    ) = withContext(RequestHash(values.hashCode())) {
         transaction {
             values.forEach { (key, value) -> insert(key, value, ignoreIfExists) }
         }
@@ -82,14 +93,15 @@ open class KeyValueStorage<T : Any>(
      * @see insert
      * @see upsert
      */
-    suspend fun update(key: String, value: T) {
+    suspend fun update(key: String, value: T) = transaction {
         entityQueries.updateEntity(
             entityName = entityName,
             entityKey = key,
             updatedAt = nowMillis(),
             expiresAt = null,
             value = serializer.serialize(type, value) ?: error("Failed to serialize value")
         )
+        updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: key.hashCode())
     }
 
     /**
@@ -99,7 +111,7 @@ open class KeyValueStorage<T : Any>(
      * @see insertAll
      * @see upsertAll
      */
-    suspend fun updateAll(values: Map<String, T>) {
+    suspend fun updateAll(values: Map<String, T>) = withContext(RequestHash(values.hashCode())) {
         transaction { values.forEach { (key, value) -> update(key, value) } }
     }
 
@@ -110,9 +122,11 @@ open class KeyValueStorage<T : Any>(
      * @see insert
      * @see update
      */
-    suspend fun upsert(key: String, value: T) = transaction {
-        update(key, value)
-        insert(key, value, ignoreIfExists = true)
+    suspend fun upsert(key: String, value: T) = withContext(RequestHash(key.hashCode())) {
+        transaction {
+            update(key, value)
+            insert(key, value, ignoreIfExists = true)
+        }
     }
 
     /**
@@ -123,10 +137,12 @@ open class KeyValueStorage<T : Any>(
      * @see insertAll
      * @see updateAll
      */
-    suspend fun upsertAll(values: Map<String, T>) = transaction {
-        values.forEach { (key, value) ->
-            update(key, value)
-            insert(key, value, ignoreIfExists = true)
+    suspend fun upsertAll(values: Map<String, T>) = withContext(RequestHash(values.hashCode())) {
+        transaction {
+            values.forEach { (key, value) ->
+                update(key, value)
+                insert(key, value, ignoreIfExists = true)
+            }
         }
     }
 
@@ -163,6 +179,7 @@ open class KeyValueStorage<T : Any>(
             )
             .asFlow()
             .mapToList(config.dispatcher)
+            .onEach { updateReadAt() }
             .map { list ->
                 if (list.isEmpty()) return@map emptyList<T>()
                 list.mapNotNull { entity -> entity.deserialize() }
@@ -195,6 +212,7 @@ open class KeyValueStorage<T : Any>(
                 offset = offset,
             )
             .asFlow()
+            .onEach { updateReadAt() }
             .mapToList(config.dispatcher)
             .map { list ->
                 if (list.isEmpty()) return@map emptyList<T>()
@@ -223,7 +241,9 @@ open class KeyValueStorage<T : Any>(
                 orderBy = orderBy,
                 limit = limit.toLong(),
                 offset = offset.toLong()
-            )
+            ).also {
+                updateReadAt()
+            }
         },
         countQuery = entityQueries.count(entityName, where = where),
         transacter = entityQueries,
@@ -247,8 +267,9 @@ open class KeyValueStorage<T : Any>(
      * @see delete
      * @see deleteAll
      */
-    suspend fun deleteByKey(key: String) {
+    suspend fun deleteByKey(key: String) = transaction {
         entityQueries.delete(entityName, entityKey = key)
+        updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: key.hashCode())
     }
 
     /**
@@ -260,8 +281,9 @@ open class KeyValueStorage<T : Any>(
      * @see deleteAll
      * @see deleteByKey
      */
-    suspend fun delete(where: Where<T>? = null) {
+    suspend fun delete(where: Where<T>? = null) = transaction {
         entityQueries.delete(entityName, where = where)
+        updateWriteAt(currentCoroutineContext()[RequestHash.Key]?.hash ?: where.hashCode())
     }
 
     fun count(): Flow<Int> {
@@ -270,6 +292,17 @@ open class KeyValueStorage<T : Any>(
             .mapToOne(config.dispatcher)
     }
 
+
+    /**
+     * Metadata for the entity, this will tell you the last time
+     * the entity store was read and written to, useful for cache invalidation.
+     */
+    fun metadata(): Flow<Metadata> = metadataQueries
+        .selectByEntityName(entityName)
+        .asFlow()
+        .mapToOneNotNull(config.dispatcher)
+        .distinctUntilChanged()
+
     private fun <T : Any> Entity?.deserialize(): T? {
         this ?: return null
         return try {
@@ -285,6 +318,27 @@ open class KeyValueStorage<T : Any>(
         }
     }
 
+    private fun updateReadAt() {
+        scope.launch(config.dispatcher) {
+            metadataQueries.upsertRead(entityName, Clock.System.now())
+        }
+    }
+
+    private val updateWriteHashes = mutableSetOf<Int>()
+
+    /**
+     * Will run after the transaction is committed. This way inside of multiple inserts we only
+     * update the write_at once.
+     */
+    private fun TransactionCallbacks.updateWriteAt(requestHash: Int) {
+        if (requestHash in updateWriteHashes) return
+        updateWriteHashes.add(requestHash)
+        afterCommit {
+            updateWriteHashes.remove(requestHash)
+            scope.launch { metadataQueries.upsertWrite(entityName, Clock.System.now()) }
+        }
+    }
+
     data class Config(
         val deserializePolicy: DeserializePolicy = DeserializePolicy.ERROR,
         val dispatcher: CoroutineDispatcher = Dispatchers.Default,
@@ -311,13 +365,15 @@ open class KeyValueStorage<T : Any>(
 inline fun <reified T : Any> keyValueStorage(
     entityName: String,
     entityQueries: EntityQueries,
+    metadataQueries: MetadataQueries,
     scope: CoroutineScope,
     serializer: SqkonSerializer = KotlinSqkonSerializer(),
     config: KeyValueStorage.Config = KeyValueStorage.Config(),
 ): KeyValueStorage<T> {
     return KeyValueStorage(
         entityName = entityName,
         entityQueries = entityQueries,
+        metadataQueries = metadataQueries,
         scope = scope,
         type = typeOf<T>(),
         serializer = serializer,

library/src/commonMain/kotlin/com/mercury/sqkon/db/MetadataQueries.kt

@@ -0,0 +1,17 @@
+package com.mercury.sqkon.db
+
+import app.cash.sqldelight.db.SqlDriver
+import com.mercury.sqkon.db.adapters.InstantColumnAdapter
+
+/**
+ * Factory method to create [MetadataQueries] instance
+ */
+internal fun MetadataQueries(driver: SqlDriver): MetadataQueries {
+    return MetadataQueries(
+        driver = driver,
+        metadataAdapter = Metadata.Adapter(
+            lastWriteAtAdapter = InstantColumnAdapter(),
+            lastReadAtAdapter = InstantColumnAdapter(),
+        )
+    )
+}

library/src/commonMain/kotlin/com/mercury/sqkon/db/Sqkon.kt

@@ -24,6 +24,7 @@ import kotlinx.serialization.json.Json
  */
 class Sqkon internal constructor(
     @PublishedApi internal val entityQueries: EntityQueries,
+    @PublishedApi internal val metadataQueries: MetadataQueries,
     @PublishedApi internal val scope: CoroutineScope,
     json: Json = SqkonJson {},
     @PublishedApi
@@ -46,7 +47,7 @@ class Sqkon internal constructor(
         name: String,
         config: KeyValueStorage.Config = this.config,
     ): KeyValueStorage<T> {
-        return keyValueStorage<T>(name, entityQueries, scope, serializer, config)
+        return keyValueStorage<T>(name, entityQueries, metadataQueries, scope, serializer, config)
     }
 
 }

library/src/commonMain/kotlin/com/mercury/sqkon/db/SqkonDatabaseDriver.kt

@@ -5,8 +5,3 @@ import app.cash.sqldelight.db.SqlDriver
 internal expect class DriverFactory {
     fun createDriver(): SqlDriver
 }
-
-internal fun createEntityQueries(driverFactory: DriverFactory): EntityQueries {
-    val driver = driverFactory.createDriver()
-    return EntityQueries(driver)
-}

library/src/commonMain/kotlin/com/mercury/sqkon/db/adapters/InstantColumnAdapter.kt

@@ -0,0 +1,14 @@
+package com.mercury.sqkon.db.adapters
+
+import app.cash.sqldelight.ColumnAdapter
+import kotlinx.datetime.Instant
+
+internal class InstantColumnAdapter : ColumnAdapter<Instant, Long> {
+    override fun decode(databaseValue: Long): Instant {
+        return Instant.fromEpochMilliseconds(databaseValue)
+    }
+
+    override fun encode(value: Instant): Long {
+        return value.toEpochMilliseconds()
+    }
+}

library/src/commonMain/kotlin/com/mercury/sqkon/db/utils/RequestHash.kt

@@ -0,0 +1,8 @@
+package com.mercury.sqkon.db.utils
+
+import kotlin.coroutines.AbstractCoroutineContextElement
+import kotlin.coroutines.CoroutineContext
+
+internal class RequestHash(val hash: Int) : AbstractCoroutineContextElement(Key) {
+    companion object Key : CoroutineContext.Key<RequestHash>
+}