fix(level-hash/persistent): auto-expand values file in case of overflow

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/DataIO.kt

@@ -55,6 +55,13 @@ interface RandomAccessIO {
    */
   fun position(position: Long)
 
+  /**
+   * Try to move to the given position.
+   *
+   * @return `true` if the position was moved successfully, `false` otherwise.
+   */
+  fun tryPosition(position: Long): Boolean
+
   /**
    * Seek the position by given count.
    */

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt

@@ -17,6 +17,8 @@
 
 package com.itsaky.androidide.levelhash
 
+import androidx.annotation.FloatRange
+
 /**
  * Level hash is a write-optimized and high-performance hashing index scheme with cost-efficient
  * resizing and low-overhead consistency guarantee for persistent memory.
@@ -136,6 +138,17 @@ interface LevelHash<K : Any, V : Any?> : AutoCloseable {
    */
   fun expand(addtionalLevelSize: Int): Boolean
 
+  /**
+   * Get the load factor of the level hash.
+   *
+   * @return The load factor. Load factor is a measure of how full the level
+   * hash is. It is calculated as the ratio of the number of occupied slots to
+   * the total number of slots in the level hash. Return value lies between 0.0
+   * and 1.0.
+   */
+  @FloatRange(from = 0.0, to = 1.0)
+  fun loadFactor(): Float
+
   /**
    * The two levels in a [LevelHash].
    */

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/AbstractLevelHash.kt

@@ -102,6 +102,10 @@ internal abstract class AbstractLevelHash<K : Any, V : Any?> internal constructo
     return getSlot(level.index, bucketIdx, slotIdx)
   }
 
+  override fun loadFactor(): Float {
+    return levelItemCounts.sum() / totalSlotCount.toFloat()
+  }
+
   override fun get(key: K): V? {
     return findSlot(key)?.value
   }
@@ -186,7 +190,6 @@ internal abstract class AbstractLevelHash<K : Any, V : Any?> internal constructo
         val emptySlot = getSlot(Level.TOP, sidx, emptyLocation)
         emptySlot.reset(key, value)
         levelItemCounts[Level.TOP.index]++
-
         return true
       }
     }
@@ -201,7 +204,6 @@ internal abstract class AbstractLevelHash<K : Any, V : Any?> internal constructo
       slot.reset(null, null)
       return oldValue
     }
-
     return null
   }
 

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt

@@ -20,10 +20,8 @@ package com.itsaky.androidide.levelhash.internal
 import com.itsaky.androidide.levelhash.DataExternalizer
 import com.itsaky.androidide.levelhash.LevelHash.ResizeFailure
 import com.itsaky.androidide.levelhash.seekInt
-import com.itsaky.androidide.levelhash.seekShort
 import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_INT
 import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_LONG
-import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_SHORT
 import com.itsaky.androidide.levelhash.util.FileUtils
 import com.itsaky.androidide.levelhash.util.MappedRandomAccessIO
 import org.slf4j.LoggerFactory
@@ -98,7 +96,7 @@ private val logger = LoggerFactory.getLogger(PersistentLevelHashIO::class.java)
  *    value values[];
  *
  *    value {
- *      u16 entry_size;
+ *      u32 entry_size;
  *      u32 key_size;
  *      u8 key[key_size];
  *      u32 value_size;
@@ -165,10 +163,11 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
   // even multiple threads, the current position cannot be determined
 
   internal val metaIo =
-    PersistentMetaIO(File(indexFile.parentFile, "${indexFile.name}._meta"), levelSize, bucketSize)
+    PersistentMetaIO(File(indexFile.parentFile, "${indexFile.name}._meta"),
+      levelSize, bucketSize)
   private val keymapFile = File(indexFile.parentFile, "${indexFile.name}._i")
   private val raKeymapFile by lazy { RandomAccessFile(keymapFile, "rw") }
-  private val raIndexFile by lazy { RandomAccessFile(indexFile, "rw") }
+  private val raValuesFile by lazy { RandomAccessFile(indexFile, "rw") }
 
   private val keymapIo = MappedRandomAccessIO()
   private var valIo = MappedRandomAccessIO()
@@ -192,14 +191,8 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
       magicNumber = VALUES_MAGIC_NUMBER)
 
     // the header region is not memory mapped
-    var buffer = raIndexFile.channel.map(FileChannel.MapMode.READ_WRITE,
-      VALUES_HEADER_SIZE_BYTES, metaIo.valuesFileSize)
-    valIo.reset(buffer, 0L, metaIo.valuesFileSize)
-
-    val keymapSize = metaIo.keymapSize
-    buffer = raKeymapFile.channel.map(FileChannel.MapMode.READ_WRITE,
-      KEYMAP_HEADER_SIZE_BYTES, keymapSize)
-    keymapIo.reset(buffer, 0L, keymapSize)
+    valuesRemap(size = metaIo.valuesFileSize)
+    keymapResize(newSize = metaIo.keymapSize)
   }
 
   private fun realValueOffset(offset: Long) = VALUES_HEADER_SIZE_BYTES + offset
@@ -207,11 +200,45 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
   private fun realKeymapOffset(offset: Long) = KEYMAP_HEADER_SIZE_BYTES + offset
 
   private fun valuesDeallocate(offset: Long, len: Long) =
-    FileUtils.deallocate(raIndexFile, realValueOffset(offset), len)
+    FileUtils.deallocate(raValuesFile, realValueOffset(offset), len)
 
   private fun keymapDeallocate(offset: Long, len: Long) =
     FileUtils.deallocate(raKeymapFile, realKeymapOffset(offset), len)
 
+  /**
+   * Resize the values file so that its size becomes [newSize] in bytes. This is
+   * risky and should be used carefully. Resizing the file incorrectly may lead
+   * to data loss. The [valIo] is reset to position 0 after this operation.
+   *
+   * @param newSize The new size of the values file, in bytes.
+   */
+  private fun valuesResize(newSize: Long)  {
+    metaIo.valuesFileSize = newSize
+    raValuesFile.setLength(newSize)
+    valuesRemap(newSize)
+  }
+
+  private fun keymapResize(newSize: Long)  {
+    if (raKeymapFile.length() != newSize) {
+      raKeymapFile.setLength(newSize)
+    }
+    keymapRemap(newSize)
+  }
+
+  private fun valuesRemap(size: Long, offset: Long = VALUES_HEADER_SIZE_BYTES) =
+    fileRemap(raValuesFile, valIo, size, offset)
+
+  private fun keymapRemap(size: Long, offset: Long = KEYMAP_HEADER_SIZE_BYTES) =
+    fileRemap(raKeymapFile, keymapIo, size, offset)
+
+  private fun fileRemap(file: RandomAccessFile, io: MappedRandomAccessIO,
+                        size: Long, offset: Long = 0L
+  ) {
+    val buffer = file.channel.map(FileChannel.MapMode.READ_WRITE, offset, size)
+    io.close()
+    io.reset(buffer, 0L, size)
+  }
+
   private fun slotAddress(
     levelIdx: Int,
     bucketIdx: Int,
@@ -223,8 +250,9 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
       else -> throw IllegalArgumentException("Invalid level index: $levelIdx")
     }
 
-    return levelPos + (KEYMAP_ENTRY_SIZE_BYTES * metaIo.bucketSize * bucketIdx) +
-      (KEYMAP_ENTRY_SIZE_BYTES * slotIdx)
+    return levelPos + // start position of level
+      (KEYMAP_ENTRY_SIZE_BYTES * metaIo.bucketSize * bucketIdx) + // bucket position
+      (KEYMAP_ENTRY_SIZE_BYTES * slotIdx) // slot position in bucket
   }
 
   /**
@@ -269,7 +297,9 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     slotIdx: Int,
   ): Boolean {
     // 0 == no value entry
-    return valueAddressForPos(levelNum, bucketIdx, slotIdx) > 0L
+    return valueAddressForPos(levelNum, bucketIdx, slotIdx).let { valAddr ->
+      valAddr > 0L && valIo.tryPosition(valAddr - 1) && valIo.readInt() > 0
+    }
   }
 
   /**
@@ -286,7 +316,7 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
       return null
     }
 
-    if (valIo.readUnsignedShort() <= 0L) {
+    if (valIo.readInt() <= 0L) {
       // entry size is 0, so the slot is empty
       return null
     }
@@ -361,6 +391,8 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     return value
   }
 
+
+
   fun writeEntry(
     levelNum: Int,
     bucketIdx: Int,
@@ -368,7 +400,6 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     key: K?,
     value: V?,
   ) {
-    // TODO: expand values file in case of buffer overflow
     val slotAddr = slotAddress(levelNum, bucketIdx, slotIdx)
 
     if (key == null) {
@@ -382,17 +413,23 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     val (isUpdate, existingEntrySize) = run {
       if (existingValueAddr > 0L) {
         valIo.position(existingValueAddr - 1)
-        true to valIo.readUnsignedShort().toLong()
+        true to valIo.readInt().toLong()
       } else {
         false to 0L
       }
     }
 
-    val tokenAddr = metaIo.valuesNextEntry
-    valIo.position(tokenAddr)
+    val thisValueAddr = metaIo.valuesNextEntry
+    if (thisValueAddr + 4 * KB_1 > metaIo.valuesFileSize) {
+      valuesResize(thisValueAddr + VALUES_SEGMENT_SIZE_BYTES)
+    }
 
-    // leave space for entry size
-    valIo.seekShort()
+    // valuesResize will reset the position to 0
+    valIo.position(thisValueAddr)
+
+    if (valIo.readInt() > 0) {
+      return
+    }
 
     val keyValPos = valIo.position()
 
@@ -402,25 +439,25 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
 
     val finalAddr = valIo.position()
     val entrySize = finalAddr - keyValPos
-    check(entrySize <= Short.MAX_VALUE) {
+    check(entrySize <= Int.MAX_VALUE) {
       "Entry size is too large: $entrySize"
     }
 
     // then set the token
-    valIo.position(tokenAddr)
-    valIo.writeShort((entrySize and 0xFFFF).toInt())
+    valIo.position(thisValueAddr)
+    valIo.writeInt(entrySize.toInt())
 
     // reset to the final position
     valIo.position(finalAddr)
 
     // then update the address in the keymap
     keymapIo.position(slotAddr)
-    keymapIo.writeLong(tokenAddr + 1)
+    keymapIo.writeLong(thisValueAddr + 1)
 
     metaIo.valuesNextEntry = finalAddr
 
     if (isUpdate) {
-      var size: Long = SIZE_SHORT
+      var size = SIZE_INT
       if (existingEntrySize > 0L) {
         size += existingEntrySize
       }
@@ -437,18 +474,18 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     keymapIo.position(slotAddr)
     val valueAddr = keymapIo.readLong()
 
-    if (valueAddr == 0L) {
-      return null
-    }
-
     // reading this region again will return 0, which is considered
     // a null pointer
     keymapDeallocate(slotAddr, SIZE_LONG)
 
-    valIo.position(valueAddr - 1)
-    valIo.seekShort() // seek over entrySize
+    if (valueAddr == 0L) {
+      return null
+    }
 
-    var entrySize = SIZE_SHORT
+    valIo.position(valueAddr - 1)
+    valIo.seekInt() // seek over entrySize
+
+    var entrySize = SIZE_INT
 
     val keySize = valIo.readInt()
     entrySize += SIZE_INT
@@ -514,7 +551,41 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
   }
 
   override fun close() {
-    closeAndLogErrs(metaIo, keymapIo, valIo, raKeymapFile, raIndexFile)
+    closeAndLogErrs(metaIo, keymapIo, valIo, raKeymapFile, raValuesFile)
+  }
+
+  internal fun dmpSlotAddrs(
+    levelNum: Int,
+    bucketIdx: Int,
+    slotIdx: Int,
+  ) {
+    val sb = StringBuilder()
+    sb.append("level: ")
+    sb.appendLine(levelNum)
+    sb.append("bucket: ")
+    sb.appendLine(bucketIdx)
+    sb.append("slot: ")
+    sb.appendLine(slotIdx)
+
+    val slotAddr = slotAddress(levelNum, bucketIdx, slotIdx)
+    sb.append("keymapAddr: ")
+    appendAddress(sb, slotAddr)
+
+    keymapIo.position(slotAddr)
+
+    val valueAddr = keymapIo.readLong()
+    sb.append("valueAddr: ")
+    appendAddress(sb, valueAddr)
+
+    println(sb.toString())
+  }
+
+  @OptIn(ExperimentalStdlibApi::class)
+  private fun appendAddress(sb: StringBuilder, value: Long) {
+    sb.append(value)
+    sb.append(" [")
+    sb.append(value.toHexString(HexFormat.UpperCase))
+    sb.appendLine("]")
   }
 
   companion object {
@@ -531,6 +602,11 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
 
     private val log = LoggerFactory.getLogger(PersistentLevelHashIO::class.java)
 
+    /**
+     * 1 Kilobyte.
+     */
+    internal const val KB_1 = 1024
+
     /**
      * The number of bytes it takes to store the magic number of the keymap/values
      * file.
@@ -548,10 +624,9 @@ internal class PersistentLevelHashIO<K : Any, V : Any?>(
     internal const val VALUES_HEADER_SIZE_BYTES: Long = MAGIC_NUMBER_SIZE_BYTES
 
     /**
-     * The maximum size of the region of the values file that can be mapped into
-     * memory at a given time.
+     * The size of one segment region in the values file.
      */
-    internal const val VALUES_INITIAL_SIZE_BYTES = 512L * 1024L
+    internal const val VALUES_SEGMENT_SIZE_BYTES = 512L * 1024L
 
     /**
      * The number of bytes used to store the header of the keymap file.

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentMetaIO.kt

@@ -18,12 +18,10 @@
 package com.itsaky.androidide.levelhash.internal
 
 import androidx.collection.MutableLongLongMap
-import com.itsaky.androidide.levelhash.LevelHash
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.KEYMAP_ENTRY_SIZE_BYTES
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_KEYMAP_VERSION
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_VALUES_VERSION
-import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_HEADER_SIZE_BYTES
-import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_INITIAL_SIZE_BYTES
+import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_SEGMENT_SIZE_BYTES
 import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_INT
 import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_LONG
 import org.slf4j.LoggerFactory
@@ -105,7 +103,7 @@ internal class PersistentMetaIO(private val metaFile: File,
       keymapVersion = LEVEL_KEYMAP_VERSION
     }
     if (valuesFileSize == 0L) {
-      valuesFileSize = VALUES_INITIAL_SIZE_BYTES
+      valuesFileSize = VALUES_SEGMENT_SIZE_BYTES
     }
     if (valuesFirstEntry == 0L) {
       valuesFirstEntry = 0

utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/util/MappedRandomAccessIO.kt

@@ -65,6 +65,10 @@ internal class MappedRandomAccessIO : DataInput, DataOutput, RandomAccessIO, Aut
   }
 
   override fun close() {
+    if (!::buffer.isInitialized) {
+      return
+    }
+
     try {
       buffer.unmap()
     } catch (err: Throwable) {
@@ -73,10 +77,23 @@ internal class MappedRandomAccessIO : DataInput, DataOutput, RandomAccessIO, Aut
   }
 
   override fun position(position: Long) {
-    val currentPos = position()
-    if (currentPos == position) return
     require(position >= 0L) { "position must be >= 0, position=$position" }
+    require(position < buffer.limit()) {
+      "position must be < buffer.limit(), position=$position buffer.limit=${buffer.limit()}"
+    }
+    tryPosition(position)
+  }
+
+  override fun tryPosition(position: Long): Boolean {
+    if (position < 0L || position >= buffer.limit()) {
+      return false
+    }
+    val currentPos = position()
+    if (currentPos == position) {
+      return true
+    }
     this.position = this.positionOffset + position
+    return true
   }
 
   override fun position(): Long {

utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt

@@ -106,7 +106,10 @@ class PersistentLevelHashTest {
       for (i in 0..<hash.totalSlotCount) {
         val key = "key${i}"
         val value = "value${i}"
-        assertThat(hash.insert(key, value)).isTrue()
+        if (!hash.insert(key, value)) {
+          println("key: $key, value: $value, load=${hash.loadFactor()}")
+          assert(false)
+        }
       }
     }
   }

utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentMetaIOTest.kt

@@ -20,12 +20,11 @@ package com.itsaky.androidide.levelhash
 import com.google.common.truth.Truth.assertThat
 import com.itsaky.androidide.levelhash.LevelHash.Companion.BUCKET_SIZE_DEFAULT
 import com.itsaky.androidide.levelhash.LevelHash.Companion.LEVEL_SIZE_DEFAULT
-import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.KEYMAP_ENTRY_SIZE_BYTES
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_KEYMAP_VERSION
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_VALUES_VERSION
 import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_HEADER_SIZE_BYTES
-import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_INITIAL_SIZE_BYTES
+import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_SEGMENT_SIZE_BYTES
 import com.itsaky.androidide.levelhash.internal.PersistentMetaIO
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -62,7 +61,7 @@ class PersistentMetaIOTest {
     assertThat(io.keymapVersion).isEqualTo(LEVEL_KEYMAP_VERSION)
     assertThat(io.valuesFirstEntry).isEqualTo(VALUES_HEADER_SIZE_BYTES)
     assertThat(io.valuesNextEntry).isEqualTo(VALUES_HEADER_SIZE_BYTES)
-    assertThat(io.valuesFileSize).isEqualTo(VALUES_INITIAL_SIZE_BYTES)
+    assertThat(io.valuesFileSize).isEqualTo(VALUES_SEGMENT_SIZE_BYTES)
     assertThat(io.levelSize).isEqualTo(LEVEL_SIZE_DEFAULT)
     assertThat(io.bucketSize).isEqualTo(BUCKET_SIZE_DEFAULT)
     assertThat(io.l0Addr).isEqualTo(0L)