diff --git a/utilities/level-hash/build.gradle.kts b/utilities/level-hash/build.gradle.kts index dc8ed19a2..0a7f2746f 100644 --- a/utilities/level-hash/build.gradle.kts +++ b/utilities/level-hash/build.gradle.kts @@ -28,7 +28,11 @@ android { } tasks.withType { - jvmArgs("--add-opens", "java.base/java.io=ALL-UNNAMED") + jvmArgs( + "--add-opens", "java.base/java.io=ALL-UNNAMED", // For FileDescriptor.fd + "--add-opens", "java.base/java.nio=ALL-UNNAMED", // For DirectByteBuffer.cleaner + "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED", // For Cleaner.clean + ) } dependencies { diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt index faf1405ce..eff3e96e3 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt @@ -24,7 +24,7 @@ package com.itsaky.androidide.levelhash * @param K The key type. * @param V The value type. */ -interface LevelHash { +interface LevelHash : AutoCloseable { /** * The size of the level size. This is `log2(addrCapacity)`. diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/BinaryFileUtils.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/BinaryFileUtils.kt index 4cefa311d..c366eaff0 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/BinaryFileUtils.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/BinaryFileUtils.kt @@ -17,10 +17,9 @@ package com.itsaky.androidide.levelhash.internal -import com.itsaky.androidide.levelhash.internal.PersistentKeymapIO.Companion.KEYMAP_HEADER_SIZE_BYTES -import com.itsaky.androidide.levelhash.internal.PersistentKeymapIO.Companion.KEYMAP_MAGIC_NUMBER import org.slf4j.LoggerFactory import java.io.DataInputStream +import java.io.EOFException import java.io.File import java.io.RandomAccessFile @@ -31,48 +30,61 @@ internal object BinaryFileUtils { private val log = LoggerFactory.getLogger(BinaryFileUtils::class.java) - fun initSparseFile(file: File, magicNumber: Long, maxLength: Long, + fun initSparseFile(file: File, magicNumber: Long? = null, conf: (raf: RandomAccessFile) -> Unit = {} ) { - val raf = RandomAccessFile(file, "rw") if (!file.exists()) { - doInit(file, raf, magicNumber, maxLength, conf) + doInit(file, magicNumber, conf) return } + val canonicalPath = file.canonicalPath + if (file.isDirectory) { - file.deleteRecursively() - } else if (file.isFile && file.length() < KEYMAP_HEADER_SIZE_BYTES) { - check(file.delete()) { "Failed to delete existing index file" } - } else { + check(file.deleteRecursively()) { + "Failed to delete existing index directory: $canonicalPath" + } + } else if (magicNumber != null) { DataInputStream(file.inputStream()).use { dis -> - val magic = dis.readLong() + val magic = try { + dis.readLong() + } catch (e: EOFException) { + 0L + } if (magic != magicNumber) { // mismatched magic number // delete file and create new one log.warn( - "Magic number mismatch for index: {}. Deleting existing file...", - file.canonicalPath) - check(file.delete()) { "Failed to delete existing index file" } + "Magic number mismatch for file: {}. Deleting existing file...", + canonicalPath) + + check(file.delete()) { + "Failed to delete existing index file; $canonicalPath" + } } } } - doInit(file, raf, magicNumber, maxLength, conf) + doInit(file, magicNumber, conf) } - private inline fun doInit(file: File, raf: RandomAccessFile, magic: Long, - maxLength: Long, + private inline fun doInit(file: File, magic: Long?, conf: (raf: RandomAccessFile) -> Unit = {} ) { - check(file.createNewFile()) { - "Failed to create keymap file: ${file.canonicalPath}" + if (!file.exists()) { + file.parentFile!!.mkdirs() + check(file.createNewFile()) { + "Failed to create file: ${file.canonicalPath}" + } } -// raf.setLength(maxLength) + RandomAccessFile(file, "rw").use { raf -> + if (magic != null) { + raf.seek(0) + raf.writeLong(magic) + } - raf.seek(0) - raf.writeLong(magic) - raf.use(conf) + conf(raf) + } } } \ No newline at end of file diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelHash.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelHash.kt index 98a1a1b58..084a748d9 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelHash.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelHash.kt @@ -123,4 +123,8 @@ internal class InMemoryLevelHash( this.levels[Level.TOP.index] = checkNotNull(this.interimLevel) this.interimLevel = null } + + override fun close() { + clear() + } } \ No newline at end of file diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/KeymapSegment.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/KeymapSegment.kt index 475267296..52a616195 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/KeymapSegment.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/KeymapSegment.kt @@ -96,6 +96,7 @@ internal class KeymapSegment( } catch (err: Throwable) { log.error("Failed to unmap keymap segment. file={}, offset={}, size={}", path, segmentOffset, segmentLength, err) + err.printStackTrace() return false } } diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentKeymapIO.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentKeymapIO.kt index 1c2cdd184..2ccae2626 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentKeymapIO.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentKeymapIO.kt @@ -54,8 +54,7 @@ internal class PersistentKeymapIO(indexFile: File init { BinaryFileUtils.initSparseFile( file = keymapFile, - magicNumber = KEYMAP_MAGIC_NUMBER, - maxLength = KEYMAP_HEADER_SIZE_BYTES + KEYMAP_SEGMENT_SIZE_BYTES * KEYMAP_MAX_SEGMENTS + magicNumber = KEYMAP_MAGIC_NUMBER ) { raf -> var position = MAGIC_NUMBER_SIZE_BYTES raf.seek(position) diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHash.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHash.kt index 13f56cd6b..0c77ae89b 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHash.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHash.kt @@ -74,6 +74,10 @@ internal class PersistentLevelHash( this.io.clear() } + override fun close() { + io.close() + } + private var interimBucketCount = -1 override fun prepareExpansion(bucketCount: Int) { check(this.interimBucketCount == -1) diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt index ec4f4a917..d254f4cc7 100644 --- a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt @@ -69,8 +69,6 @@ private val logger = LoggerFactory.getLogger(PersistentLevelHashIO::class.java) * ``` * keymap { * u64 magic_number; - * u32 segment_count; - * u64 segment_size; * segment segments[segment_count]; * * segment { @@ -96,8 +94,6 @@ private val logger = LoggerFactory.getLogger(PersistentLevelHashIO::class.java) * ``` * values { * u64 magic_number; - * u64 first_entry; - * u64 next_entry; * value values[]; * * value { @@ -110,6 +106,47 @@ private val logger = LoggerFactory.getLogger(PersistentLevelHashIO::class.java) * } * } * ``` + * + * The `values` structure constains fields : + * - `magic_number` - A magic number that is used to identify the values file. + * - `values` - The value entries. + * + * Each `value` entry contains fields : + * - `is_occupied` - A 1-byte boolean value (0=false or 1=true) that indicates + * if the entry is occupied or not. + * - `entry_size` - The size of the entry in bytes (excluding the size of + * `is_occupied` and `entry_size` itself). + * - `key_size` - The size of the key in bytes. + * - `key` - The key of `key_size` bytes. + * - `value_size` - The size of the value in bytes. + * - `value` - The value of `value_size` bytes. + * + * + * ## Metadata + * + * Structure of the metadata file : + * + * ``` + * meta { + * u32 values_version; + * u32 keymap_version; + * u64 values_first_entry; + * u64 values_next_entry; + * u64 values_file_size_bytes; + * u32 km_segment_count; + * u64 km_segment_size; + * } + * ``` + * + * The `meta` structure contains the fields : + * - `values_version` - The version of the values file. + * - `keymap_version` - The version of the keymap file. + * - `values_first_entry` - The address of the first entry in the values file. + * - `values_next_entry` - The address of the next entry in the values file. + * - `values_file_size_bytes` - The (occupied) size of the values file in bytes. + * - `km_segment_count` - The number of segments in the keymap file. + * - `km_segment_size` - The size of each segment in the keymap file. + * * @author Akash Yadav */ internal class PersistentLevelHashIO( @@ -124,44 +161,25 @@ internal class PersistentLevelHashIO( // as the values file is randomly accessed, and by multiple functions, or // even multiple threads, the current position cannot be determined + private val metaIo = + PersistentMetaIO(File(indexFile.parentFile, "${indexFile.name}._meta")) private val keymapIo = PersistentKeymapIO(indexFile) - - private val metaFile = File(indexFile.parentFile, "${indexFile.name}._meta") - private val raMetaFile by lazy { RandomAccessFile(metaFile, "rw") } private val raIndexFile by lazy { RandomAccessFile(indexFile, "rw") } - private var valFirstEntryPos = 0L - private var valNxtEntryPos = 0L - private var valIdxSize = 0L private var valIo = MappedRandomAccessIO() init { BinaryFileUtils.initSparseFile( file = indexFile, - magicNumber = VALUES_MAGIC_NUMBER, - maxLength = VALUES_HEADER_SIZE_BYTES + valIdxSize - ) { raf -> - var position = MAGIC_NUMBER_SIZE_BYTES - raf.seek(position) - raf.writeLong(0) // first entry - position += SIZE_LONG - - raf.seek(position) - raf.writeLong(0) // next entry - } - - // TODO: Load the below values from file when the file already exists - - valNxtEntryPos = 0 - valFirstEntryPos = VALUES_HEADER_SIZE_BYTES - valIdxSize = VALUES_MAX_MAP_SIZE_BYTES + magicNumber = VALUES_MAGIC_NUMBER + ) // the header region is not memory mapped log.debug("mmap values file: offset={}, size={}", VALUES_HEADER_SIZE_BYTES, - valIdxSize) + metaIo.valuesFileSize) val buffer = raIndexFile.channel.map(FileChannel.MapMode.READ_WRITE, - VALUES_HEADER_SIZE_BYTES, valIdxSize) - valIo.reset(buffer, 0L, valIdxSize) + VALUES_HEADER_SIZE_BYTES, metaIo.valuesFileSize) + valIo.reset(buffer, 0L, metaIo.valuesFileSize) } private fun getValuesRealOffset(offset: Long) = @@ -343,7 +361,7 @@ internal class PersistentLevelHashIO( } } - val tokenAddr = valNxtEntryPos + val tokenAddr = metaIo.valuesNextEntry valIo.position(tokenAddr) // leave space for isOccupied token and entry size @@ -374,7 +392,7 @@ internal class PersistentLevelHashIO( segment.position(positionInSegment) segment.writeLong(tokenAddr + 1) - valNxtEntryPos = finalAddr + metaIo.valuesNextEntry = finalAddr if (isUpdate) { var size: Long = SIZE_BOOLEAN + SIZE_SHORT @@ -452,7 +470,7 @@ internal class PersistentLevelHashIO( fun clear() { keymapIo.deallocate(KEYMAP_HEADER_SIZE_BYTES, KEYMAP_SEGMENT_SIZE_BYTES * KEYMAP_MAX_SEGMENTS) - valuesDeallocate(VALUES_HEADER_SIZE_BYTES, valIdxSize) + valuesDeallocate(VALUES_HEADER_SIZE_BYTES, metaIo.valuesFileSize) } @Suppress("UNUSED", "UNUSED_PARAMETER") @@ -500,15 +518,15 @@ internal class PersistentLevelHashIO( } try { - raIndexFile.close() + metaIo.close() } catch (e: Throwable) { - logger.error("Failed to close index file", e) + log.error("Failed to close meta file", e) } try { - raMetaFile.close() + raIndexFile.close() } catch (e: Throwable) { - logger.error("Failed to close meta file", e) + logger.error("Failed to close index file", e) } } @@ -531,15 +549,16 @@ internal class PersistentLevelHashIO( /** * The size of the header (bytes) in the values file. */ - internal const val VALUES_HEADER_SIZE_BYTES: Long = - MAGIC_NUMBER_SIZE_BYTES + SIZE_LONG + // first entry address - SIZE_LONG // next entry address + internal const val VALUES_HEADER_SIZE_BYTES: Long = MAGIC_NUMBER_SIZE_BYTES /** * The maximum size of the region of the values file that can be mapped into * memory at a given time. */ - internal const val VALUES_MAX_MAP_SIZE_BYTES = 512L * 1024L; + internal const val VALUES_INITIAL_SIZE_BYTES = 512L * 1024L + + internal const val LEVEL_VALUES_VERSION = 1 + internal const val LEVEL_KEYMAP_VERSION = 1 } } \ No newline at end of file diff --git a/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentMetaIO.kt b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentMetaIO.kt new file mode 100644 index 000000000..5708d27dc --- /dev/null +++ b/utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentMetaIO.kt @@ -0,0 +1,248 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.levelhash.internal + +import androidx.collection.MutableLongLongMap +import com.itsaky.androidide.levelhash.internal.PersistentKeymapIO.Companion.KEYMAP_SEGMENT_SIZE_BYTES +import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_KEYMAP_VERSION +import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.LEVEL_VALUES_VERSION +import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_HEADER_SIZE_BYTES +import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO.Companion.VALUES_INITIAL_SIZE_BYTES +import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_INT +import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_LONG +import org.slf4j.LoggerFactory +import java.io.EOFException +import java.io.File +import java.io.RandomAccessFile + +/** + * @author Akash Yadav + */ +internal class PersistentMetaIO(private val metaFile: File +) : AutoCloseable { + + private val raFile by lazy { RandomAccessFile(metaFile, "rw") } + private val cache = MutableLongLongMap(META__FIELD_COUNT) + + var valuesVersion: Int + get() { + return readInt(VAL__VERSION__OFFSET) + } + set(value) { + writeInt(VAL__VERSION__OFFSET, value) + } + + var keymapVersion: Int + get() { + return readInt(KM__VERSION__OFFSET) + } + set(value) { + writeInt(KM__VERSION__OFFSET, value) + } + + var valuesFirstEntry: Long + get() { + return readLong(VAL__FIRST_ENTRY__OFFSET) + } + set(value) { + writeLong(VAL__FIRST_ENTRY__OFFSET, value) + } + + var valuesNextEntry: Long + get() { + return readLong(VAL__NEXT_ENTRY__OFFSET) + } + set(value) { + writeLong(VAL__NEXT_ENTRY__OFFSET, value) + } + + var valuesFileSize: Long + get() { + return readLong(VAL__VAL_SIZE__OFFSET) + } + set(value) { + writeLong(VAL__VAL_SIZE__OFFSET, value) + } + + var keymapSegmentCount: Int + get() { + return readInt(KM__SEGMENT_COUNT__OFFSET) + } + set(value) { + writeInt(KM__SEGMENT_COUNT__OFFSET, value) + } + + var keymapSegmentSize: Long + get() { + return readLong(KM__SEGMENT_SIZE__OFFSET) + } + set(value) { + writeLong(KM__SEGMENT_SIZE__OFFSET, value) + } + + init { + BinaryFileUtils.initSparseFile(metaFile) + raFile.setLength(META__SIZE_BYTES) + + // initialize default state + if (valuesVersion == 0) { + valuesVersion = LEVEL_VALUES_VERSION + } + if (keymapVersion == 0) { + keymapVersion = LEVEL_KEYMAP_VERSION + } + if (valuesFileSize == 0L) { + valuesFileSize = VALUES_INITIAL_SIZE_BYTES + } + if (valuesFirstEntry == 0L) { + valuesFirstEntry = VALUES_HEADER_SIZE_BYTES + } + if (valuesNextEntry == 0L) { + valuesNextEntry = valuesFirstEntry + } + if (keymapSegmentCount == 0) { + // initially, there should be at least one segment + keymapSegmentCount = 1 + } + if (keymapSegmentSize == 0L) { + keymapSegmentSize = KEYMAP_SEGMENT_SIZE_BYTES + } + } + + private fun readInt(position: Long): Int = fromCacheOrRead(position) { + readOrDefault(0) { raFile.readInt() } + } + + private fun writeInt(position: Long, value: Int) = + writeAndCache(position, value.toLong()) { + raFile.writeInt(value) + } + + private fun readLong(position: Long): Long = fromCacheOrRead(position) { + readOrDefault(0L) { raFile.readLong() } + } + + private fun writeLong(position: Long, value: Long) = + writeAndCache(position, value) { + raFile.writeLong(value) + } + + private inline fun fromCacheOrRead(position: Long, + read: () -> T + ): T { + val cached = cache.getOrDefault(position, 0L) + if (cached != 0L) { + return cast(cached) + } + + raFile.seek(position) + return read().also { value -> cache(position, value.toLong()) } + } + + private inline fun writeAndCache(position: Long, value: Long, + writer: () -> Unit + ) { + raFile.seek(position) + writer() + cache(position, value) + } + + private fun cache(position: Long, value: Long) { + cache[position] = value + } + + override fun close() { + try { + cache.clear() + raFile.close() + } catch (e: Exception) { + log.error("Failed to close meta file", e) + } + } + + private inline fun readOrDefault(default: T, + read: () -> T + ): T { + return try { + read() + } catch (e: EOFException) { + default + } + } + + private inline fun cast(v: Long): T { + if (T::class == Long::class) { + return v as T + } + + return when { + T::class == Byte::class -> v.toByte() as T + T::class == Short::class -> v.toShort() as T + T::class == Int::class -> v.toInt() as T + T::class == Float::class -> v.toFloat() as T + T::class == Double::class -> v.toDouble() as T + else -> throw UnsupportedOperationException("Unsupported data type") + } + } + + companion object { + + private val log = LoggerFactory.getLogger(PersistentMetaIO::class.java) + + private const val VAL__VERSION__OFFSET = 0L + private const val VAL__VERSION__SIZE_BYTES = SIZE_INT + + private const val KM__VERSION__OFFSET = + VAL__VERSION__OFFSET + VAL__VERSION__SIZE_BYTES + private const val KM__VERSION__SIZE_BYTES = SIZE_INT + + private const val VAL__FIRST_ENTRY__OFFSET = + KM__VERSION__OFFSET + KM__VERSION__SIZE_BYTES + private const val VAL__FIRST_ENTRY__SIZE_BYTES = SIZE_LONG + + private const val VAL__NEXT_ENTRY__OFFSET = + VAL__FIRST_ENTRY__OFFSET + VAL__FIRST_ENTRY__SIZE_BYTES + private const val VAL__NEXT_ENTRY__SIZE_BYTES = SIZE_LONG + + private const val VAL__VAL_SIZE__OFFSET = + VAL__NEXT_ENTRY__OFFSET + VAL__NEXT_ENTRY__SIZE_BYTES + private const val VAL__VAL_SIZE__SIZE_BYTES = SIZE_LONG + + private const val KM__SEGMENT_COUNT__OFFSET = + VAL__VAL_SIZE__OFFSET + VAL__VAL_SIZE__SIZE_BYTES + private const val KM__SEGMENT_COUNT__SIZE_BYTES = SIZE_INT + + private const val KM__SEGMENT_SIZE__OFFSET = + KM__SEGMENT_COUNT__OFFSET + KM__SEGMENT_COUNT__SIZE_BYTES + private const val KM__SEGMENT_SIZE__SIZE_BYTES = SIZE_LONG + + // TODO: The properties below must be updated when a new field is added to the meta file + + /** + * The number of fields in the meta file. + */ + private const val META__FIELD_COUNT = 7 + + /** + * The size of the meta file in bytes. + */ + // Offset of the last field + its size + private const val META__SIZE_BYTES = + KM__SEGMENT_SIZE__OFFSET + KM__SEGMENT_SIZE__SIZE_BYTES + } +} \ No newline at end of file diff --git a/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt b/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt index 494ed42ef..80381db30 100644 --- a/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt +++ b/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt @@ -41,16 +41,17 @@ class PersistentLevelHashTest { private inline fun persistentStringHash(fileName: String, valueExternalizer: DataExternalizer, + createNew: Boolean = true, conf: PersistentHashBuilder.() -> Unit = {} ): LevelHash { - val indexFile = File("./build/sample-index/${fileName}.storage") - if (indexFile.parentFile!!.exists()) { + val indexFile = + File("build/level-index/hash-${fileName}/${fileName}.storage") + if (indexFile.parentFile!!.exists() && createNew) { indexFile.parentFile!!.deleteRecursively() + } else { + indexFile.parentFile!!.mkdirs() } - indexFile.parentFile!!.mkdirs() - indexFile.createNewFile() - val hash = LevelHash.persistentHashBuilder() .keyExternalizer(DataExternalizers.STRING) .valueExternalizer(valueExternalizer) @@ -66,45 +67,49 @@ class PersistentLevelHashTest { } private inline fun persistentStringHash(fileName: String, + createNew: Boolean = true, conf: PersistentHashBuilder.() -> Unit = {} ): LevelHash { - return persistentStringHash(fileName, DataExternalizers.STRING, conf) + return persistentStringHash(fileName, DataExternalizers.STRING, createNew, + conf) } @Test fun testInsertion() { - val hash = persistentStringHash("insert") + persistentStringHash("insert").use { hash -> + val key = "key" + val value = "value" + assertThat(hash.insert(key, value)).isTrue() - val key = "key" - val value = "value" - assertThat(hash.insert(key, value)).isTrue() - - val slot = hash.findSlot(key) - assertThat(slot).isNotNull() - assertThat(slot!!.value).isEqualTo(value) + val slot = hash.findSlot(key) + assertThat(slot).isNotNull() + assertThat(slot!!.value).isEqualTo(value) + } } @Test(expected = OverflowException::class) fun testOverflow() { - val hash = persistentStringHash("overflow") { autoExpand(false) } - for (i in 0.. + for (i in 0.. + assertThat(hash.insert("key", "value")).isTrue() + assertThat(hash.findSlot("key")).isNotNull() + assertThat(hash.findSlot("key")?.value).isEqualTo("value") - assertThat(hash.remove("key")).isEqualTo("value") - assertThat(hash.findSlot("key")).isNull() + assertThat(hash.remove("key")).isEqualTo("value") + assertThat(hash.findSlot("key")).isNull() + } } @Test @@ -120,37 +125,62 @@ class PersistentLevelHashTest { @Test(expected = LevelHash.EntryNotFoundException::class) fun `test value update for non-existing entry`() { - val hash = persistentStringHash("update-non-existing-entry") + persistentStringHash("update-non-existing-entry").use { hash -> + assertThat(hash.insert("k", "v")).isTrue() - assertThat(hash.insert("k", "v")).isTrue() + val slot = hash.findSlot("k") + assertThat(slot).isNotNull() + assertThat(slot!!.key).isEqualTo("k") + assertThat(slot.value).isEqualTo("v") - val slot = hash.findSlot("k") - assertThat(slot).isNotNull() - assertThat(slot!!.key).isEqualTo("k") - assertThat(slot.value).isEqualTo("v") - - assertThat(hash.set("kk", "newV")).isNull() + assertThat(hash.set("kk", "newV")).isNull() + } } @Test fun `test value update for existing entry with null value`() { - val hash = persistentStringHash( - fileName = "update-existing-with-null-value", + persistentStringHash(fileName = "update-existing-with-null-value", + valueExternalizer = DataExternalizers.STRING.nullable()).use { hash -> + assertThat(hash.insert("k", null)).isTrue() + + var slot = hash.findSlot("k") + assertThat(slot).isNotNull() + assertThat(slot!!.key).isEqualTo("k") + assertThat(slot.value).isNull() + + assertThat(hash.set("k", "newV")).isNull() + + slot = hash.findSlot("k") + assertThat(slot).isNotNull() + assertThat(slot!!.key).isEqualTo("k") + assertThat(slot.value).isEqualTo("newV") + } + + } + + @Test + fun `test existing level init`() { + persistentStringHash( + fileName = "init-existing", valueExternalizer = DataExternalizers.STRING.nullable() - ) + ).use { hash -> + hash.insert("key", "value") + hash.insert("null", null) + hash.insert("long", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - assertThat(hash.insert("k", null)).isTrue() + assertThat(hash["key"]).isEqualTo("value") + assertThat(hash["null"]).isEqualTo(null) + assertThat(hash["long"]).isEqualTo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + } - var slot = hash.findSlot("k") - assertThat(slot).isNotNull() - assertThat(slot!!.key).isEqualTo("k") - assertThat(slot.value).isNull() - - assertThat(hash.set("k", "newV")).isNull() - - slot = hash.findSlot("k") - assertThat(slot).isNotNull() - assertThat(slot!!.key).isEqualTo("k") - assertThat(slot.value).isEqualTo("newV") + persistentStringHash( + fileName = "init-existing", + valueExternalizer = DataExternalizers.STRING.nullable(), + createNew = false + ).use { hash -> + assertThat(hash["key"]).isEqualTo("value") + assertThat(hash["null"]).isEqualTo(null) + assertThat(hash["long"]).isEqualTo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + } } } diff --git a/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentMetaIOTest.kt b/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentMetaIOTest.kt new file mode 100644 index 000000000..cc0d00092 --- /dev/null +++ b/utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentMetaIOTest.kt @@ -0,0 +1,92 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.levelhash + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.levelhash.internal.PersistentKeymapIO +import com.itsaky.androidide.levelhash.internal.PersistentLevelHashIO +import com.itsaky.androidide.levelhash.internal.PersistentMetaIO +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File + +/** + * @author Akash Yadav + */ +@RunWith(JUnit4::class) +class PersistentMetaIOTest { + + private fun createMetaIo( + name: String, + createNew: Boolean = true, + ): PersistentMetaIO { + val metaFile = File("build/level-hash/meta-${name}/${name}.storage._meta") + if (metaFile.parentFile!!.exists() && createNew) { + metaFile.parentFile!!.deleteRecursively() + } else { + metaFile.parentFile!!.mkdirs() + } + + return PersistentMetaIO(metaFile) + } + + @Test + fun `test meta init with default values`() { + val io = createMetaIo("init-with-default") + assertThat(io.valuesVersion).isEqualTo(PersistentLevelHashIO.LEVEL_VALUES_VERSION) + assertThat(io.keymapVersion).isEqualTo(PersistentLevelHashIO.LEVEL_KEYMAP_VERSION) + assertThat(io.valuesFirstEntry).isEqualTo(PersistentLevelHashIO.VALUES_HEADER_SIZE_BYTES) + assertThat(io.valuesNextEntry).isEqualTo(PersistentLevelHashIO.VALUES_HEADER_SIZE_BYTES) + assertThat(io.valuesFileSize).isEqualTo(PersistentLevelHashIO.VALUES_INITIAL_SIZE_BYTES) + assertThat(io.keymapSegmentCount).isEqualTo(1) + assertThat(io.keymapSegmentSize).isEqualTo(PersistentKeymapIO.KEYMAP_SEGMENT_SIZE_BYTES) + } + + @Test + fun `test meta init with existing file`() { + createMetaIo(name = "init-with-existing").use { io -> + assertThat(io.valuesVersion).isEqualTo(PersistentLevelHashIO.LEVEL_VALUES_VERSION) + assertThat(io.keymapVersion).isEqualTo(PersistentLevelHashIO.LEVEL_KEYMAP_VERSION) + assertThat(io.valuesFirstEntry).isEqualTo(PersistentLevelHashIO.VALUES_HEADER_SIZE_BYTES) + assertThat(io.valuesNextEntry).isEqualTo(PersistentLevelHashIO.VALUES_HEADER_SIZE_BYTES) + assertThat(io.valuesFileSize).isEqualTo(PersistentLevelHashIO.VALUES_INITIAL_SIZE_BYTES) + assertThat(io.keymapSegmentCount).isEqualTo(1) + assertThat(io.keymapSegmentSize).isEqualTo(PersistentKeymapIO.KEYMAP_SEGMENT_SIZE_BYTES) + + io.valuesVersion = 2 + io.keymapVersion = 3 + io.valuesFirstEntry = 100 + io.valuesNextEntry = 200 + io.valuesFileSize = 1024 + + io.keymapSegmentCount = 10 + io.keymapSegmentSize = 1024 + } + + createMetaIo(name = "init-with-existing", createNew = false).use { io -> + assertThat(io.valuesVersion).isEqualTo(2) + assertThat(io.keymapVersion).isEqualTo(3) + assertThat(io.valuesFirstEntry).isEqualTo(100) + assertThat(io.valuesNextEntry).isEqualTo(200) + assertThat(io.valuesFileSize).isEqualTo(1024) + assertThat(io.keymapSegmentCount).isEqualTo(10) + assertThat(io.keymapSegmentSize).isEqualTo(1024) + } + } +} \ No newline at end of file