Studio/AndroidIDE
5
0
Fork 0
mirror of https://gitee.com/zhangsken/AndroidIDE.git synced 2026-10-16 07:49:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add level-hash implementations for persistent index storage

Browse Source
This commit is contained in:
Akash Yadav
2024-07-26 23:41:47 +05:30
parent fac8ac61e1
commit e0e6d50e8f
43 changed files with 3477 additions and 690 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/codeStyles/codeStyleConfig.xml
Generated
+1 -1
View File
@@ -1,5 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
<option name="PREFERRED_PROJECT_CODE_STYLE" value="AndroidIDEStyle" />
</state>
</component>
.idea/compiler.xml
Generated
+1
View File
@@ -271,6 +271,7 @@
<module name="AndroidIDE.treesitter.test" target="1.7" />
<module name="AndroidIDE.treeview" target="17" />
<module name="AndroidIDE.uidesigner" target="17" />
<module name="AndroidIDE.utilities.level-hash.test" target="11" />
<module name="AndroidIDE.xml-inflater" target="17" />
<module name="AndroidIDE.xml.resources-api.test" target="11" />
<module name="AndroidIDE.xml.resources-indexing" target="17" />
.idea/inspectionProfiles/Project_Default.xml
Generated
+1
View File
@@ -6,6 +6,7 @@
</inspection_tool>
<inspection_tool class="KotlinLoggerInitializedWithForeignClass" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="LoggingConditionDisagreesWithLogLevelStatement" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="ReassignedToPlainText" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SpellCheckingInspection" enabled="true" level="INFORMATION" enabled_by_default="true">
<option name="processCode" value="true" />
<option name="processLiterals" value="true" />
core/app/build.gradle.kts
+4 -2
View File
@@ -43,8 +43,6 @@ buildscript {
}
}
android {
namespace = BuildConfig.packageName
@@ -108,6 +106,10 @@ dependencies {
implementation(libs.google.gson)
implementation(libs.google.guava)
// JNA
implementation(libs.jna.core)
implementation(libs.jna.platform)
// Git
implementation(libs.git.jgit)
gradle/libs.versions.toml
+5 -1
View File
@@ -18,9 +18,9 @@ retrofit = "2.9.0"
markwon = "4.6.2"
maven-publish-plugin = "0.27.0"
logback = "1.5.3"
realm = "10.18.0"
benchmark = "1.2.4"
benchmark-junit4 = "1.2.4"
jna = "5.14.0"
[libraries]
@@ -161,6 +161,10 @@ tooling-builderModel = { module = "com.android.tools.build:builder-model", versi
tooling-gradleApi = { module = "com.itsaky.androidide.gradle:gradle-tooling-api", version.ref = "gradle-tooling" }
tooling-slf4j = { module = "org.slf4j:slf4j-api", version = "2.0.12" }
#JNA
jna-core = { module = "net.java.dev.jna:jna", version.ref = "jna" }
jna-platform = { module = "net.java.dev.jna:jna-platform", version.ref = "jna" }
# Classpaths
android-gradle-plugin = { module = "com.android.tools.build:gradle", version.ref = "agp" }
kotlin-gradle-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
java/lsp/src/androidTest/java/com/itsaky/androidide/lsp/java/indexing/JavaClassCRUDTest.kt
-119
View File
@@ -1,119 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.lsp.java.indexing
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.google.common.truth.Truth.assertThat
import com.itsaky.androidide.lsp.java.indexing.classfile.JavaClass
import com.itsaky.androidide.lsp.java.utils.JavaType
import com.itsaky.androidide.testing.android.rules.RealmDBTestRule
import io.realm.RealmList
import io.realm.RealmObject
import openjdk.tools.classfile.AccessFlags
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
/**
* @author Akash Yadav
*/
@RunWith(AndroidJUnit4::class)
class JavaClassCRUDTest {
@Rule
@JvmField
val dbTestRule = RealmDBTestRule(JavaIndexingRealmModule())
@Test
fun testSimpleJavaClassCRUD() {
dbTestRule.withDb("java-class-CRUD") {
val klass = ModelBuilderTestUtil.createTestClass()
// CREATE
executeTransaction {
it.insert(klass)
}
var dbKlass: JavaClass? = null
executeTransaction {
// READ
dbKlass = where(JavaClass::class.java)
.equalTo("fqn", "com/itsaky/androidide/indexing/TestClass")
.findFirst()
assertThat(dbKlass).isNotNull()
assertThat(dbKlass?.fqn).isEqualTo("com/itsaky/androidide/indexing/TestClass")
assertThat(dbKlass?.name).isEqualTo("TestClass")
assertThat(dbKlass?.packageName).isEqualTo("com/itsaky/androidide/indexing")
assertThat(dbKlass?.accessFlags).isEqualTo(AccessFlags.ACC_PUBLIC or AccessFlags.ACC_FINAL)
assertThat(dbKlass?.superClassFqn).isEqualTo("java/lang/Object")
assertThat(dbKlass?.superInterfacesFqn).isEqualTo(RealmList<String>())
assertThat(dbKlass?.fields).isNotEmpty()
assertThat(dbKlass?.fields).hasSize(1)
dbKlass?.fields?.get(0).apply {
assertThat(this).isNotNull()
assertThat(this?.name).isEqualTo("someString")
assertThat(this?.type).isEqualTo(JavaType.STRING)
assertThat(this?.accessFlags).isEqualTo(AccessFlags.ACC_PRIVATE)
}
assertThat(dbKlass?.methods).isNotEmpty()
assertThat(dbKlass?.methods).hasSize(1)
dbKlass?.methods?.get(0).apply {
assertThat(this).isNotNull()
assertThat(this?.name).isEqualTo("getSomeString")
assertThat(this?.paramsTypes).isEqualTo(RealmList<String>())
assertThat(this?.returnType).isEqualTo(JavaType.STRING)
assertThat(this?.accessFlags).isEqualTo(AccessFlags.ACC_PUBLIC)
}
}
assertThat(dbKlass).isNotNull()
// UPDATE
executeTransaction {
dbKlass?.accessFlags = AccessFlags.ACC_PUBLIC or AccessFlags.ACC_ABSTRACT
}
// verify UPDATE
executeTransaction {
assertThat(
where(JavaClass::class.java)
.equalTo("fqn", "com/itsaky/androidide/indexing/TestClass")
.findFirst()?.accessFlags
).isEqualTo(AccessFlags.ACC_PUBLIC or AccessFlags.ACC_ABSTRACT)
}
// DELETE
executeTransaction {
RealmObject.deleteFromRealm(dbKlass!!)
}
// verify DELTE
executeTransaction {
assertThat(
where(JavaClass::class.java)
.equalTo("fqn", "com/itsaky/androidide/indexing/TestClass")
.findFirst()
).isNull()
}
}
}
}
java/lsp/src/androidTest/java/com/itsaky/androidide/lsp/java/indexing/JavaIndexerTest.kt
-99
View File
@@ -1,99 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.lsp.java.indexing
import android.content.Context
import androidx.test.core.app.ApplicationProvider
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.blankj.utilcode.util.ConvertUtils
import com.google.common.truth.Truth
import com.itsaky.androidide.testing.android.rules.RealmDBTestRule
import io.realm.RealmModel
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import java.io.File
/**
* @author Akash Yadav
*/
@RunWith(AndroidJUnit4::class)
class JavaIndexerTest {
@Rule
@JvmField
val dbTestRule = RealmDBTestRule(JavaIndexingRealmModule())
val androidJar: File by lazy {
val context = ApplicationProvider.getApplicationContext<Context>()
val file = File.createTempFile("ajar", null, context.cacheDir)
context.assets.open("android.jar").use { asset ->
Truth.assertThat(asset).isNotNull()
file.outputStream().buffered().use { out ->
asset.copyTo(out)
out.flush()
}
}
file
}
@Test
fun testSimpleAndroidJarIndexingDurationCheck() {
val worker = JavaJarModelBuilder(androidJar)
val batches = mutableMapOf<Class<*>, MutableList<RealmModel>>()
var dbWriteDuration = 0L
dbTestRule.withDb("android-jar-classes") {
val totalStart = System.currentTimeMillis()
worker.consumeTypes { type ->
val batched = batches.computeIfAbsent(type.javaClass) { mutableListOf() }
batched.add(type)
if (batched.size >= 100) {
val start = System.currentTimeMillis()
executeTransaction {
insertOrUpdate(batched)
}
dbWriteDuration += System.currentTimeMillis() - start
batched.clear()
}
}
for ((_, batched) in batches) {
if (batched.isNotEmpty()) {
val start = System.currentTimeMillis()
executeTransaction {
insertOrUpdate(batched)
}
dbWriteDuration += System.currentTimeMillis() - start
}
}
println(
"Took ${dbWriteDuration}ms to write android.jar (${
ConvertUtils.byte2FitMemorySize(
androidJar.length()
)
}) classes to Realm"
)
println("Total time: ${System.currentTimeMillis() - totalStart}ms")
}
}
}
java/lsp/src/androidTest/java/com/itsaky/androidide/lsp/java/indexing/ModelBuilderTestUtil.kt
-52
View File
@@ -1,52 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.lsp.java.indexing
import com.itsaky.androidide.lsp.java.indexing.classfile.JavaClass
import com.itsaky.androidide.lsp.java.indexing.classfile.JavaField
import com.itsaky.androidide.lsp.java.indexing.classfile.JavaMethod
import com.itsaky.androidide.lsp.java.utils.JavaType
import io.realm.RealmList
import openjdk.tools.classfile.AccessFlags
/**
* @author Akash Yadav
*/
object ModelBuilderTestUtil {
fun createTestClass() = JavaClass.newInstance(
fqn = "com/itsaky/androidide/indexing/TestClass",
name = "TestClass",
packageName = "com/itsaky/androidide/indexing",
accessFlags = AccessFlags.ACC_PUBLIC or AccessFlags.ACC_FINAL,
superClassFqn = "java/lang/Object",
superInterfacesFqn = RealmList(),
fields = RealmList<JavaField>().apply {
add(JavaField.newField("someString", JavaType.STRING, AccessFlags.ACC_PRIVATE))
},
methods = RealmList<JavaMethod>().apply {
add(
JavaMethod.newInstance(
name = "getSomeString",
paramsTypes = RealmList(),
returnType = JavaType.STRING,
accessFlags = AccessFlags.ACC_PUBLIC
)
)
})
}
java/lsp/src/androidTest/java/com/itsaky/androidide/lsp/java/indexing/SharedModelCRUDTest.kt
-275
View File
@@ -1,275 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.lsp.java.indexing
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.google.common.truth.Truth.assertThat
import com.itsaky.androidide.indexing.core.platform.ApiInfo
import com.itsaky.androidide.lsp.java.indexing.classfile.ArrayAnnotationElementValue
import com.itsaky.androidide.lsp.java.indexing.classfile.ClassAnnotationElementValue
import com.itsaky.androidide.lsp.java.indexing.classfile.EnumAnnotationElementValue
import com.itsaky.androidide.lsp.java.indexing.classfile.IAnnotationElementValue
import com.itsaky.androidide.lsp.java.indexing.classfile.JavaConstant
import com.itsaky.androidide.lsp.java.utils.JavaType
import com.itsaky.androidide.lsp.java.indexing.classfile.PrimitiveAnnotationElementValue
import com.itsaky.androidide.testing.android.rules.RealmDBTestRule
import io.realm.RealmAny
import io.realm.RealmList
import io.realm.exceptions.RealmPrimaryKeyConstraintException
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
/**
* @author Akash Yadav
*/
@RunWith(AndroidJUnit4::class)
class SharedModelCRUDTest {
@Rule
@JvmField
val dbTestRule = RealmDBTestRule(JavaIndexingRealmModule())
@Test
fun testApiInfoCreate() {
dbTestRule.withDb("api-info") {
val apiInfo = com.itsaky.androidide.indexing.core.platform.ApiInfo.newInstance(since = 1, deprecatedIn = 21, removedIn = 23)
assertInsertUnique(apiInfo)
val apiInfo2 = apiInfo.clone().update(removedIn = 26)
assertInsertion(allowUpdate = true, apiInfo, apiInfo2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(apiInfo, apiInfo2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testApiInfoDuplicationError() {
dbTestRule.withDb("api-info-duplication-error") {
val apiInfo = com.itsaky.androidide.indexing.core.platform.ApiInfo.newInstance(since = 1, deprecatedIn = 21, removedIn = 23)
assertInsertSingle(apiInfo)
assertInsertSingle(apiInfo)
}
}
@Test
fun testConstantCreate() {
dbTestRule.withDb("java-constant-create") {
val constant =
JavaConstant.newInstance(kind = JavaType.KIND_INT, value = RealmAny.valueOf(1.toInt()))
val constant2 =
JavaConstant.newInstance(kind = JavaType.KIND_INT, value = RealmAny.valueOf(2.toInt()))
assertInsertUnique(constant)
assertInsertion(allowUpdate = true, constant, constant2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(constant, constant2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testConstantDuplicationError() {
dbTestRule.withDb("java-constant-duplication-error") {
val constant =
JavaConstant.newInstance(kind = JavaType.KIND_INT, value = RealmAny.valueOf(1.toInt()))
assertInsertSingle(constant)
assertInsertSingle(constant)
}
}
@Test
fun testJavaTypeCreate() {
dbTestRule.withDb("java-type-create") {
assertInsertUnique(JavaType.INT)
assertInsertion(JavaType.STRING, JavaType.OBJECT) { models ->
assertThat(models).hasSize(3)
assertThat(models).containsExactly(JavaType.INT, JavaType.STRING, JavaType.OBJECT)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testJavaTypeDuplicationError() {
dbTestRule.withDb("java-type-duplication-error") {
assertInsertSingle(JavaType.OBJECT)
assertInsertSingle(JavaType.OBJECT)
}
}
@Test
fun testPrimitiveAnnotationValueCreate() {
dbTestRule.withDb("annotation-value-primitive") {
val prim1 = PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_INT,
JavaConstant.newInstance(JavaType.KIND_INT, RealmAny.valueOf(1.toInt()))
)
val prim2 = PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_STRING,
JavaConstant.newInstance(JavaType.KIND_REF, RealmAny.valueOf("Something something"))
)
assertInsertUnique(prim1)
assertInsertion(allowUpdate = true, prim1, prim2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(prim1, prim2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testPrimitiveAnnotationValueDuplicationError() {
dbTestRule.withDb("annotation-value-primitive-duplication") {
val prim1 = PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_INT,
JavaConstant.newInstance(JavaType.KIND_INT, RealmAny.valueOf(1.toInt()))
)
assertInsertSingle(prim1)
assertInsertSingle(prim1)
}
}
@Test
fun testArrayAnnotationValueCreate() {
dbTestRule.withDb("annotation-value-array") {
val obj1 = ArrayAnnotationElementValue.newInstance(
RealmList()
)
val obj2 = ArrayAnnotationElementValue.newInstance(
RealmList<RealmAny>().apply {
add(
RealmAny.valueOf(
PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_STRING,
JavaConstant.newInstance(JavaType.KIND_REF, RealmAny.valueOf("Something something"))
)
)
)
RealmAny.valueOf(
PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_INT,
JavaConstant.newInstance(JavaType.KIND_INT, RealmAny.valueOf(123.toInt()))
)
)
}
)
assertInsertUnique(obj1)
assertInsertion(allowUpdate = true, obj1, obj2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(obj1, obj2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testArrayAnnotationValueDuplicationError() {
dbTestRule.withDb("annotation-value-array-duplication") {
val obj1 = ArrayAnnotationElementValue.newInstance(
RealmList<RealmAny>().apply {
add(
RealmAny.valueOf(
PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_STRING,
JavaConstant.newInstance(JavaType.KIND_REF, RealmAny.valueOf("Something something"))
)
)
)
RealmAny.valueOf(
PrimitiveAnnotationElementValue.newInstance(
IAnnotationElementValue.KIND_INT,
JavaConstant.newInstance(JavaType.KIND_INT, RealmAny.valueOf(123.toInt()))
)
)
}
)
assertInsertSingle(obj1)
assertInsertSingle(obj1)
}
}
@Test
fun testClassAnnotationValueCreate() {
dbTestRule.withDb("annotation-value-class") {
val obj1 = ClassAnnotationElementValue.newInstance(
JavaType.newInstance("com/example/SomeType", JavaType.KIND_REF)
)
val obj2 = ClassAnnotationElementValue.newInstance(
JavaType.newInstance("com/example/SomeType", JavaType.KIND_REF, arrayDims = 5)
)
assertInsertUnique(obj1)
assertInsertion(allowUpdate = true, obj1, obj2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(obj1, obj2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testClassAnnotationValueDuplicationError() {
dbTestRule.withDb("annotation-value-class-duplication") {
val obj1 = ClassAnnotationElementValue.newInstance(
JavaType.newInstance("com/example/SomeType", JavaType.KIND_REF)
)
assertInsertSingle(obj1)
assertInsertSingle(obj1)
}
}
@Test
fun testEnumAnnotationValueCreate() {
dbTestRule.withDb("annotation-value-enum") {
val obj1 = EnumAnnotationElementValue.newInstance(
"SomeEnum_ENTRY",
JavaType.newInstance("com/example/SomeEnum", JavaType.KIND_REF),
)
val obj2 = EnumAnnotationElementValue.newInstance(
"SomeEnum_ENTRY_2",
JavaType.newInstance("com/example/SomeEnum", JavaType.KIND_REF),
)
assertInsertUnique(obj1)
assertInsertion(allowUpdate = true, obj1, obj2) { models ->
assertThat(models).hasSize(2)
assertThat(models).containsExactly(obj1, obj2)
}
}
}
@Test(expected = RealmPrimaryKeyConstraintException::class)
fun testEnumAnnotationValueDuplicationError() {
dbTestRule.withDb("annotation-value-enum-duplication") {
val obj1 = EnumAnnotationElementValue.newInstance(
"SomeEnum_ENTRY",
JavaType.newInstance("com/example/SomeEnum", JavaType.KIND_REF),
)
assertInsertSingle(obj1)
assertInsertSingle(obj1)
}
}
}
java/lsp/src/androidTest/java/com/itsaky/androidide/lsp/java/indexing/crudExts.kt
-72
View File
@@ -1,72 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.lsp.java.indexing
import com.google.common.truth.Truth.assertThat
import io.realm.Realm
import io.realm.RealmModel
import io.realm.RealmResults
inline fun <reified T : RealmModel> Realm.assertInsertion(
vararg models: T,
crossinline verify: Realm.(models: RealmResults<T>) -> Unit
) {
assertInsertion(false, *models, verify = verify)
}
inline fun <reified T : RealmModel> Realm.assertInsertion(
allowUpdate: Boolean = false,
vararg models: T,
crossinline verify: Realm.(models: RealmResults<T>) -> Unit
) {
val modelList = models.toList()
assertThat(modelList).isNotEmpty()
executeTransaction {
if (allowUpdate) {
it.insertOrUpdate(modelList)
} else {
it.insert(modelList)
}
}
executeTransaction {
verify(it.where(modelList[0].javaClass).findAll())
}
}
fun Realm.assertInsertSingle(model: RealmModel) {
assertInsertion(model) { models ->
assertThat(models).isNotNull()
assertThat(models).isNotEmpty()
assertThat(models).hasSize(1)
assertThat(models[0]).isEqualTo(model)
}
}
fun Realm.assertInsertUnique(model: RealmModel) {
assertInsertSingle(model)
assertInsertion(true, model) { models ->
assertThat(models).isNotNull()
assertThat(models).isNotEmpty()
assertThat(models).hasSize(1)
assertThat(models[0]).isEqualTo(model)
}
}
settings.gradle.kts
+3
View File
@@ -17,6 +17,9 @@
@file:Suppress("UnstableApiUsage")
include(":utilities:level-hash")
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
pluginManagement {
termux/application/build.gradle.kts
-4
View File
@@ -17,8 +17,6 @@
@file:Suppress("UnstableApiUsage")
import com.itsaky.androidide.build.config.BuildConfig
import com.itsaky.androidide.plugins.TerminalBootstrapPackagesPlugin
@@ -31,8 +29,6 @@ apply {
plugin(TerminalBootstrapPackagesPlugin::class.java)
}
val packageVariant = System.getenv("TERMUX_PACKAGE_VARIANT") ?: "apt-android-7" // Default: "apt-android-7"
android {
termux/view/build.gradle.kts
-2
View File
@@ -23,8 +23,6 @@ plugins {
id("kotlin-android")
}
android {
namespace = "com.termux.view"
ndkVersion = BuildConfig.ndkVersion
testing/androidTest/src/main/java/com/itsaky/androidide/testing/android/rules/RealmDBTestRule.kt
-61
View File
@@ -1,61 +0,0 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.testing.android.rules
import com.itsaky.androidide.db.IRealmProvider
import com.itsaky.androidide.testing.android.util.NoOpStatement
import io.realm.Realm
import io.realm.log.LogLevel
import io.realm.log.RealmLog
import org.junit.runner.Description
import org.junit.runners.model.Statement
/**
* Rule for Realm DB tests.
*
* @author Akash Yadav
*/
class RealmDBTestRule(
val baseModule: Any? = null,
vararg val additionalModules: Array<Any>,
) : AbstractAndroidTestRule() {
inline fun withDb(dbName: String, deleteDbAfterTest: Boolean = true, action: Realm.() -> Unit) {
val name = dbName.replace(IRealmProvider.PATH_SEPARATOR, '-')
val realm = IRealmProvider.instance().get("/indexing/java/$name") {
baseModule?.let { baseModule ->
modules(baseModule, *additionalModules)
}
}
try {
realm.action()
} finally {
if (deleteDbAfterTest) {
realm.configuration.realmDirectory.deleteRecursively()
}
}
}
override fun apply(base: Statement?, description: Description?): Statement {
Realm.init(context.applicationContext)
RealmLog.setLevel(LogLevel.ALL)
base?.evaluate()
return NoOpStatement()
}
}
testing/commonTest/build.gradle.kts
+3 -1
View File
@@ -30,7 +30,9 @@ dependencies {
api(libs.google.auto.service.annotations)
api(libs.tests.junit)
api(projects.logging.logger)
api(libs.jna.core)
api(libs.jna.platform)
api(projects.logging.logger)
api(projects.tooling.impl)
}
testing/commonTest/src/main/java/com/itsaky/androidide/testing/common/LinuxOnlyTestRule.kt
+41
View File
@@ -0,0 +1,41 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.testing.common
import com.sun.jna.Platform
import org.junit.rules.TestRule
import org.junit.runner.Description
import org.junit.runners.model.Statement
/**
* A [TestRule] which runs the tests only in a Linux environment.
*
* @author Akash Yadav
*/
class LinuxOnlyTestRule : TestRule {
override fun apply(base: Statement, description: Description?): Statement {
if (Platform.isLinux()) {
return base
}
return object : Statement() {
override fun evaluate() {}
}
}
}
utilities/level-hash/.gitignore
+1
View File
@@ -0,0 +1 @@
/build
utilities/level-hash/README.md
+12
View File
@@ -0,0 +1,12 @@
# Level Hash
Based on the work of Pengfei Zuo, Yu Hua, and Jie
Wu. [Read more](https://csyhua.github.io/csyhua/hua-tos2019.pdf).
## References
- [Write-Optimized and High-Performance Hashing Index Scheme for Persistent
Memory](https://csyhua.github.io/csyhua/hua-tos2019.pdf) - Pengfei Zuo, Yu Hua, and Jie
Wu.
- [C implementation by @Pfzuo](https://github.com/Pfzuo/Level-Hashing) - Source code is licensed
under GNU General Public License v3.0.
utilities/level-hash/build.gradle.kts
+56
View File
@@ -0,0 +1,56 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
import com.itsaky.androidide.build.config.BuildConfig
plugins {
id("com.android.library")
id("kotlin-android")
}
android {
namespace = "${BuildConfig.packageName}.levelhash"
ndkVersion = BuildConfig.ndkVersion
}
tasks.withType<Test> {
jvmArgs("--add-opens", "java.base/java.io=ALL-UNNAMED")
}
dependencies {
implementation(libs.androidx.collection)
implementation(libs.google.guava)
// Let the consuming project decide which platform to use
compileOnly(libs.jna.core)
compileOnly(libs.jna.platform)
implementation(projects.logging.logger)
implementation(projects.utilities.shared)
// compileOnly, in order to use JVM variant in tests and Android variant at
// runtime
compileOnly(libs.jna.core)
compileOnly(libs.jna.platform)
// For JVM
testImplementation(libs.jna.core)
testImplementation(libs.jna.platform)
testImplementation(projects.testing.unitTest)
androidTestImplementation(projects.testing.androidTest)
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/DataIO.kt
+133
View File
@@ -0,0 +1,133 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
import com.itsaky.androidide.levelhash.util.DataExternalizers
import java.io.DataInput
import java.io.DataOutput
/**
* A [DataExternalizer] serializes and deserializes data in a level hash.
*
* @author Akash Yadav
*/
interface DataExternalizer<T> {
/**
* Serialize the given [data] into the [out][DataOutput].
*
* @param out The output stream to serialize to.
* @param data The data to serialize.
*/
fun write(out: DataOutput, data: T)
/**
* Deserialize the data from the [input][DataInput].
*
* @param input The input stream to deserialize from.
* @return Data deserialized from the input stream.
*/
fun read(input: DataInput): T
}
/**
* Allows random access to the underlying data.
*/
interface RandomAccessIO {
/**
* Set the position to read from.
*/
fun position(position: Long)
/**
* Seek the position by given count.
*/
fun seekRelative(count: Int) {
seekRelative(count.toLong())
}
/**
* Seek the position by given count.
*/
fun seekRelative(count: Long) {
position(position() + count)
}
/**
* Get the current positon where data is read from.
*/
fun position(): Long
}
/**
* Seek the position by the size of a boolean.
*/
fun RandomAccessIO.seekBoolean() {
seekRelative(DataExternalizers.SIZE_BOOLEAN)
}
/**
* Seek the position by the size of a byte.
*/
fun RandomAccessIO.seekByte() {
seekRelative(DataExternalizers.SIZE_BYTE)
}
/**
* Seek the position by the size of a short.
*/
fun RandomAccessIO.seekShort() {
seekRelative(DataExternalizers.SIZE_SHORT)
}
/**
* Seek the position by the size of a char.
*/
fun RandomAccessIO.seekChar() {
seekRelative(DataExternalizers.SIZE_CHAR)
}
/**
* Seek the position by the size of an int.
*/
fun RandomAccessIO.seekInt() {
seekRelative(DataExternalizers.SIZE_INT)
}
/**
* Seek the position by the size of a long.
*/
fun RandomAccessIO.seekLong() {
seekRelative(DataExternalizers.SIZE_LONG)
}
/**
* Seek the position by the size of a float.
*/
fun RandomAccessIO.seekFloat() {
seekRelative(DataExternalizers.SIZE_FLOAT)
}
/**
* Seek the position by the size of a double.
*/
fun RandomAccessIO.seekDouble() {
seekRelative(DataExternalizers.SIZE_DOUBLE)
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelBucket.kt
+33
View File
@@ -0,0 +1,33 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
/**
* A bucket in [LevelHash].
*
* @author Akash Yadav
*/
interface LevelBucket<K, V> {
/**
* Get the slot at the given index.
*
* @param index The index of the slot to retrieve.
*/
fun getSlot(index: Int): LevelSlot<K, V>
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelHash.kt
+178
View File
@@ -0,0 +1,178 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
/**
* Level hash is a write-optimized and high-performance hashing index scheme with cost-efficient
* resizing and low-overhead consistency guarantee for persistent memory.
*
* @param K The key type.
* @param V The value type.
*/
interface LevelHash<K : Any, V : Any?> {
/**
* The size of the level size. This is `log2(addrCapacity)`.
*/
val levelSize: Int
/**
* The number of buckets in the top level. The number of buckets in the bottom
* level is exactly half of this.
*/
val topLevelBucketCount: Int
/**
* The size of each bucket, denoting number of slots in a bucket.
*/
val bucketSize: Int
/**
* The number of buckets in the level hash (including both, top and bottom levels).
*/
val totalBucketCount: Int
get() = topLevelBucketCount + (topLevelBucketCount shr 1)
/**
* The number of slots in the level hash (including both, top and bottom levels).
*/
val totalSlotCount: Int
get() = totalBucketCount * bucketSize
/**
* Get the value associated with the given key.
*
* @param key The key for the value.
* @return The value associated with the given key, or `null` if not found.
*/
operator fun get(key: K): V?
/**
* Update the value associated with the given key.
*
* @param key The key.
* @param value The value.
* @return The old value associated with the given key, or `null` if not found
* (or if the value was already `null`).
*/
operator fun set(key: K, value: V): V?
/**
* Insert the given key-value pair into the level has.
*
* @param key The key.
* @param value The value.
* @return Whether the key-value pair was inserted successfully.
*/
fun insert(key: K, value: V): Boolean
/**
* Remove the entry with the given key, if any.
*
* @param key The key.
* @return The (nullable) value associated with the given key, or `null` if
* not found.
*/
fun remove(key: K): V?
/**
* Clear the level hash, removing all entries.
*/
fun clear()
/**
* Find the slot with the given key.
*
* @param key The key to find.
* @return The slot if found, or `null`.
*/
fun findSlot(key: K): LevelSlot<K, V>?
/**
* Expand the level hash by the given ADDITIONAL level size.
*
* @param addtionalLevelSize The additional level size of the level hash. If this is
* less than the current level size, an exception is thrown. If this is equal
* to the current level size, this function has no effect.
*
* @return Whether the level hash was resized.
* @throws IllegalArgumentException If the level size if less than the current
* level size.
*/
fun expand(addtionalLevelSize: Int): Boolean
/**
* The two levels in a [LevelHash].
*/
enum class Level(internal val index: Int) {
TOP(0),
BOTTOM(1)
}
companion object {
/**
* The default level size.
*/
const val BUCKET_SIZE_DEFAULT = 4
/**
* Build an in-memory level hash.
*/
fun <K : Any, V : Any?> inMemoryHashBuilder(): InMemoryHashBuilder<K, V> =
InMemoryHashBuilder()
/**
* Build a persistent level hash.
*/
fun <K : Any, V : Any?> persistentHashBuilder(): PersistentHashBuilder<K, V> =
PersistentHashBuilder()
}
open class LevelHashException(msg: String) : RuntimeException(msg)
/**
* A [RuntimeException] thrown when *insert* operation fails in a [LevelHash].
*/
open class InsertionError(msg: String) : LevelHashException(msg)
/**
* An [InsertionError] which is thrown when trying to insert a key in a [LevelHash] and an entry
* with the same key already exists.
*/
class DuplicateKeyException(key: Any) :
InsertionError("Key '$key' already exists")
/**
* An [InsertionError] which is thrown when the level hash is full and unable
* to insert new values.
*/
class OverflowException : InsertionError("Level hash has reached is capacity")
/**
* Thrown when the level hash cannot be resized.
*/
class ResizeFailure(msg: String) : InsertionError(msg)
/**
* Thrown when *update* operation fails to find the slot in a [LevelHash].
*/
class EntryNotFoundException(key: Any) :
LevelHashException("Entry not found for key '$key'")
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/LevelSlot.kt
+52
View File
@@ -0,0 +1,52 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
/**
* A slot in a [LevelBucket].
*
* @author Akash Yadav
*/
interface LevelSlot<K, V> {
/**
* The key for this slot.
*/
val key: K
/**
* The value for this slot.
*/
val value: V?
/**
* Whether the slot is occupied.
*/
fun isOccupied(): Boolean
/**
* Reset the value of this entry.
*
* @param key The new key.
* @param value The new value.
*/
fun reset(
key: K? = null,
value: V? = null,
)
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/builders.kt
+180
View File
@@ -0,0 +1,180 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
import com.itsaky.androidide.levelhash.LevelHash.Companion.BUCKET_SIZE_DEFAULT
import com.itsaky.androidide.levelhash.internal.InMemoryLevelHash
import com.itsaky.androidide.levelhash.internal.PersistentLevelHash
import com.sun.jna.Platform
import java.io.File
/**
* Builder used to build instances of [LevelHash].
*/
abstract class AbstractLevelHashBuilder<K : Any, V : Any?> {
protected var levelSize = -1
protected var bucketSize = BUCKET_SIZE_DEFAULT
protected var hashFn: LevelHashFn<K>? = null
protected var uniqueKeys = false
protected var autoExpand = true
protected var seedGenerator: SeedGeneratorFn = ::levelHashDefaultSeeds
/**
* Set the level size.
*
* @param levelSize The level size.
*/
fun levelSize(levelSize: Int) = apply {
this.levelSize = levelSize
}
/**
* Set the bucket size.
*
* @param bucketSize The number of slots in a bucket.
*/
fun bucketSize(bucketSize: Int) = apply {
this.bucketSize = bucketSize
}
/**
* Set the hash function.
*
* @param hashFn The hash function.
*/
fun hashFn(hashFn: LevelHashFn<K>) = apply {
this.hashFn = hashFn
}
/**
* Set the unique keys flag.
*
* @param uniqueKeys Whether the keys in the level hash must be unique.
*/
fun uniqueKeys(uniqueKeys: Boolean) = apply {
this.uniqueKeys = uniqueKeys
}
/**
* Set the auto expand flag.
*
* @param autoExpand Whether the level hash should auto expand.
*/
fun autoExpand(autoExpand: Boolean) = apply {
this.autoExpand = autoExpand
}
/**
* Set the seed generator.
*
* @param seedGenerator The seed generator.
*/
fun seedGenerator(seedGenerator: SeedGeneratorFn) = apply {
this.seedGenerator = seedGenerator
}
/**
* Build the level hash instance.
*/
abstract fun build(): LevelHash<K, V>
protected fun baseChecks() {
require(this.levelSize > 0) { "Level size must be > 0" }
require(this.bucketSize in 1..32) { "Slot size must be in range [1, 33)" }
requireNotNull(this.hashFn) { "Hash function must be set" }
}
}
class InMemoryHashBuilder<K : Any, V : Any?> :
AbstractLevelHashBuilder<K, V>() {
override fun build(): LevelHash<K, V> {
baseChecks()
return InMemoryLevelHash(
levelSize = levelSize,
bucketSize = bucketSize,
levelHashFn = hashFn!!,
seeds = seedGenerator(),
uniqueKeys = uniqueKeys,
autoExpand = autoExpand
)
}
}
class PersistentHashBuilder<K : Any, V : Any?> :
AbstractLevelHashBuilder<K, V>() {
private var keyExternalizer: DataExternalizer<K>? = null
private var valueExternalizer: DataExternalizer<V>? = null
private var indexFile: File? = null
init {
// Persistent level hash is only supported on Linux
check(Platform.isLinux())
}
/**
* Set the index file where the index will be stored.
*
* @param indexFile The index directory.
*/
fun indexFile(indexFile: File) = apply {
this.indexFile = indexFile
}
/**
* Set the key exernalizer.
*
* @param externalizer The [DataExternalizer] for keys.
*/
fun keyExternalizer(externalizer: DataExternalizer<K>) = apply {
this.keyExternalizer = externalizer
}
/**
* Set the value externalizer.
*
* @param externalizer The [DataExternalizer] for values.
*/
fun valueExternalizer(externalizer: DataExternalizer<V>) = apply {
this.valueExternalizer = externalizer
}
override fun build(): LevelHash<K, V> {
baseChecks()
val keyExternalizer =
requireNotNull(this.keyExternalizer) { "Key externalizer must be set" }
val valueExternalizer = requireNotNull(
this.valueExternalizer) { "Value externalizer must be set" }
val seeds = seedGenerator()
return PersistentLevelHash(
levelSize = levelSize,
bucketSize = bucketSize,
uniqueKeys = uniqueKeys,
autoExpand = autoExpand,
levelHashFn = hashFn!!,
seeds = seeds,
keyExternalizer = keyExternalizer,
valueExternalizer = valueExternalizer,
indexFile = indexFile!!
)
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/AbstractLevelHash.kt
+445
View File
@@ -0,0 +1,445 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import androidx.annotation.VisibleForTesting
import com.itsaky.androidide.levelhash.HashT
import com.itsaky.androidide.levelhash.LevelHash
import com.itsaky.androidide.levelhash.LevelHash.DuplicateKeyException
import com.itsaky.androidide.levelhash.LevelHash.EntryNotFoundException
import com.itsaky.androidide.levelhash.LevelHash.Level
import com.itsaky.androidide.levelhash.LevelHash.OverflowException
import com.itsaky.androidide.levelhash.LevelHash.ResizeFailure
import com.itsaky.androidide.levelhash.LevelHashFn
import com.itsaky.androidide.levelhash.LevelSlot
import org.slf4j.LoggerFactory
import kotlin.math.pow
/**
* @author Akash Yadav
*/
internal abstract class AbstractLevelHash<K : Any, V : Any?> internal constructor(
levelSize: Int,
final override val bucketSize: Int,
protected val uniqueKeys: Boolean,
protected val autoExpand: Boolean,
protected val levelHashFn: LevelHashFn<K>,
protected val seeds: Pair<HashT, HashT>,
) : LevelHash<K, V> {
final override var levelSize: Int = levelSize
set(value) {
require(value <= 30) { "Level size must be <= 30" }
field = value
}
override var topLevelBucketCount = 2.0.pow(levelSize).toInt()
/**
* The number of occupied slots in each level.
*/
@VisibleForTesting
internal val levelItemCounts = IntArray(LEVEL_COUNT)
/**
* The number of times the level hash has been expanded.
*/
protected var expandCount: Int = 0
/**
* The state indicating whether the level hash is being resized or not.
* 0 - Not resizing
* 1 - Expanding
* 2 - Shrinking
*/
protected var resizeState = RESIZE_STATE_NOT_RESIZING
protected val logger = LoggerFactory.getLogger(javaClass)
companion object {
/**
* The number of levels in level hash.
*/
const val LEVEL_COUNT = 2
const val RESIZE_STATE_NOT_RESIZING = 0
const val RESIZE_STATE_EXPANDING = 1
const val RESIZE_STATE_SHRINKING = 2
}
/**
* Get the slot for the given slot position.
*
* @param levelIdx The index of the level.
* @param bucketIdx The index of the bucket in the level.
* @param slotIdx The index of the slot in the bucket.
* @return The slot for the given slot position.
*/
abstract fun getSlot(levelIdx: Int, bucketIdx: Int, slotIdx: Int
): LevelSlot<K, V>
/**
* Get the slot for the given slot position.
*/
fun getSlot(level: Level, bucketIdx: Int, slotIdx: Int
): LevelSlot<K, V> {
return getSlot(level.index, bucketIdx, slotIdx)
}
override fun get(key: K): V? {
return findSlot(key)?.value
}
override fun set(key: K, value: V): V? {
val slot = findSlot(key)
if (slot != null) {
val oldValue = slot.value
slot.reset(key, value)
return oldValue
}
throw EntryNotFoundException(key)
}
override fun insert(key: K, value: V): Boolean {
if (this.levelItemCounts.sum() == this.totalSlotCount) {
if (autoExpand) {
check(expand(1)) {
"Failed to expand the level hash"
}
} else {
throw OverflowException()
}
}
val insertFn =
fun(levelIdx: Int, bucketIdx: Int, slotIdx: Int): Boolean {
val slot = getSlot(levelIdx, bucketIdx, slotIdx)
if (!slot.isOccupied()) {
slot.reset(key, value)
levelItemCounts[levelIdx]++
return true
} else if (uniqueKeys && getSlot(levelIdx, bucketIdx,
slotIdx).key == key
) {
throw DuplicateKeyException(key)
}
return false
}
val fhash = fhash(key)
val shash = shash(key)
// Check if there are any empty slots available in any of the levels
// If there are, insert the key-value pair and return true
for (i in 0..<LEVEL_COUNT) {
val fidx = buckIdxLvl(fhash, i)
val sidx = buckIdxLvl(shash, i)
for (j in 0..<bucketSize) {
if (insertFn(i, fidx, j) || insertFn(i, sidx, j)) {
return true
}
}
}
for (i in 0..<LEVEL_COUNT) {
val fidx = buckIdxLvl(fhash, i)
val sidx = buckIdxLvl(shash, i)
if (tryMovement(i, fidx, key, value)) {
return true
}
if (tryMovement(i, sidx, key, value)) {
return true
}
}
if (expandCount > 0) {
val fidx = buckIdxLvl(fhash, Level.TOP.index)
val sidx = buckIdxLvl(shash, Level.TOP.index)
var emptyLocation = b2tMovement(fidx)
if (emptyLocation != -1) {
val emptySlot = getSlot(Level.TOP, fidx, emptyLocation)
emptySlot.reset(key, value)
levelItemCounts[Level.TOP.index]++
return true
}
emptyLocation = b2tMovement(sidx)
if (emptyLocation != -1) {
val emptySlot = getSlot(Level.TOP, sidx, emptyLocation)
emptySlot.reset(key, value)
levelItemCounts[Level.TOP.index]++
return true
}
}
return false
}
override fun remove(key: K): V? {
return findSlot(key).let { slot ->
if (slot != null) {
val oldValue = slot.value
slot.reset(null, null)
return@let oldValue
}
return@let null
}
}
override fun findSlot(key: K): LevelSlot<K, V>? {
val fhash = fhash(key)
val shash = shash(key)
var init = 0
var lim = LEVEL_COUNT
var step = 1
if (levelItemCounts[0] < levelItemCounts[1]) {
// if there are more occupied slots in the bottom level
// than in the top level, then scan the bottom level first
init = LEVEL_COUNT - 1
lim = -1
step = -1
}
var i = init
while (i != lim) {
val fidx = buckIdxLvl(fhash, i)
val sidx = buckIdxLvl(shash, i)
for (j in 0..<bucketSize) {
var slot = getSlot(i, fidx, j)
if (slot.isOccupied() && slot.key == key) {
return slot
}
slot = getSlot(i, sidx, j)
if (slot.isOccupied() && slot.key == key) {
return slot
}
}
i += step
}
return null
}
override fun expand(addtionalLevelSize: Int): Boolean {
check(this.resizeState == RESIZE_STATE_NOT_RESIZING) {
"Level hash is already being resized. Maybe on a different thread?"
}
require(addtionalLevelSize > 0) { "levelSize must be > 0" }
try {
this.resizeState = RESIZE_STATE_EXPANDING
return doExpand(this.levelSize + addtionalLevelSize)
} finally {
this.resizeState = RESIZE_STATE_NOT_RESIZING
}
}
private fun doExpand(levelSize: Int): Boolean {
val newTopLevelCapacity = 2.0.pow(levelSize).toInt()
var newLevelItemCount = 0
prepareExpansion(newTopLevelCapacity)
for (oldBuckIdx in 0..<(this.topLevelBucketCount shr 1)) {
for (oldSlotIdx in 0..<this.bucketSize) {
val oldSlot = getSlot(Level.BOTTOM, oldBuckIdx, oldSlotIdx)
if (oldSlot.isOccupied()) {
val fhash = fhash(oldSlot.key)
val shash = shash(oldSlot.key)
val fidx = buckIdxCap(fhash, newTopLevelCapacity)
val sidx = buckIdxCap(shash, newTopLevelCapacity)
var insertSuccess = false
for (newSlotIdx in 0..<this.bucketSize) {
if (tryMoveToInterim(oldSlot, fidx, newSlotIdx) || tryMoveToInterim(
oldSlot, sidx, newSlotIdx)
) {
insertSuccess = true
newLevelItemCount++
break
}
}
if (!insertSuccess) {
throw ResizeFailure("Failed to move slot to the interim level")
}
oldSlot.reset(null, null)
}
}
}
onExpand(levelSize, newLevelItemCount)
this.levelItemCounts[1] = this.levelItemCounts[0]
this.levelItemCounts[0] = newLevelItemCount
this.expandCount++
return true
}
/**
* Prepare the interim level for a expansion, ensuring that it can hold at least
* [bucketCount] buckets.
*
* @param bucketCount The bucket count of the interim level.
*/
protected abstract fun prepareExpansion(bucketCount: Int)
/**
* Called to notify that the expand operation was successful.
*
* @param newLevelSize The new level size of the level hash.
* @param interimItemCount The number of slot that are occupied in the
* interim level.
*/
protected abstract fun onExpand(newLevelSize: Int, interimItemCount: Int
)
/**
* Move the given slot to the interim level at the given slot in the given bucket.
* The caller does not check whether the slot is occupied or not. If the given
* slot position is already occupied, this method must return `false`.
*
* @param slot The existing slot to move.
* @param bucketIdx The index of the bucket in the interim level where the slot
* should be moved.
* @param slotIdx The index of the slot bucket of the interim level where the
* slot should be moved.
* @return Whether the slot was moved to the interim level.
*/
protected abstract fun tryMoveToInterim(slot: LevelSlot<K, V>,
bucketIdx: Int, slotIdx: Int
): Boolean
/**
* Try to move an item from the current bucket to its same-level alternative bucket.
*
* @param levelIndex The index of the level.
* @param bucketIndex The index of the bucket.
* @param key The key to insert.
* @param value The value to associate with key.
*/
private fun tryMovement(
levelIndex: Int,
bucketIndex: Int,
key: K,
value: V,
): Boolean {
for (i in 0..<bucketSize) {
val thisSlot = getSlot(levelIndex, bucketIndex, i)
val thisKey = thisSlot.key
val thisValue = thisSlot.value
val fhash = fhash(thisKey)
val shash = shash(thisKey)
val fidx = buckIdxLvl(fhash, levelIndex)
val sidx = buckIdxLvl(shash, levelIndex)
val jidx = if (fidx == bucketIndex) sidx else fidx
for (j in 0..<bucketSize) {
val thatSlot = getSlot(levelIndex, jidx, j)
if (!thatSlot.isOccupied()) {
thatSlot.reset(thisKey, thisValue)
thisSlot.reset(key, value)
levelItemCounts[levelIndex]++
return true
}
}
}
return false
}
/**
* Try to move an item from a bottom-level to its top-level alternative buckets.
*
* @param bucketIndex The index of the bucket in the bottom level.
* @return The index of the slot in the bucket at [bucketIndex] which was moved to the top-level.
*/
private fun b2tMovement(bucketIndex: Int): Int {
val b2tMover =
fun(bottomSlot: LevelSlot<K, V>, topSlot: LevelSlot<K, V>): Boolean {
if (!topSlot.isOccupied()) {
topSlot.reset(bottomSlot.key, bottomSlot.value)
bottomSlot.reset(null, null)
levelItemCounts[Level.TOP.index]++
levelItemCounts[Level.BOTTOM.index]--
return true
}
return false
}
for (i in 0..<bucketSize) {
val bottomSlot = getSlot(Level.BOTTOM, bucketIndex, i)
val fhash = fhash(bottomSlot.key)
val shash = shash(bottomSlot.key)
val fidx = buckIdxLvl(fhash, Level.TOP.index)
val sidx = buckIdxLvl(shash, Level.TOP.index)
for (j in 0..<bucketSize) {
var topSlot = getSlot(Level.TOP, fidx, j)
if (b2tMover(bottomSlot, topSlot)) {
return i
}
topSlot = getSlot(Level.TOP, sidx, j)
if (b2tMover(bottomSlot, topSlot)) {
return i
}
}
}
return -1
}
private fun fhash(key: K): HashT {
return levelHashFn(key, seeds.first)
}
private fun shash(key: K): HashT {
return levelHashFn(key, seeds.second)
}
private fun buckIdxLvl(keyHash: HashT, levelIndex: Int): Int {
require(levelIndex == 0 || levelIndex == 1)
var capacity = topLevelBucketCount
if (levelIndex == Level.BOTTOM.index) {
// divide by 2
capacity = capacity shr 1
}
return buckIdxCap(keyHash, capacity)
}
private fun buckIdxCap(keyHash: HashT, capacity: Int): Int {
// since capacity is a power of two and key hash is unsigned
// keyHash % capacity can be simplified with simple bit shift operation
return (keyHash and (capacity - 1).toULong()).toInt()
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelBucket.kt
+42
View File
@@ -0,0 +1,42 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import com.itsaky.androidide.levelhash.LevelBucket
import com.itsaky.androidide.levelhash.LevelSlot
/**
* @author Akash Yadav
*/
internal class InMemoryLevelBucket<K : Any, V : Any?> internal constructor(
private val size: Int,
) : LevelBucket<K, V> {
private val slots = Array(size) {
InMemoryLevelSlot.newInstance<K, V>()
}
override fun getSlot(index: Int): LevelSlot<K, V> {
checkIndex(index)
return slots[index]
}
private fun checkIndex(index: Int) {
require(index in 0..<size) { "Index must be in [0, $size)" }
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelHash.kt
+126
View File
@@ -0,0 +1,126 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import com.google.common.base.Preconditions
import com.itsaky.androidide.levelhash.HashT
import com.itsaky.androidide.levelhash.LevelHash.Level
import com.itsaky.androidide.levelhash.LevelHashFn
import com.itsaky.androidide.levelhash.LevelSlot
import kotlin.math.pow
/**
* An in-memory implementation of [AbstractLevelHash].
*
* @author Akash Yadav
*/
internal class InMemoryLevelHash<K : Any, V : Any?>(
levelSize: Int,
bucketSize: Int,
uniqueKeys: Boolean,
autoExpand: Boolean,
levelHashFn: LevelHashFn<K>,
seeds: Pair<HashT, HashT>,
) : AbstractLevelHash<K, V>(levelSize, bucketSize, uniqueKeys, autoExpand,
levelHashFn, seeds) {
private val levels = Array(LEVEL_COUNT) { levelIndex ->
allocateLevel(levelSize, levelIndex, bucketSize)
}
private fun allocateLevel(levelSize: Int, levelIndex: Int, bucketSize: Int
) = List(2.0.pow(levelSize - levelIndex).toInt()) {
InMemoryLevelBucket<K, V>(size = bucketSize)
}
private var interimLevel: List<InMemoryLevelBucket<K, V>>? = null
/**
* Get the bucket at the given position.
*
* @param level The level of the bucket to retrieve.
* @param bucketIndex The index of the bucket in [level].
*/
fun getBucket(level: Level, bucketIndex: Long): InMemoryLevelBucket<K, V> {
return getBucket(level.index, bucketIndex)
}
/**
* Get the bucket at the given position.
*
* @param levelIndex The index of the level of the bucket to retrieve.
* @param bucketIndex The index of the bucket.
*/
fun getBucket(levelIndex: Int, bucketIndex: Long): InMemoryLevelBucket<K, V> {
Preconditions.checkElementIndex(levelIndex, LEVEL_COUNT)
Preconditions.checkElementIndex(bucketIndex.toInt(),
levels[levelIndex].size)
return levels[levelIndex][bucketIndex.toInt()]
}
override fun clear() {
// reallocate levels, and let the GC handle the rest
for (i in 0 ..< LEVEL_COUNT) {
levels[i] = allocateLevel(levelSize, i, bucketSize)
}
this.interimLevel = null
}
override fun getSlot(levelIdx: Int, bucketIdx: Int, slotIdx: Int
): LevelSlot<K, V> {
return levels[levelIdx][bucketIdx].getSlot(slotIdx)
}
override fun prepareExpansion(bucketCount: Int) {
check(interimLevel == null) {
"Interim level must have been null. Is the level hash being resized on multiple threads?"
}
this.interimLevel = List(bucketCount) { InMemoryLevelBucket(bucketSize) }
}
override fun tryMoveToInterim(slot: LevelSlot<K, V>, bucketIdx: Int,
slotIdx: Int
): Boolean {
val interimLevel = checkNotNull(interimLevel)
slot as InMemoryLevelSlot<K, V>
val interimSlot = interimLevel[bucketIdx].getSlot(slotIdx)
if (interimSlot.isOccupied()) {
return false
}
interimSlot.reset(slot.key, slot.value)
slot.reset(null, null)
return true
}
override fun onExpand(newLevelSize: Int, interimItemCount: Int) {
this.levelSize = newLevelSize
this.topLevelBucketCount = 2.0.pow(newLevelSize).toInt()
// after expand, interim level becomes the new top level
// and the current top level becomes the new bottom level
// the current bottom level is released
this.levels[Level.BOTTOM.index] = levels[Level.TOP.index]
this.levels[Level.TOP.index] = checkNotNull(this.interimLevel)
this.interimLevel = null
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/InMemoryLevelSlot.kt
+64
View File
@@ -0,0 +1,64 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import com.itsaky.androidide.levelhash.LevelSlot
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.withLock
/**
* @author Akash Yadav
*/
internal class InMemoryLevelSlot<K, V> private constructor(
private var _key: K? = null,
private var _value: V? = null,
) : LevelSlot<K, V> {
private val lock = ReentrantReadWriteLock()
override val key: K
get() = lock.readLock().withLock { checkNotNull(_key) }
override val value: V?
get() = lock.readLock().withLock { _value }
companion object {
/**
* Create a new slot.
*/
fun <K, V> newInstance(): InMemoryLevelSlot<K, V> {
return InMemoryLevelSlot()
}
}
/**
* Whether the slot is occupied.
*/
override fun isOccupied(): Boolean = lock.readLock().withLock { _key != null }
/**
* Reset the value of this entry.
*/
override fun reset(key: K?, value: V?) {
lock.writeLock().withLock {
_key = key ?: _key
_value = value ?: _value
}
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHash.kt
+95
View File
@@ -0,0 +1,95 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import com.itsaky.androidide.levelhash.DataExternalizer
import com.itsaky.androidide.levelhash.HashT
import com.itsaky.androidide.levelhash.LevelHashFn
import com.itsaky.androidide.levelhash.LevelSlot
import java.io.File
import kotlin.math.pow
/**
* An file-based persistent implementation of [AbstractLevelHash].
*
* @author Akash Yadav
*/
internal class PersistentLevelHash<K : Any, V : Any?>(
levelSize: Int,
bucketSize: Int,
uniqueKeys: Boolean,
autoExpand: Boolean,
levelHashFn: LevelHashFn<K>,
seeds: Pair<HashT, HashT>,
keyExternalizer: DataExternalizer<K>,
valueExternalizer: DataExternalizer<V>,
indexFile: File,
) : AbstractLevelHash<K, V>(
levelSize = levelSize,
bucketSize = bucketSize,
uniqueKeys = uniqueKeys,
autoExpand = autoExpand,
levelHashFn = levelHashFn,
seeds = seeds
) {
private val io =
PersistentLevelHashIO(indexFile, keyExternalizer, valueExternalizer)
override fun getSlot(levelIdx: Int, bucketIdx: Int, slotIdx: Int
): LevelSlot<K, V> {
return object : LevelSlot<K, V> {
override val key: K
get() = checkNotNull(io.readKey(levelIdx, bucketIdx, slotIdx))
override val value: V
get() = checkNotNull(io.readValue(levelIdx, bucketIdx, slotIdx))
override fun isOccupied(): Boolean {
return io.isOccupied(levelIdx, bucketIdx, slotIdx)
}
override fun reset(key: K?, value: V?) {
io.writeEntry(levelIdx, bucketIdx, slotIdx, key, value)
}
}
}
override fun clear() {
this.io.clear()
}
private var interimBucketCount = -1
override fun prepareExpansion(bucketCount: Int) {
check(this.interimBucketCount == -1)
this.interimBucketCount = bucketCount
}
override fun tryMoveToInterim(
slot: LevelSlot<K, V>,
bucketIdx: Int,
slotIdx: Int
): Boolean {
TODO()
}
override fun onExpand(newLevelSize: Int, interimItemCount: Int) {
this.levelSize = newLevelSize
this.topLevelBucketCount = 2.0.pow(newLevelSize).toInt()
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/internal/PersistentLevelHashIO.kt
+757
View File
@@ -0,0 +1,757 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.internal
import com.itsaky.androidide.jna.Falloc
import com.itsaky.androidide.jna.LibC
import com.itsaky.androidide.levelhash.DataExternalizer
import com.itsaky.androidide.levelhash.HashT
import com.itsaky.androidide.levelhash.RandomAccessIO
import com.itsaky.androidide.levelhash.seekBoolean
import com.itsaky.androidide.levelhash.seekInt
import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_BOOLEAN
import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_INT
import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_LONG
import com.itsaky.androidide.levelhash.util.FileUtils
import com.itsaky.androidide.levelhash.util.MappedRandomAccessIO
import com.itsaky.androidide.utils.FDUtils
import com.sun.jna.Native
import org.slf4j.LoggerFactory
import java.io.DataInput
import java.io.DataOutput
import java.io.File
import java.io.RandomAccessFile
import java.nio.MappedByteBuffer
import java.nio.channels.FileChannel
private val logger = LoggerFactory.getLogger(PersistentLevelHashIO::class.java)
/**
* This class is responsible for performing the IO operations for the level hash
* backed by the disk.
*
* The peristent level hash is backed by three files :
*
* - `{name}._meta` - metadata for the level hash.
* - `{name}._i` - maps the hash of the slot position
* (`hash(level_num, bucket_idx, slot_idx`) to the address of entry in the
* `${name}` file. This is known as the keymap.
* - `{name}` - Contains all the variable-length entry values of the level hash.
*
* ## Keymap
*
* - The keymap file header occupies [PersistentLevelHashIO.KEYMAP_HEADER_SIZE_BYTES]
* bytes.
* - Each segment is occupies [PersistentLevelHashIO.KEYMAP_SEGMENT_SIZE_BYTES]
* bytes.
* - There can be at most [PersistentLevelHashIO.KEYMAP_MAX_SEGMENTS] segments in
* the keymap file.
* - Each segment can store at most [PersistentLevelHashIO.KEYMAP_ENTRIES_PER_SEGMENT]
* key-value pairs.
*
* Considering the default values, a keymap file can:
* - be `8 + 4 + 1024 * 1024 * 100 = ~100 MB` in size.
* - hold `(1024 * 1024 * 100) / 16 = 6553600 entries`
*
* Structure of the keymap file:
*
* ```
* keymap {
* u32 segment_count;
* u64 segment_size;
* segment segments[segment_count];
*
* segment {
* u64 value_address[];
* }
* }
* ```
*
* - `segment_count` - The number of segments in the file.
* - `segment_size` - The size of each segment.
* - `segments` - The segments of the file. There are `segment_count` entries,
* each of exactly `segment_size` bytes.
* - `value_address` - A 1-based 64-bit offset pointing the value entry in the values
* file. Note that the offset is 1-based because 0 is considered as an invalid
* value for the address. This helps in recognizing if an entry in the keymap
* points to a valid address or not. We can't really fill up the keymap entries
* with a negative offset as that would just take up unnecessary space.
*
* ## Values
*
* Structure of the values file :
*
* ```
* values {
* u64 magic_number;
* u64 first_entry;
* u64 next_entry;
* value values[];
*
* value {
* u8 is_occupied;
* u32 key_size;
* u8 key[key_size];
* u32 value_size;
* u8 value[value_size];
* }
* }
* ```
* @author Akash Yadav
*/
class PersistentLevelHashIO<K : Any, V : Any?>(
private val indexFile: File,
private val keyExternalizer: DataExternalizer<K>,
private val valueExternalizer: DataExternalizer<V>,
) : AutoCloseable {
// IMPLEMENTATION NOTE
// Whenever you access the valuesIo property, you must ensure that the
// position is set to the desired location in the values file.
// as the values file is randomly accessed, and by multiple functions, or
// even multiple threads, the current position cannot be determined
private val raIndexFile by lazy {
RandomAccessFile(indexFile, "rw")
}
private val keymapFile by lazy {
File(indexFile.parentFile, "${indexFile.name}._i")
}
private val raKeymapFile by lazy {
RandomAccessFile(keymapFile, "rw")
}
private val metaFile by lazy {
File(indexFile.parentFile, "${indexFile.name}._meta")
}
private val raMetaFile by lazy {
RandomAccessFile(metaFile, "rw")
}
private val segmentCount by lazy {
if (!keymapFile.exists() || !keymapFile.isFile || keymapFile.length() <= KEYMAP_HEADER_SIZE_BYTES) {
// there should be 1 segment initially
return@lazy 1
}
raKeymapFile.seek(KEYMAP_HEADER_SIZE_BYTES.toLong())
val count = raKeymapFile.readLong()
raKeymapFile.seek(0)
return@lazy count
}
// TODO(itsaky): use LFU cache to limit the number of segments mapped in memory at once.
/**
* The segments for the keymap file.
*/
private val keymapSegments = arrayOfNulls<KeymapSegment>(KEYMAP_MAX_SEGMENTS)
private var valuesFirstEntryAt = 0L
private var valuesLastEntryAt = 0L
private var valuesWriteAt = 0L
private var valuesMapSize = 0L
private var valuesIo = MappedRandomAccessIO()
init {
if (indexFile.exists()) {
if (indexFile.isDirectory) {
indexFile.deleteRecursively()
} else if (indexFile.isFile && indexFile.length() < VALUES_HEADER_SIZE_BYTES) {
indexFile.delete()
}
} else {
indexFile.createNewFile()
}
// TODO(itsaky): Load the below values from file when the file already exists
valuesLastEntryAt = -1
valuesWriteAt = 0
valuesFirstEntryAt = VALUES_HEADER_SIZE_BYTES
valuesMapSize = VALUES_MAX_MAP_SIZE_BYTES
// the header region is not memory mapped
log.debug("mmap values file: offset={}, size={}", VALUES_HEADER_SIZE_BYTES,
valuesMapSize)
val buffer = raIndexFile.channel.map(FileChannel.MapMode.READ_WRITE,
VALUES_HEADER_SIZE_BYTES, valuesMapSize)
valuesIo.reset(buffer, 0L, valuesMapSize)
}
private fun getValuesRealOffset(offset: Long) =
VALUES_HEADER_SIZE_BYTES + offset
private fun getSegmentOffset(index: Int): Long {
return KEYMAP_HEADER_SIZE_BYTES + index * KEYMAP_SEGMENT_SIZE_BYTES
}
/**
* Get or create a segment for the keymap file.
*/
private fun getOrCreateSegment(index: Int): KeymapSegment {
var segment = keymapSegments[index]
if (segment == null) {
segment = KeymapSegment.create(getSegmentOffset(index), raKeymapFile,
keymapFile.canonicalPath)
keymapSegments[index] = segment
}
return segment
}
private fun segmentAndPosForPos(
levelIdx: Int,
bucketIdx: Int,
slotIdx: Int,
): Pair<KeymapSegment, Long> {
val hash = hash(levelIdx, bucketIdx, slotIdx)
val segmentIdx = segmentIndex(hash)
val positionInSegment = positionInSegment(hash)
val segment = getOrCreateSegment(segmentIdx)
return segment to positionInSegment
}
/**
* Get the address of the value entry in the values for the given slot
* position.
*
* @return An address in the values file. If the address is 0, then value
* does not exist in the values file. The returned value is guaranteed to be
* in the range [[0, valuesMapSize]].
*/
private fun valueAddressForPos(
levelIdx: Int,
bucketIdx: Int,
slotIdx: Int,
): Long {
val (segment, positionInSegment) = segmentAndPosForPos(levelIdx, bucketIdx,
slotIdx)
segment.position(positionInSegment)
return segment.readLong().also { address ->
check(
address >= 0) { "Invalid value address in keymap at $positionInSegment: $address" }
}
}
fun isOccupied(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
): Boolean {
val valueAddr = valueAddressForPos(levelNum, bucketIdx, slotIdx)
if (valueAddr == 0L) {
// 0 == no value entry
return false
}
valuesIo.position(valueAddr - 1)
// token must be set and key size must be > 0
return valuesIo.readBoolean() && valuesIo.readInt() > 0
}
private fun moveToSlot(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
): Boolean {
val valueAddr = valueAddressForPos(levelNum, bucketIdx, slotIdx)
if (valueAddr == 0L) {
// 0 == no value entry
return false
}
valuesIo.position(valueAddr - 1)
return true
}
/**
* Visits the key region for the given slot position.
*
* @return - `null` - If the slot at the given position is not occupied.
* - `null to <keySize>` - If [readKey] is `false`.
* - `key to keySize` - If [readKey] is `true`.
*/
private fun visitKeyRegion(levelNum: Int, bucketIdx: Int, slotIdx: Int,
readKey: Boolean = false
): Pair<K?, Int>? {
if (!moveToSlot(levelNum, bucketIdx, slotIdx)) {
return null
}
val isOccupied = valuesIo.readBoolean()
if (!isOccupied) {
return null
}
val keySize = valuesIo.readInt()
if (keySize == 0) {
if (readKey) {
logReadFailure("key", "key size is 0", levelNum, bucketIdx, slotIdx)
}
return null
}
if (readKey) {
val keyAddr = valuesIo.position()
val key = keyExternalizer.read(valuesIo)
val readBytes = (valuesIo.position() - keyAddr).toInt()
check(readBytes == keySize) {
"Key size mismatch. Expected to read $keySize bytes, but $readBytes bytes were read."
}
return key to keySize
}
if (keySize > 0) {
valuesIo.skipBytes(keySize)
}
return null to keySize
}
/**
* Read the key at the given slot position.
*/
fun readKey(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
): K? {
return visitKeyRegion(levelNum = levelNum, bucketIdx = bucketIdx,
slotIdx = slotIdx, readKey = true)?.first
}
/**
* Read the value at the given slot position.
*/
fun readValue(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
): V? {
if (visitKeyRegion(levelNum = levelNum, bucketIdx = bucketIdx,
slotIdx = slotIdx, readKey = false)?.second == null
) {
// if the key is empty (specifically, keySize == 0), then this slot is empty
return null
}
val valueSize = valuesIo.readInt()
if (valueSize == 0) {
logReadFailure("value", "value size is 0", levelNum, bucketIdx, slotIdx)
return null
}
val valAddr = valuesIo.position()
val value = valueExternalizer.read(valuesIo)
val readBytes = (valuesIo.position() - valAddr).toInt()
check(readBytes == valueSize) {
"Value size mismatch. Expected to read $valueSize bytes, but $readBytes bytes were read."
}
return value
}
fun writeEntry(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
key: K?,
value: V?,
) {
val (segment, positionInSegment) = segmentAndPosForPos(levelNum, bucketIdx,
slotIdx)
if (key == null) {
// delete entry
deleteEntry(segment, positionInSegment)
return
}
// TODO(itsaky): Punch holes in the region of the existing entry on update
// to ensure the disk space is released
// leave space for the 'is_occupied' token
val tokenAddr = valuesWriteAt
valuesIo.position(tokenAddr)
valuesIo.seekBoolean()
// write key and value
writeSlotKeyOrVal(key, keyExternalizer)
writeSlotKeyOrVal(value, valueExternalizer)
val finalAddr = valuesIo.position()
// then set the token
valuesIo.position(tokenAddr)
valuesIo.writeBoolean(true)
// reset to the final position
valuesIo.position(finalAddr)
// then update the address in the keymap
segment.position(positionInSegment)
segment.writeLong(tokenAddr + 1)
valuesWriteAt = finalAddr
}
private fun deleteEntry(
segment: KeymapSegment,
positionInSegment: Long,
readValue: Boolean = false,
): V? {
segment.position(positionInSegment)
val valueAddr = segment.readLong()
if (valueAddr == 0L) {
return null
}
FileUtils.deallocate(raKeymapFile, positionInSegment, SIZE_LONG)
valuesIo.position(valueAddr - 1)
var entrySize = 0L
if (!valuesIo.readBoolean()) {
// slot is not occupied
return null
}
entrySize += SIZE_BOOLEAN // isOccupied
val keySize = valuesIo.readInt()
entrySize += SIZE_INT
entrySize += keySize
if (keySize > 0) {
valuesIo.seekRelative(keySize)
}
val valueSize = valuesIo.readInt()
entrySize += SIZE_INT
entrySize += valueSize
var value: V? = null
if (readValue && valueSize > 0) {
val valAddr = valuesIo.position()
value = valueExternalizer.read(valuesIo)
// ensure that the externalizer read exactly the expected bytes
check((valuesIo.position() - valAddr).toInt() == valueSize)
} else if (valueSize > 0) {
valuesIo.seekRelative(valueSize)
}
// punch holes in the file
FileUtils.deallocate(raIndexFile, getValuesRealOffset(valueAddr - 1),
entrySize)
// TODO(itsaky): When deleting an entry, update the corresponding keymap entry with -1
// TODO(itsaky): When an entry is deleted, update valueFirstEntryAt and valueLastEntryAt accordingly
return value
}
fun clear() {
// TODO: This should not clear the header data in the files
FileUtils.deallocate(raKeymapFile, 0, KEYMAP_SEGMENT_SIZE_BYTES * KEYMAP_MAX_SEGMENTS)
FileUtils.deallocate(raIndexFile, 0, valuesMapSize + VALUES_HEADER_SIZE_BYTES)
// FileUtils.deallocate(raMetaFile)
}
fun tryMoveToInterim(
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
key: K?,
value: V?,
): V? {
TODO()
}
private fun <T> writeSlotKeyOrVal(obj: T?, externalizer: DataExternalizer<T>
) {
if (obj != null) {
// obj is not null
// leave space for the obj size and write the key
// then write the number of bytes written for obj to the sizeAddr
val sizeAddr = valuesIo.position()
valuesIo.seekInt()
val valueAddr = valuesIo.position()
externalizer.write(valuesIo, obj)
val finalAddr = valuesIo.position()
valuesIo.position(sizeAddr)
valuesIo.writeInt((finalAddr - valueAddr).toInt().also {
check(it > 0) { "Key/Value must be at least 1 byte in size" }
})
valuesIo.position(finalAddr)
} else {
// object is null, write the size of the obj as 0
// since obj size is 0, we don't need to write the obj
valuesIo.writeInt(0)
}
}
override fun close() {
keymapSegments.forEach { segment ->
try {
segment?.unmap()
} catch (err: Throwable) {
logger.error("Failed to unmap segment", err)
}
}
try {
raIndexFile.close()
} catch (e: Throwable) {
logger.error("Failed to close index file", e)
}
try {
raKeymapFile.close()
} catch (e: Throwable) {
logger.error("Failed to close keymap file", e)
}
try {
raMetaFile.close()
} catch (e: Throwable) {
logger.error("Failed to close meta file", e)
}
}
private fun logReadFailure(
what: String,
reason: String,
levelNum: Int,
bucketIdx: Int,
slotIdx: Int,
) {
log.info(
"Cannot read '$what' at slot ($reason): level={}, bucket={}, slot={}",
levelNum, bucketIdx, slotIdx)
}
companion object {
private val log = LoggerFactory.getLogger(PersistentLevelHashIO::class.java)
/**
* The number of bytes it takes to store the magic number of the keymap/values
* file.
*/
private const val MAGIC_NUMBER_SIZE_BYTES = SIZE_LONG
/**
* The number of bytes in the header of the keymap file that is used to store
* the segment count.
*/
private const val KEYMAP_HEADER_SEGCOUNT_BYTES = SIZE_INT
/**
* The number of bytes in the header of the keymap file that is used to store
* the size of the segment structures.
*/
private const val KEYMAP_HEADER_SEGSIZE_BYTES = SIZE_LONG
/**
* The number of bytes used to store the header of the keymap file.
*/
private const val KEYMAP_HEADER_SIZE_BYTES: Long =
MAGIC_NUMBER_SIZE_BYTES + KEYMAP_HEADER_SEGCOUNT_BYTES + KEYMAP_HEADER_SEGSIZE_BYTES
/**
* The size of one segment in the key map file.
*/
private const val KEYMAP_SEGMENT_SIZE_BYTES: Long = 1024 * 1024 // 1MB
/**
* The number of bytes that are used to store an entry in a key map file.
*/
private const val KEYMAP_ENTRY_BYTES: Long = SIZE_LONG
/**
* The number of entries that can be stored in a segment in the key map file.
*/
private const val KEYMAP_ENTRIES_PER_SEGMENT =
KEYMAP_SEGMENT_SIZE_BYTES / KEYMAP_ENTRY_BYTES
/**
* The maximum number of segments that can be stored for an index.
*/
private const val KEYMAP_MAX_SEGMENTS = 100
/**
* The maximum number of entries that can be stored in all of the keymap segments combined.
*/
const val KEYMAP_MAX_ENTRIES =
KEYMAP_MAX_SEGMENTS * KEYMAP_ENTRIES_PER_SEGMENT
/**
* Magic number that is used as the file signature to identify the keymap file.
*/
private const val KEYMAP_MAGIC_NUMBER = 0x414944584B
/**
* Magic number that is used as the file signature to identify the values file.
*/
private const val VALUES_MAGIC_NUMBER = 0x4149445856
/**
* The size of the header (bytes) in the values file.
*/
private const val VALUES_HEADER_SIZE_BYTES: Long =
MAGIC_NUMBER_SIZE_BYTES + SIZE_LONG + // first entry address
SIZE_LONG // next entry address
/**
* The maximum size of the region of the values file that can be mapped into
* memory at a given time.
*/
private const val VALUES_MAX_MAP_SIZE_BYTES = 512L * 1024L;
/**
* Get the ID of the segment that should be used to store the store the
* given position hash.
*/
private fun segmentIndex(positionHash: HashT): Int {
return (positionHash % KEYMAP_MAX_SEGMENTS.toULong()).toInt()
}
/**
* Get the position of the entry in the segment where the entry should be
* written. The returned position is in bytes from the beginning of the
* segment.
*/
private fun positionInSegment(positionHash: HashT): Long {
return ((positionHash % KEYMAP_ENTRIES_PER_SEGMENT.toULong()).toLong() * KEYMAP_ENTRY_BYTES)
}
/**
* Get the hash of the position coordinates.
*/
private fun hash(levelNum: Int, bucketIdx: Int, slotIdx: Int
): HashT {
var hash = 17L
hash = 31L * hash + levelNum
hash = 31L * hash + bucketIdx
hash = 31L * hash + slotIdx
return hash.toULong()
}
}
/**
* A segment of the key map file.
*/
private class KeymapSegment(
val file: RandomAccessFile,
val path: String,
private val segmentOffset: Long,
private val segmentLength: Long,
private val io: MappedRandomAccessIO = MappedRandomAccessIO(),
) : DataInput by io, DataOutput by io, RandomAccessIO by io, AutoCloseable {
lateinit var mappedBuffer: MappedByteBuffer
init {
remap(segmentOffset, segmentLength, true)
}
/**
* Map a region of the backing file into memory, unmapping previous mappings,
* if any.
*/
@Suppress("SameParameterValue")
private fun remap(position: Long, size: Long, forceRemap: Boolean = false
) {
if (!forceRemap && (segmentOffset == position && segmentLength == size)) {
// the given region is already mapped into memory
return
}
unmap()
this.mappedBuffer =
file.channel.map(FileChannel.MapMode.READ_WRITE, segmentOffset,
segmentLength)
this.io.reset(mappedBuffer, segmentOffset, segmentLength)
}
/**
* Unmap the memory mapped region, if mapped.
*/
override fun close() {
unmap()
}
/**
* Unmap the memory mapped buffer. This [KeymapSegment] becomes invalid after
* this operation and must not be used further unless remapped.
*
* @return `true` if the unmapping request was successful, `false` otherwise.
*/
fun unmap(): Boolean {
if (!::mappedBuffer.isInitialized) {
return false
}
try {
val cleanerF = mappedBuffer.javaClass.getDeclaredField("cleaner")
cleanerF.isAccessible = true
val cleaner = cleanerF.get(mappedBuffer)
val cleanM = cleaner.javaClass.getDeclaredMethod("clean")
cleanM.isAccessible = true
cleanM.invoke(cleaner)
return true
} catch (err: Throwable) {
logger.error(
"Failed to unmap keymap segment. file={}, offset={}, size={}", path,
segmentOffset, segmentLength, err)
return false
}
}
companion object {
/**
* Create a new segment.
*
* @param offset The offset of the segment in the keymap file.
* @param file The file to create the segment in.
* @param path The path of the file (used for debugging).
*/
fun create(offset: Long, file: RandomAccessFile, path: String
): KeymapSegment {
return KeymapSegment(file = file, path = path, segmentOffset = offset,
segmentLength = KEYMAP_SEGMENT_SIZE_BYTES.toLong())
}
}
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/levelHashingExts.kt
+68
View File
@@ -0,0 +1,68 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
import com.google.common.hash.Hashing
import java.nio.charset.StandardCharsets
import kotlin.random.Random
private val seeds by lazy {
// val random = Random(System.currentTimeMillis())
val random = Random(6248403840530382848)
var seed1: HashT
var seed2: HashT
do {
// its fine to perform unchecked sign conversion here
seed1 = random.nextLong().toULong()
seed2 = random.nextLong().toULong()
// see docs for `shl` function for more details about why '63' is used
seed1 = seed1 shl (random.nextLong().toULong() % 63UL).toInt()
seed2 = seed2 shl (random.nextLong().toULong() % 63UL).toInt()
} while (seed1 == seed2)
seed1 to seed2
}
/**
* Generate two randomized seeds for hash functions.
*/
internal fun levelHashDefaultSeeds() : Pair<HashT, HashT> {
return seeds
}
/**
* A simple [LevelHashFn] for hashing string keys using FarmHash's Fingerprint64, an open-source
* algorithm.
*
* @param key The key to hash.
* @param seed The seed to use for hashing. For this implementation, this value is simply hashed
* along with the string key.
*/
@Suppress("UnstableApiUsage")
fun stringHash(key: String, seed: HashT) : HashT {
val hasher = Hashing.farmHashFingerprint64().newHasher()
return hasher
.putLong(seed.toLong())
.putString(key, StandardCharsets.UTF_8)
.hash()
.asLong()
.toULong()
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/levelTypes.kt
+35
View File
@@ -0,0 +1,35 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
/**
* The result of a hashing operation. This is defined as a 64-bit unsigned long
* as the value must always be positive in order to compute the bucket index
* correctly.
*/
typealias HashT = ULong
/**
* A function which computes the hash of a given key.
*/
typealias LevelHashFn<K> = (key: K, seed: HashT) -> HashT
/**
* A function which generates two seed values for the random generator.
*/
typealias SeedGeneratorFn = () -> Pair<HashT, HashT>
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/util/DataExternalizers.kt
+182
View File
@@ -0,0 +1,182 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.util
import com.itsaky.androidide.levelhash.DataExternalizer
import java.io.DataInput
import java.io.DataOutput
/**
* [DataExternalizer] implementations for various types.
*
* @author Akash Yadav
*/
object DataExternalizers {
const val SIZE_BOOLEAN = 1L
const val SIZE_BYTE = 1L
const val SIZE_SHORT = 2L
const val SIZE_CHAR = 2L
const val SIZE_INT = 4L
const val SIZE_LONG = 8L
const val SIZE_FLOAT = 4L
const val SIZE_DOUBLE = 8L
/**
* Data externalizer for unsigned [Long]s.
*/
val BOOLEAN = object : DataExternalizer<Boolean> {
override fun read(input: DataInput): Boolean {
return input.readBoolean()
}
override fun write(out: DataOutput, data: Boolean) {
out.writeBoolean(data)
}
}
/**
* Data externalizer for [Byte]s.
*/
val BYTE = object : DataExternalizer<Byte> {
override fun read(input: DataInput): Byte {
return input.readByte()
}
override fun write(out: DataOutput, data: Byte) {
out.write(data.toInt())
}
}
/**
* Data externalizer for unsigned [Byte]s.
*/
val UBYTE = object : DataExternalizer<Int> {
override fun read(input: DataInput): Int {
return input.readUnsignedByte()
}
override fun write(out: DataOutput, data: Int) {
out.writeByte(data)
}
}
/**
* Data externalizer for [Short]s.
*/
val SHORT = object : DataExternalizer<Short> {
override fun read(input: DataInput): Short {
return input.readShort()
}
override fun write(out: DataOutput, data: Short) {
out.writeShort(data.toInt())
}
}
/**
* Data externalizer for unsigned [Short]s.
*/
val USHORT = object : DataExternalizer<Int> {
override fun read(input: DataInput): Int {
return input.readUnsignedShort()
}
override fun write(out: DataOutput, data: Int) {
out.writeShort(data)
}
}
/**
* Data externalizer for [Char]s.
*/
val CHAR = object : DataExternalizer<Char> {
override fun read(input: DataInput): Char {
return input.readChar()
}
override fun write(out: DataOutput, data: Char) {
out.writeChar(data.code)
}
}
/**
* Data externalizer for [Int]s.
*/
val INT = object : DataExternalizer<Int> {
override fun read(input: DataInput): Int {
return input.readInt()
}
override fun write(out: DataOutput, data: Int) {
out.write(data)
}
}
/**
* Data externalizer for [Long]s.
*/
val LONG = object : DataExternalizer<Long> {
override fun read(input: DataInput): Long {
return input.readLong()
}
override fun write(out: DataOutput, data: Long) {
out.writeLong(data)
}
}
/**
* Data externalizer for [Float]s.
*/
val FLOAT = object : DataExternalizer<Float> {
override fun read(input: DataInput): Float {
return input.readFloat()
}
override fun write(out: DataOutput, data: Float) {
out.writeFloat(data)
}
}
/**
* Data externalizer for [Double]s.
*/
val DOUBLE = object : DataExternalizer<Double> {
override fun read(input: DataInput): Double {
return input.readDouble()
}
override fun write(out: DataOutput, data: Double) {
out.writeDouble(data)
}
}
/**
* Data externalizer for [String]s.
*/
val STRING = object : DataExternalizer<String> {
override fun read(input: DataInput): String {
return input.readUTF()
}
override fun write(out: DataOutput, data: String) {
out.writeUTF(data)
}
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/util/FileUtils.kt
+43
View File
@@ -0,0 +1,43 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.util
import com.itsaky.androidide.jna.Falloc
import com.itsaky.androidide.jna.LibC
import com.itsaky.androidide.utils.FDUtils
import com.sun.jna.Native
import java.io.RandomAccessFile
/**
* @author Akash Yadav
*/
internal object FileUtils {
/**
* Deallocate the given region of the file. Uses `fallocate(2)` with
* [Falloc.FALLOC_FL_PUNCH_HOLE].
*/
fun deallocate(file: RandomAccessFile, offset: Long, len: Long) {
val fd = FDUtils.getFd(file.fd)
val result = LibC.INSTANCE.fallocate(fd,
Falloc.FALLOC_FL_PUNCH_HOLE or Falloc.FALLOC_FL_KEEP_SIZE, offset, len)
if (result != 0) {
throw RuntimeException("fallocate(2) failed: ${Native.getLastError()}")
}
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/util/MappedRandomAccessIO.kt
+207
View File
@@ -0,0 +1,207 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.util
import com.itsaky.androidide.levelhash.RandomAccessIO
import com.itsaky.androidide.levelhash.seekByte
import com.itsaky.androidide.levelhash.seekChar
import com.itsaky.androidide.levelhash.seekDouble
import com.itsaky.androidide.levelhash.seekFloat
import com.itsaky.androidide.levelhash.seekInt
import com.itsaky.androidide.levelhash.seekLong
import com.itsaky.androidide.levelhash.seekShort
import com.itsaky.androidide.levelhash.util.DataExternalizers.SIZE_BYTE
import java.io.DataInput
import java.io.DataInputStream
import java.io.DataOutput
import java.io.EOFException
import java.nio.MappedByteBuffer
import kotlin.math.min
/**
* Implementation of the [DataInput] and [DataOutput]
* backed by a [MappedByteBuffer].
*
* @author Akash Yadav
*/
internal class MappedRandomAccessIO : DataInput, DataOutput, RandomAccessIO {
private var position = 0L
private var size = 0L
private lateinit var buffer: MappedByteBuffer
fun reset(buffer: MappedByteBuffer, offset: Long, length: Long) {
this.buffer = buffer
this.position = offset
this.size = length
}
private fun buf(): MappedByteBuffer = buf(this.position)
private fun buf(position: Long): MappedByteBuffer {
return this.buffer.apply { position(position.toInt()) }
}
override fun position(position: Long) {
val currentPos = position()
if (currentPos == position) return
require(position >= 0L) { "position must be >= 0, position=$position" }
require(
position < size) { "position must be < size:$size, position=$position" }
this.position = position
}
override fun position(): Long {
return this.position
}
fun size() = this.size
override fun readFully(bytes: ByteArray) {
if (bytes.isEmpty()) return
readFully(bytes, 0, bytes.size)
}
override fun readFully(bytes: ByteArray, offset: Int, length: Int) {
if (this.position() + length > size()) {
throw EOFException("Cannot read past end of segment")
}
for (i in 0 until length) {
bytes[offset + i] = readByte()
}
}
override fun skipBytes(count: Int): Int {
val position = position()
val avltoRead = this.size() - position
val skip = min(avltoRead.toInt(), count)
position(position + skip)
return skip
}
override fun readBoolean(): Boolean {
return readUnsignedByte() == 1
}
override fun readByte(): Byte {
return buf(position).get().also {
position += SIZE_BYTE
}
}
override fun readUnsignedByte(): Int {
val b = readByte()
return b.toInt() and 0xFF
}
override fun readShort(): Short {
return buf().short.also { seekShort() }
}
override fun readUnsignedShort(): Int {
val r = readShort()
return r.toInt() and 0xFFFF
}
override fun readChar(): Char {
return buf().char.also { seekChar() }
}
override fun readInt(): Int {
return buf().int.also { seekInt() }
}
override fun readLong(): Long {
return buf().long.also { seekLong() }
}
override fun readFloat(): Float {
return buf().float.also { seekFloat() }
}
override fun readDouble(): Double {
return buf().double.also { seekDouble() }
}
override fun readLine(): String? {
throw UnsupportedOperationException()
}
override fun readUTF(): String {
return DataInputStream.readUTF(this)
}
override fun write(b: Int) {
buf().put(b.toByte())
seekByte()
}
override fun write(bytes: ByteArray) {
write(bytes, 0, bytes.size)
}
override fun write(bytes: ByteArray, offset: Int, length: Int) {
buf().put(bytes, offset, length).also { seekRelative(length) }
}
override fun writeBoolean(boolean: Boolean) {
write(if (boolean) 1 else 0)
}
override fun writeByte(byte: Int) {
write(byte)
}
override fun writeShort(short: Int) {
buf().putShort(short.toShort()).also { seekShort() }
}
override fun writeChar(char: Int) {
buf().putChar(char.toChar()).also { seekChar() }
}
override fun writeInt(int: Int) {
buf().putInt(int).also { seekInt() }
}
override fun writeLong(long: Long) {
buf().putLong(long).also { seekLong() }
}
override fun writeFloat(float: Float) {
buf().putFloat(float).also { seekFloat() }
}
override fun writeDouble(double: Double) {
buf().putDouble(double).also { seekDouble() }
}
override fun writeBytes(str: String) {
throw UnsupportedOperationException()
}
override fun writeChars(str: String) {
throw UnsupportedOperationException()
}
override fun writeUTF(str: String) {
UtfIO.writeUTF(str, this)
}
}
utilities/level-hash/src/main/java/com/itsaky/androidide/levelhash/util/UtfIO.java
+85
View File
@@ -0,0 +1,85 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash.util;
import java.io.DataOutput;
import java.io.IOException;
import java.io.UTFDataFormatException;
/**
* @author Akash Yadav
*/
public class UtfIO {
// From DataOutputStream
// https://cs.android.com/android/platform/superproject/main/+/main:libcore/ojluni/src/main/java/java/io/DataOutputStream.java
public static int writeUTF(String str, DataOutput out) throws IOException {
int strlen = str.length();
int utflen = 0;
int c, count = 0;
/* use charAt instead of copying String to char array */
for (int i = 0; i < strlen; i++) {
c = str.charAt(i);
if ((c >= 0x0001) && (c <= 0x007F)) {
utflen++;
} else if (c > 0x07FF) {
utflen += 3;
} else {
utflen += 2;
}
}
if (utflen > 65535) {
throw new UTFDataFormatException("encoded string too long: " + utflen + " bytes");
}
byte[] bytearr = new byte[utflen + 2];
bytearr[count++] = (byte) ((utflen >>> 8) & 0xFF);
bytearr[count++] = (byte) ((utflen) & 0xFF);
int i;
for (i = 0; i < strlen; i++) {
c = str.charAt(i);
if (!((c >= 0x0001) && (c <= 0x007F))) {
break;
}
bytearr[count++] = (byte) c;
}
for (; i < strlen; i++) {
c = str.charAt(i);
if ((c >= 0x0001) && (c <= 0x007F)) {
bytearr[count++] = (byte) c;
} else if (c > 0x07FF) {
bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
} else {
bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
bytearr[count++] = (byte) (0x80 | ((c) & 0x3F));
}
}
out.write(bytearr, 0, utflen + 2);
return utflen + 2;
}
}
utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/InMemoryLevelHashTest.kt
+199
View File
@@ -0,0 +1,199 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
import com.google.common.truth.Truth.assertThat
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import kotlin.math.pow
/**
* @author Akash Yadav
*/
@RunWith(JUnit4::class)
class InMemoryLevelHashTest {
private fun <V: String?> stringStringLevelHash(conf: InMemoryHashBuilder<String, V>.() -> Unit = {}) =
LevelHash.inMemoryHashBuilder<String, V>()
.apply(conf)
.levelSize(2)
.hashFn(::stringHash)
.build()
@Test
fun `test simple hash insert`() {
val hash = stringStringLevelHash<String>()
hash.insert("Hello", "World")
val slot = hash.findSlot("Hello")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("Hello")
assertThat(slot.value).isEqualTo("World")
}
@Test(expected = LevelHash.DuplicateKeyException::class)
fun `test simple hash key re-insertion`() {
val hash = stringStringLevelHash<String> { uniqueKeys(true) }
hash.insert("Hello", "World")
val slot = hash.findSlot("Hello")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("Hello")
assertThat(slot.value).isEqualTo("World")
hash.insert("Hello", "World")
}
@Test
fun `test flaky key hash computation`() {
repeat(100) {
// sometimes, the calculated fidx and sidx values are invalid bucket indices
// this happens due to the two randomly generated seeds are (were) not deterministic
// this test is to ensure that changes in the default SeedGeneratorFn do not
// lead to invalid bucket indices again
val hash = stringStringLevelHash<String>()
hash.insert("key1", "value1")
hash.insert("key2", "value2")
}
}
@Test(expected = LevelHash.OverflowException::class)
fun `test bucket limit`() {
val hash = stringStringLevelHash<String> { autoExpand(false) }
// fill up the level hash
repeat(hash.totalSlotCount) { idx ->
assertThat(hash.insert("key${idx}", "value${idx}")).isTrue()
}
// inserting any more elements must fail
assertThat(hash.insert("key${hash.totalSlotCount}",
"value${hash.totalSlotCount}")).isFalse()
}
@Test
fun `test level hash auto expands`() {
val hash = stringStringLevelHash<String> { autoExpand(true) }
val levelSize = hash.levelSize
val slotCount = hash.totalSlotCount
val topBucketCount = hash.topLevelBucketCount
val bucketCount = hash.totalBucketCount
assertThat(levelSize).isEqualTo(2)
assertThat(topBucketCount).isEqualTo(2.0.pow(levelSize).toLong())
assertThat(bucketCount).isEqualTo(
(2.0.pow(levelSize) + 2.0.pow(levelSize shr 1)).toLong())
assertThat(slotCount).isEqualTo(bucketCount * hash.bucketSize)
// fill up the level hash
repeat(hash.totalSlotCount) { idx ->
assertThat(hash.insert("key${idx}", "value${idx}")).isTrue()
}
// inserting a new element must expand the level hash
hash.insert("key${hash.totalSlotCount}", "value${hash.totalSlotCount}")
// level size is incremented by 1
assertThat(hash.levelSize).isEqualTo(levelSize + 1)
assertThat(hash.bucketSize).isEqualTo(LevelHash.BUCKET_SIZE_DEFAULT)
// level hash doubles its capacity after an expansion
assertThat(hash.totalSlotCount).isEqualTo(slotCount * 2)
assertThat(hash.totalBucketCount).isEqualTo(bucketCount * 2)
assertThat(hash.topLevelBucketCount).isEqualTo(topBucketCount * 2)
}
@Test
fun `test value update for existing entry`() {
val hash = stringStringLevelHash<String>()
assertThat(hash.insert("k", "v")).isTrue()
var slot = hash.findSlot("k")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("k")
assertThat(slot.value).isEqualTo("v")
assertThat(hash.set("k", "newV")).isEqualTo("v")
slot = hash.findSlot("k")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("k")
assertThat(slot.value).isEqualTo("newV")
}
@Test(expected = LevelHash.EntryNotFoundException::class)
fun `test value update for non-existing entry`() {
val hash = stringStringLevelHash<String>()
assertThat(hash.insert("k", "v")).isTrue()
val slot = hash.findSlot("k")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("k")
assertThat(slot.value).isEqualTo("v")
assertThat(hash.set("kk", "newV")).isNull()
}
@Test
fun `test value update for existing entry with null value`() {
val hash = stringStringLevelHash<String?>()
assertThat(hash.insert("k", null)).isTrue()
var slot = hash.findSlot("k")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("k")
assertThat(slot.value).isNull()
assertThat(hash.set("k", "newV")).isNull()
slot = hash.findSlot("k")
assertThat(slot).isNotNull()
assertThat(slot!!.key).isEqualTo("k")
assertThat(slot.value).isEqualTo("newV")
}
@Test
fun `test entry removal`() {
val hash = stringStringLevelHash<String>()
assertThat(hash.insert("key", "value")).isTrue()
assertThat(hash.findSlot("key")).isNotNull()
assertThat(hash.findSlot("key")?.value).isEqualTo("value")
assertThat(hash.remove("key")).isEqualTo("value")
assertThat(hash.findSlot("key")).isNull()
}
@Test
fun `test level hash clear`() {
val hash = stringStringLevelHash<String>()
assertThat(hash.insert("key", "value")).isTrue()
assertThat(hash.findSlot("key")).isNotNull()
hash.clear()
assertThat(hash.findSlot("key")).isNull()
}
}
utilities/level-hash/src/test/java/com/itsaky/androidide/levelhash/PersistentLevelHashTest.kt
+112
View File
@@ -0,0 +1,112 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.levelhash
import com.google.common.truth.Truth.assertThat
import com.itsaky.androidide.levelhash.LevelHash.OverflowException
import com.itsaky.androidide.levelhash.util.DataExternalizers
import com.itsaky.androidide.testing.common.LinuxOnlyTestRule
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import java.io.File
/**
* @author Akash Yadav
*/
@RunWith(JUnit4::class)
class PersistentLevelHashTest {
// Persistent level hash is only supported on Linux
@JvmField
@Rule
val linuxOnlyTestRule = LinuxOnlyTestRule()
private fun persistentStringHash(
fileName: String,
conf: PersistentHashBuilder<String, String>.() -> Unit = {}
): LevelHash<String, String> {
val indexFile = File("./build/sample-index/${fileName}.storage")
if (indexFile.parentFile!!.exists()) {
indexFile.parentFile!!.deleteRecursively()
}
indexFile.parentFile!!.mkdirs()
val hash = LevelHash.persistentHashBuilder<String, String>()
.keyExternalizer(DataExternalizers.STRING)
.valueExternalizer(DataExternalizers.STRING)
.indexFile(indexFile)
.apply(conf)
.levelSize(2)
.hashFn(::stringHash)
.build()
assertThat(hash).isNotNull()
return hash
}
@Test
fun testInsertion() {
val hash = persistentStringHash("insert")
val key = "key"
val value = "value"
assertThat(hash.insert(key, value)).isTrue()
val slot = hash.findSlot(key)
assertThat(slot).isNotNull()
assertThat(slot!!.value).isEqualTo(value)
}
@Test(expected = OverflowException::class)
fun testOverflow() {
val hash = persistentStringHash("overflow") { autoExpand(false) }
for (i in 0..<hash.totalSlotCount) {
val key = "key${i}"
val value = "value${i}"
assertThat(hash.insert(key, value)).isTrue()
}
assertThat(hash.insert("k", "v")).isFalse()
}
@Test
fun testRemoval() {
val hash = persistentStringHash("remove")
assertThat(hash.insert("key", "value")).isTrue()
assertThat(hash.findSlot("key")).isNotNull()
assertThat(hash.findSlot("key")?.value).isEqualTo("value")
assertThat(hash.remove("key")).isEqualTo("value")
assertThat(hash.findSlot("key")).isNull()
}
@Test
fun `test level hash clear`() {
val hash = persistentStringHash("clear")
assertThat(hash.insert("key", "value")).isTrue()
assertThat(hash.findSlot("key")).isNotNull()
hash.clear()
assertThat(hash.findSlot("key")).isNull()
}
}
utilities/shared/build.gradle.kts
+13 -1
View File
@@ -22,9 +22,17 @@ plugins {
id("org.jetbrains.kotlin.jvm")
}
tasks.withType<Test> {
jvmArgs("--add-opens", "java.base/java.io=ALL-UNNAMED")
}
dependencies {
// this is compileOnly in order to allow different variants of the library
// in Android and JVM environments.
compileOnly(libs.jna.core)
compileOnly(libs.jna.platform)
api(libs.androidx.annotation)
api(libs.androidx.collection)
@@ -36,4 +44,8 @@ dependencies {
testImplementation(libs.tests.google.truth)
testImplementation(libs.tests.junit)
testImplementation(libs.jna.core)
testImplementation(libs.jna.platform)
testImplementation(projects.testing.commonTest)
}
utilities/shared/src/main/java/com/itsaky/androidide/jna/Falloc.java
+99
View File
@@ -0,0 +1,99 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.jna;
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/**
* <code>falloc.h</code>
*
* @author Akash Yadav
*/
public interface Falloc {
/**
* default is extend size
*/
int FALLOC_FL_KEEP_SIZE = 0x01;
/**
* de-allocates range
*/
int FALLOC_FL_PUNCH_HOLE = 0x02;
/**
* reserved codepoint
*/
int FALLOC_FL_NO_HIDE_STALE = 0x04;
/**
* FALLOC_FL_COLLAPSE_RANGE is used to remove a range of a file without leaving a hole in the
* file. The contents of the file beyond the range being removed is appended to the start offset
* of the range being removed (i.e. the hole that was punched is "collapsed"), resulting in a file
* layout that looks like the range that was removed never existed. As such collapsing a range of
* a file changes the size of the file, reducing it by the same length of the range that has been
* removed by the operation.
* <p>
* Different filesystems may implement different limitations on the granularity of the operation.
* Most will limit operations to filesystem block size boundaries, but this boundary may be larger
* or smaller depending on the filesystem and/or the configuration of the filesystem or file.
* <p>
* Attempting to collapse a range that crosses the end of the file is considered an illegal
* operation - just use ftruncate(2) if you need to collapse a range that crosses EOF.
*/
int FALLOC_FL_COLLAPSE_RANGE = 0x08;
/**
* FALLOC_FL_ZERO_RANGE is used to convert a range of file to zeros preferably without issuing
* data IO. Blocks should be preallocated for the regions that span holes in the file, and the
* entire range is preferable converted to unwritten extents - even though file system may choose
* to zero out the extent or do whatever which will result in reading zeros from the range while
* the range remains allocated for the file.
* <p>
* This can be also used to preallocate blocks past EOF in the same way as with fallocate. Flag
* FALLOC_FL_KEEP_SIZE should cause the inode size to remain the same.
*/
int FALLOC_FL_ZERO_RANGE = 0x10;
/**
* FALLOC_FL_INSERT_RANGE is use to insert space within the file size without overwriting any
* existing data. The contents of the file beyond offset are shifted towards right by len bytes to
* create a hole. As such, this operation will increase the size of the file by len bytes.
* <p>
* Different filesystems may implement different limitations on the granularity of the operation.
* Most will limit operations to filesystem block size boundaries, but this boundary may be larger
* or smaller depending on the filesystem and/or the configuration of the filesystem or file.
* <p>
* Attempting to insert space using this flag at OR beyond the end of the file is considered an
* illegal operation - just use ftruncate(2) or fallocate(2) with mode 0 for such type of
* operations.
*/
int FALLOC_FL_INSERT_RANGE = 0x20;
/**
* FALLOC_FL_UNSHARE_RANGE is used to unshare shared blocks within the file size without
* overwriting any existing data. The purpose of this call is to preemptively reallocate any
* blocks that are subject to copy-on-write.
* <p>
* Different filesystems may implement different limitations on the granularity of the operation.
* Most will limit operations to filesystem block size boundaries, but this boundary may be larger
* or smaller depending on the filesystem and/or the configuration of the filesystem or file.
* <p>
* This flag can only be used with allocate-mode fallocate, which is to say that it cannot be used
* with the punch, zero, collapse, or insert range modes.
*/
int FALLOC_FL_UNSHARE_RANGE = 0x40;
}
utilities/shared/src/main/java/com/itsaky/androidide/jna/LibC.java
+42
View File
@@ -0,0 +1,42 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.jna;
import com.sun.jna.Native;
/**
* @author Akash Yadav
*/
public interface LibC extends com.sun.jna.platform.linux.LibC {
String NAME = "c";
LibC INSTANCE = Native.load(NAME, LibC.class);
/**
* `fallocate` system call.
*
* @param fd The file descriptor.
* @param mode Operation identifier. See {@link com.sun.jna.platform.linux.Fcntl Fcntl} for
* valid operation codes.
* @param offset The offset in the file to operate on.
* @param len The len of the region to operate on.
* @return The result of `fallocate`. See {@link com.sun.jna.platform.linux.Fcntl Fcntl} for more
* details.
*/
int fallocate(int fd, int mode, long offset, long len);
}
utilities/shared/src/main/java/com/itsaky/androidide/utils/FDUtils.java
+56
View File
@@ -0,0 +1,56 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.utils;
import java.io.FileDescriptor;
import java.lang.reflect.Field;
/**
* @author Akash Yadav
*/
public final class FDUtils {
private FDUtils() {
throw new UnsupportedOperationException();
}
/**
* Get the file descriptor for the
* @param descriptor
* @return
*/
public static int getFd(FileDescriptor descriptor) {
try {
Field fd;
try {
// Android
fd = descriptor.getClass().getDeclaredField("descriptor");
} catch (NoSuchFieldException e) {
// JVM
fd = descriptor.getClass().getDeclaredField("fd");
}
fd.setAccessible(true);
return fd.getInt(descriptor);
} catch (NoSuchFieldException
| IllegalAccessException
| IllegalArgumentException e
) {
throw new RuntimeException(e);
}
}
}
utilities/shared/src/test/java/com/itsaky/androidide/jna/LibCTest.kt
+103
View File
@@ -0,0 +1,103 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package com.itsaky.androidide.jna
import com.google.common.truth.Truth.assertThat
import com.itsaky.androidide.testing.common.LinuxOnlyTestRule
import com.itsaky.androidide.utils.FDUtils
import com.sun.jna.Native
import com.sun.jna.platform.linux.ErrNo
import com.sun.jna.platform.linux.Fcntl
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import java.io.File
import java.io.RandomAccessFile
import java.nio.charset.StandardCharsets
/**
* @author Akash Yadav
*/
@RunWith(JUnit4::class)
class LibCTest {
@JvmField
@Rule
val linuxOnlyTestRule = LinuxOnlyTestRule()
private fun createFile(name: String) : Pair<File, RandomAccessFile> {
val file = File("build/${name}.tmp")
val raFile = RandomAccessFile(file, "rw")
if (file.exists()) {
file.delete()
}
file.createNewFile()
return file to raFile
}
@Test
fun `test fallocate for sparse files`() {
val data = "random-access-file".toByteArray(StandardCharsets.UTF_8)
val (file, raFile) = createFile("fallocate-normal")
try {
val len = 1024L * 1024L
// set length to 1MB, effectively equals to a ftruncate call
raFile.setLength(len)
// seek to half of file length, and write some data
raFile.seek(len / 2)
raFile.write(data)
// ensure that the fallocate call works
val fileDescriptor = FDUtils.getFd(raFile.fd)
val result = LibC.INSTANCE.fallocate(fileDescriptor,
Falloc.FALLOC_FL_PUNCH_HOLE or Falloc.FALLOC_FL_KEEP_SIZE, len / 2,
data.size.toLong())
assertThat(result).isEqualTo(0)
} finally {
file.delete()
}
}
@Test
fun `test fallocate for bad fd`() {
val (file, raFile) = createFile("fallocate-normal")
try {
raFile.setLength(1024)
// get the file descriptor and then close the file
// this should make the file descriptor invalid
val fileDescriptor = FDUtils.getFd(raFile.fd)
raFile.close()
val result = LibC.INSTANCE.fallocate(fileDescriptor,
Falloc.FALLOC_FL_PUNCH_HOLE or Falloc.FALLOC_FL_KEEP_SIZE, 0, 1024)
assertThat(result).isEqualTo(-1)
assertThat(Native.getLastError()).isEqualTo(ErrNo.EBADF)
} finally {
file.delete()
}
}
}