From 42bbe6a39c4a3b485267848cce530a114476b5c1 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Thu, 26 Oct 2023 17:53:27 +0530 Subject: [PATCH] fix(editor/treesitter): IOOB exception in TsAnalyzeWorker when deleting text --- .../treesitter/api/TreeSitterInputEdit.kt | 6 ++--- .../treesitter/api/TreeSitterObjectFactory.kt | 8 +++++- .../sora/editor/ts/TsAnalyzeManager.kt | 24 ++++++++++++----- .../rosemoe/sora/editor/ts/TsAnalyzeWorker.kt | 27 ++++++++++++------- .../io/github/rosemoe/sora/editor/ts/Utils.kt | 2 +- .../sora/editor/ts/predicate/builtin/Utils.kt | 15 ++++------- gradle/libs.versions.toml | 14 +++++----- 7 files changed, 57 insertions(+), 39 deletions(-) diff --git a/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterInputEdit.kt b/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterInputEdit.kt index 44416439e..4e451b619 100644 --- a/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterInputEdit.kt +++ b/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterInputEdit.kt @@ -41,9 +41,9 @@ class TreeSitterInputEdit @JvmOverloads internal constructor( fun obtain(startByte: Int, oldEndByte: Int, newEndByte: Int, - startPoint: TSPoint?, - oldEndPoint: TSPoint?, - newEndPoint: TSPoint?): TreeSitterInputEdit { + startPoint: TSPoint, + oldEndPoint: TSPoint, + newEndPoint: TSPoint): TreeSitterInputEdit { return obtainFromPool().apply { this.startByte = startByte diff --git a/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterObjectFactory.kt b/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterObjectFactory.kt index a0e1d38c1..2a9496a8c 100644 --- a/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterObjectFactory.kt +++ b/editor-api/src/main/java/com/itsaky/androidide/treesitter/api/TreeSitterObjectFactory.kt @@ -32,6 +32,8 @@ import com.itsaky.androidide.treesitter.TSRange import com.itsaky.androidide.treesitter.TSTree import com.itsaky.androidide.treesitter.TSTreeCursor import com.itsaky.androidide.treesitter.TSTreeCursorNode +import com.itsaky.androidide.treesitter.string.SynchronizedUTF16String +import com.itsaky.androidide.treesitter.string.UTF16String import com.itsaky.androidide.treesitter.util.TSObjectFactory /** @@ -42,7 +44,7 @@ import com.itsaky.androidide.treesitter.util.TSObjectFactory class TreeSitterObjectFactory : TSObjectFactory { override fun createInputEdit(startByte: Int, oldEndByte: Int, newEndByte: Int, - startPoint: TSPoint?, oldEndPoint: TSPoint?, newEndPoint: TSPoint?): TSInputEdit { + startPoint: TSPoint, oldEndPoint: TSPoint, newEndPoint: TSPoint): TSInputEdit { return TreeSitterInputEdit.obtain( startByte, oldEndByte, @@ -124,4 +126,8 @@ class TreeSitterObjectFactory : TSObjectFactory { override fun createLanguage(name: String?, pointers: LongArray?): TSLanguage { return TreeSitterNativeLanguage.obtain(name, pointers) } + + override fun createString(pointer: Long, isSynchronized: Boolean): UTF16String { + return if (isSynchronized) SynchronizedUTF16String(pointer) else UTF16String(pointer) + } } \ No newline at end of file diff --git a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeManager.kt b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeManager.kt index d4dfa3725..07057a32b 100644 --- a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeManager.kt +++ b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeManager.kt @@ -42,8 +42,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme open var styles = Styles() private var analyzeWorker: TsAnalyzeWorker? = null - protected val analyzerScope = CoroutineScope( - Dispatchers.Default + CoroutineName("TsAnalyzeManager")) open fun updateTheme(theme: TsTheme) { this.theme = theme @@ -63,8 +61,14 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme } override fun insert(start: CharPosition, end: CharPosition, insertedContent: CharSequence) { - val edit = TSInputEdit.create(start.index shl 1, start.index shl 1, end.index shl 1, - start.toTSPoint(), start.toTSPoint(), end.toTSPoint()) + val edit = TSInputEdit.create( + start.index shl 1, + start.index shl 1, + end.index shl 1, + start.toTSPoint(), + start.toTSPoint(), + end.toTSPoint() + )!! (styles.spans as LineSpansGenerator?)?.apply { lineCount = reference!!.lineCount edit(edit) @@ -73,8 +77,14 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme } override fun delete(start: CharPosition, end: CharPosition, deletedContent: CharSequence) { - val edit = TSInputEdit.create(start.index shl 1, start.index shl 1, end.index shl 1, - start.toTSPoint(), start.toTSPoint(), end.toTSPoint()) + val edit = TSInputEdit.create( + start.index shl 1, + end.index shl 1, + start.index shl 1, + start.toTSPoint(), + end.toTSPoint(), + start.toTSPoint() + )!! (styles.spans as LineSpansGenerator?)?.apply { lineCount = reference!!.lineCount edit(edit) @@ -96,7 +106,7 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme analyzeWorker!!.apply { this.stylesReceiver = this@TsAnalyzeManager.stylesReceiver init(Init(initText)) - start(analyzerScope) + start() } } diff --git a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeWorker.kt b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeWorker.kt index fb0770080..96ecc6dda 100644 --- a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeWorker.kt +++ b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/TsAnalyzeWorker.kt @@ -32,6 +32,7 @@ import io.github.rosemoe.sora.lang.analysis.StyleReceiver import io.github.rosemoe.sora.lang.styling.CodeBlock import io.github.rosemoe.sora.lang.styling.Styles import io.github.rosemoe.sora.text.ContentReference +import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -54,11 +55,13 @@ internal class TsAnalyzeWorker( ) { companion object { + private val log = ILogger.newInstance("TsAnalyzeWorker") } var stylesReceiver: StyleReceiver? = null + private val analyzerScope = CoroutineScope(Dispatchers.Default + CoroutineName("TsAnalyzeWorker")) private val messageChannel = LinkedBlockingQueue>() private var analyzerJob: Job? = null @@ -77,15 +80,15 @@ internal class TsAnalyzeWorker( } fun stop() { - analyzerJob?.cancel(CancellationException("Analyzer is being rerun")) messageChannel.clear() - parser.close() - tree?.close() + analyzerJob?.cancel(CancellationException("Requested to be stopped")) localText.close() + tree?.close() + parser.close() } - fun start(scope: CoroutineScope) { - analyzerJob = scope.launch(Dispatchers.Default) { + fun start() { + analyzerJob = analyzerScope.launch(Dispatchers.Default) { while (isActive) { processNextMessage() } @@ -101,9 +104,7 @@ internal class TsAnalyzeWorker( } private suspend fun processNextMessage() { - val message = withContext(Dispatchers.IO) { - messageChannel.take() - } + val message = withContext(Dispatchers.IO) { messageChannel.take() } try { when (message) { @@ -111,8 +112,14 @@ internal class TsAnalyzeWorker( is Mod -> doMod(message) } } catch (err: Throwable) { + val langName = languageSpec.language.name + val msgType = message.javaClass.simpleName + val msgTypeSuffix = if (message is Mod) { + "[start=${message.data.start}, end=${message.data.end}, type=${if (message.data.changedText == null) "delete" else "insert"}]" + } else "" + val pendingMsgs = messageChannel.size log.error( - "AnalyzeWorker[lang=${languageSpec.language.name}, pendingMsgs=${messageChannel.size}] crashed", + "AnalyzeWorker[lang=$langName, message=${msgType}${msgTypeSuffix}], pendingMsgs=$pendingMsgs] crashed", err) } } @@ -145,7 +152,7 @@ internal class TsAnalyzeWorker( oldTree.edit(edit) if (newText == null) { - localText.delete(textMod.start, textMod.end) + localText.deleteBytes(edit.startByte, edit.oldEndByte) } else { if (textMod.start == localText.length) { localText.append(newText) diff --git a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/Utils.kt b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/Utils.kt index e37d9eef9..de8188a5e 100644 --- a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/Utils.kt +++ b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/Utils.kt @@ -32,7 +32,7 @@ import io.github.rosemoe.sora.text.CharPosition /** * Convert a [CharPosition] object to a [TSPoint] object */ -fun CharPosition.toTSPoint() = TSPoint.create(line, column * 2) +fun CharPosition.toTSPoint(): TSPoint = TSPoint.create(line, column shl 1)!! fun TSQuery.validateOrThrow(name: String = "unknown") { if (errorType != TSQueryError.None) { diff --git a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/predicate/builtin/Utils.kt b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/predicate/builtin/Utils.kt index 8a9f86b7d..9e4853386 100644 --- a/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/predicate/builtin/Utils.kt +++ b/editor-treesitter/src/main/java/io/github/rosemoe/sora/editor/ts/predicate/builtin/Utils.kt @@ -52,15 +52,10 @@ fun getCaptureContent( captureName: String, text: CharSequence ) = match.captures.filter { tsQuery.getCaptureNameForId(it.index) == captureName } - .map { - if (text is UTF16String) { - val utf16Name = text.subseqChars(it.node.startByte / 2, it.node.endByte / 2) - val name = utf16Name.toString() - utf16Name.close() - name - } else if (text is Content) { - text.substring(it.node.startByte / 2, it.node.endByte / 2) - } else { - text.substring(it.node.startByte / 2, it.node.endByte / 2) + .map { capture -> + when (text) { + is UTF16String -> text.subseqBytes(capture.node.startByte, capture.node.endByte).use(UTF16String::toString) + is Content -> text.substring(capture.node.startByte shr 1, capture.node.endByte shr 1) + else -> text.substring(capture.node.startByte shr 1, capture.node.endByte shr 1) } } \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a1a0876ef..7dc674f69 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,7 +5,7 @@ gradle-tooling = "8.3" junit-jupiter = "5.10.0" kotlin = "1.9.10" kotlin-coroutines = "1.7.3" -tree-sitter = "4.0.0-internal.1+branch.dev.commit.1bfacde.timestamp.20231023182724-SNAPSHOT" +tree-sitter = "4.0.0-internal.1+branch.dev.commit.3936b64.timestamp.20231025142931-SNAPSHOT" editor = "0.22.1-f7ee449-SNAPSHOT" glide = "4.16.0" androidx-vectordrawable = "1.1.0" @@ -20,12 +20,12 @@ retrofit = "2.9.0" # AndroidIDE androidide-javac = { module = "io.github.itsaky:nb-javac-android", version = "17.0.0.4-SNAPSHOT" } -androidide-ts = { module = "com.itsaky.androidide:android-tree-sitter", version.ref = "tree-sitter" } -androidide-ts-java = { module = "com.itsaky.androidide:tree-sitter-java", version.ref = "tree-sitter" } -androidide-ts-json = { module = "com.itsaky.androidide:tree-sitter-json", version.ref = "tree-sitter" } -androidide-ts-kotlin = { module = "com.itsaky.androidide:tree-sitter-kotlin", version.ref = "tree-sitter" } -androidide-ts-log = { module = "com.itsaky.androidide:tree-sitter-log", version.ref = "tree-sitter" } -androidide-ts-xml = { module = "com.itsaky.androidide:tree-sitter-xml", version.ref = "tree-sitter" } +androidide-ts = { module = "com.itsaky.androidide.treesitter:android-tree-sitter", version.ref = "tree-sitter" } +androidide-ts-java = { module = "com.itsaky.androidide.treesitter:tree-sitter-java", version.ref = "tree-sitter" } +androidide-ts-json = { module = "com.itsaky.androidide.treesitter:tree-sitter-json", version.ref = "tree-sitter" } +androidide-ts-kotlin = { module = "com.itsaky.androidide.treesitter:tree-sitter-kotlin", version.ref = "tree-sitter" } +androidide-ts-log = { module = "com.itsaky.androidide.treesitter:tree-sitter-log", version.ref = "tree-sitter" } +androidide-ts-xml = { module = "com.itsaky.androidide.treesitter:tree-sitter-xml", version.ref = "tree-sitter" } # Common common-editor = { module = "io.github.Rosemoe.sora-editor:editor", version.ref = "editor" }