fix(editor/treesitter): reimplement TsAnalyzeManager for better thread safety and error handling

This commit is contained in:
Akash Yadav
2023-10-23 21:27:30 +05:30
parent 5db4f33721
commit 7923243b3b
9 changed files with 481 additions and 46 deletions
@@ -18,6 +18,7 @@
package com.itsaky.androidide.treesitter.api
import com.itsaky.androidide.treesitter.TSNode
import com.itsaky.androidide.treesitter.TSParser
import com.itsaky.androidide.treesitter.TSQuery
import com.itsaky.androidide.treesitter.TSQueryCursor
import com.itsaky.androidide.treesitter.TSQueryMatch
@@ -131,7 +132,7 @@ private fun <ResultT> TSQueryCursor.doSafeExecQueryCursor(
}
if (!node.canAccess() || node.hasChanges()) {
log.debug("$debugName: Cannot execute query, node is not accessible or has been edited", "node.canAccess=${node.canAccess()}", "node.hasChanges=${node.hasChanges()}")
log.debug("$debugName: Cannot execute query, node is not accessible or has been edited", "node.canAccess=${node.canAccess()}", "node.hasChanges=${node.canAccess() && node.hasChanges()}")
return null
}
@@ -1,3 +1,20 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
/*******************************************************************************
* sora-editor - the awesome code editor for Android
* https://github.com/Rosemoe/sora-editor
@@ -38,7 +55,6 @@ import io.github.rosemoe.sora.lang.styling.TextStyle
import io.github.rosemoe.sora.text.CharPosition
import io.github.rosemoe.sora.text.Content
import io.github.rosemoe.sora.widget.schemes.EditorColorScheme
import java.util.concurrent.atomic.AtomicBoolean
/**
* Spans generator for tree-sitter. Results are cached.
@@ -54,8 +70,6 @@ class LineSpansGenerator(
private val spanFactory: TsSpanFactory
) : Spans {
private val isValid = AtomicBoolean(true)
companion object {
const val CACHE_THRESHOLD = 60
@@ -66,11 +80,6 @@ class LineSpansGenerator(
fun edit(edit: TSInputEdit) {
tree.edit(edit)
// node ranges change when the tree is edited
// in this case, the query cursor should be stopped in order to prevent SEGV_MAPERR
// when accessing the query matches
isValid.set(false)
}
fun queryCache(line: Int): MutableList<Span>? {
@@ -95,7 +104,7 @@ class LineSpansGenerator(
fun captureRegion(startIndex: Int, endIndex: Int): MutableList<Span> {
val list = mutableListOf<Span>()
if (!isValid.get() || !tree.canAccess()) {
if (!tree.canAccess()) {
list.add(emptySpan(0))
return list
}
@@ -109,12 +118,10 @@ class LineSpansGenerator(
query = languageSpec.tsQuery,
tree = tree,
recycleNodeAfterUse = true,
matchCondition = { isValid.get() },
onClosedOrEdited = {
captures.clear()
log.debug("Tree closed or edited while in captureRegion")
},
debugName = "LineSpansGenerator.captureRegion"
matchCondition = null,
onClosedOrEdited = null,
debugLogging = false,
debugName = "LineSpansGenerator.captureRegion()"
) { match ->
if (languageSpec.queryPredicator.doPredicate(
languageSpec.predicates,
@@ -180,7 +187,6 @@ class LineSpansGenerator(
(capture as? TreeSitterQueryCapture?)?.recycle()
}
if (lastIndex != endIndex) {
list.add(emptySpan(lastIndex))
}
@@ -0,0 +1,112 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package io.github.rosemoe.sora.editor.ts
import android.os.Bundle
import com.itsaky.androidide.treesitter.TSInputEdit
import io.github.rosemoe.sora.editor.ts.spans.DefaultSpanFactory
import io.github.rosemoe.sora.editor.ts.spans.TsSpanFactory
import io.github.rosemoe.sora.lang.analysis.AnalyzeManager
import io.github.rosemoe.sora.lang.analysis.StyleReceiver
import io.github.rosemoe.sora.lang.styling.Styles
import io.github.rosemoe.sora.text.CharPosition
import io.github.rosemoe.sora.text.ContentReference
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
/**
* @author Akash Yadav
*/
open class TSAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme) : AnalyzeManager {
var stylesReceiver: StyleReceiver? = null
var reference: ContentReference? = null
var spanFactory: TsSpanFactory = DefaultSpanFactory()
open var styles = Styles()
private var analyzeWorker: TsAnalyzeWorker? = null
protected val analyzerScope = CoroutineScope(
Dispatchers.Default + CoroutineName("TSAnalyzeManager"))
open fun updateTheme(theme: TsTheme) {
this.theme = theme
(styles.spans as LineSpansGenerator?)?.also {
it.theme = theme
}
}
override fun setReceiver(receiver: StyleReceiver?) {
stylesReceiver = receiver
analyzeWorker?.stylesReceiver = receiver
}
override fun reset(content: ContentReference, extraArguments: Bundle) {
reference = content
rerun()
}
override fun insert(start: CharPosition, end: CharPosition, insertedContent: CharSequence) {
val edit = TSInputEdit.create(start.index shl 1, start.index shl 1, end.index shl 1,
start.toTSPoint(), start.toTSPoint(), end.toTSPoint())
(styles.spans as LineSpansGenerator?)?.apply {
lineCount = reference!!.lineCount
edit(edit)
}
analyzeWorker?.onMod(Mod(TextMod(start.index, end.index, edit, insertedContent.toString())))
}
override fun delete(start: CharPosition, end: CharPosition, deletedContent: CharSequence) {
val edit = TSInputEdit.create(start.index shl 1, start.index shl 1, end.index shl 1,
start.toTSPoint(), start.toTSPoint(), end.toTSPoint())
(styles.spans as LineSpansGenerator?)?.apply {
lineCount = reference!!.lineCount
edit(edit)
}
analyzeWorker?.onMod(Mod(TextMod(start.index, end.index, edit, null)))
}
override fun rerun() {
analyzeWorker?.stop()
analyzeWorker = null
(styles.spans as LineSpansGenerator?)?.tree?.close()
styles.spans = null
styles = Styles()
val initText = reference?.reference?.toString() ?: ""
analyzeWorker = TsAnalyzeWorker(this, languageSpec, theme, styles, reference!!, spanFactory)
analyzeWorker!!.apply {
this.stylesReceiver = this@TSAnalyzeManager.stylesReceiver
init(Init(initText))
start(analyzerScope)
}
}
override fun destroy() {
analyzeWorker?.stop()
analyzeWorker = null
(styles.spans as LineSpansGenerator?)?.tree?.close()
styles.spans = null
spanFactory.close()
}
}
@@ -32,9 +32,7 @@ import com.itsaky.androidide.treesitter.TSParser
import com.itsaky.androidide.treesitter.TSQueryCursor
import com.itsaky.androidide.treesitter.TSTree
import com.itsaky.androidide.treesitter.api.TreeSitterInputEdit
import com.itsaky.androidide.treesitter.api.TreeSitterNode
import com.itsaky.androidide.treesitter.api.TreeSitterQueryCapture
import com.itsaky.androidide.treesitter.api.TreeSitterQueryMatch
import com.itsaky.androidide.treesitter.api.safeExecQueryCursor
import com.itsaky.androidide.treesitter.string.UTF16String
import com.itsaky.androidide.treesitter.string.UTF16StringFactory
@@ -55,7 +53,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
var currentReceiver: StyleReceiver? = null
var reference: ContentReference? = null
var extraArguments: Bundle? = null
var thread: TsLooperThread? = null
var spanFactory: TsSpanFactory = DefaultSpanFactory()
@@ -74,7 +71,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
override fun reset(content: ContentReference, extraArguments: Bundle) {
reference = content
this.extraArguments = extraArguments
rerun()
}
@@ -95,6 +91,18 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
insertedContent.toString()
)
)
(styles.spans as LineSpansGenerator?)?.apply {
lineCount = reference!!.lineCount
edit(TSInputEdit.create(
start.index * 2,
start.index * 2,
end.index * 2,
start.toTSPoint(),
start.toTSPoint(),
end.toTSPoint()
))
}
}
override fun delete(start: CharPosition, end: CharPosition, deletedContent: CharSequence) {
@@ -114,6 +122,18 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
null
)
)
(styles.spans as LineSpansGenerator?)?.apply {
lineCount = reference!!.lineCount
edit(TSInputEdit.create(
start.index * 2,
end.index * 2,
start.index * 2,
start.toTSPoint(),
end.toTSPoint(),
start.toTSPoint()
))
}
}
override fun rerun() {
@@ -124,8 +144,8 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
}
}
// thread should handle this
// (styles.spans as LineSpansGenerator?)?.tree?.close()
(styles.spans as LineSpansGenerator?)?.tree?.close()
styles.spans = null
val initText = reference?.reference?.toString() ?: ""
@@ -144,8 +164,8 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
it.abort = true
}
}
// thread should handle this
// (styles.spans as LineSpansGenerator?)?.tree?.close()
(styles.spans as LineSpansGenerator?)?.tree?.close()
spanFactory.close()
}
@@ -175,7 +195,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
it.language = languageSpec.language
}
var tree: TSTree? = null
var spansGenerator: LineSpansGenerator? = null
fun offerMessage(what: Int, obj: Any?) {
val msg = Message.obtain()
@@ -190,7 +209,7 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
}
fun updateStyles() {
if (thread != this || !messageQueue.isEmpty()) {
if (thread != this || messageQueue.isNotEmpty()) {
return
}
@@ -205,9 +224,7 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
languageSpec,
scopedVariables,
spanFactory
).also {
spansGenerator = it
}
)
val oldBlocks = styles.blocks
updateCodeBlocks()
if (oldBlocks != null) {
@@ -280,7 +297,7 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
}
}
}
val distinct = blocks.distinct().toMutableList()
val distinct = blocks.asSequence().distinct().toMutableList()
styles.blocks = distinct
styles.finishBuilding()
}
@@ -306,6 +323,9 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
MSG_INIT -> {
localText.append(msg.obj!! as String)
if (!abort && !isInterrupted) {
if (parser.isParsing) {
parser.requestCancellation()
}
tree = parser.parseString(localText)
updateStyles()
}
@@ -314,10 +334,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
MSG_MOD -> {
if (!abort && !isInterrupted) {
val modification = msg.obj!! as TextModification
spansGenerator?.apply {
lineCount = reference!!.lineCount
edit(modification.tsEdition)
}
val newText = modification.changedText
val t = tree!!
t.edit(modification.tsEdition)
@@ -331,6 +347,9 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
}
}
(modification.tsEdition as? TreeSitterInputEdit?)?.recycle()
if (tree != null && parser.isParsing) {
parser.requestCancellation()
}
tree = parser.parseString(t, localText)
t.close()
updateStyles()
@@ -352,14 +371,6 @@ open class TsAnalyzeManager(val languageSpec: TsLanguageSpec, var theme: TsTheme
parser.close()
tree?.close()
localText.close()
try {
spansGenerator?.tree?.close()
} catch (e: Exception) {
e.printStackTrace()
} finally {
spansGenerator = null
}
}
}
@@ -0,0 +1,301 @@
/*
* This file is part of AndroidIDE.
*
* AndroidIDE is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* AndroidIDE is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with AndroidIDE. If not, see <https://www.gnu.org/licenses/>.
*/
package io.github.rosemoe.sora.editor.ts
import com.itsaky.androidide.treesitter.TSInputEdit
import com.itsaky.androidide.treesitter.TSParser
import com.itsaky.androidide.treesitter.TSQueryCursor
import com.itsaky.androidide.treesitter.TSTree
import com.itsaky.androidide.treesitter.api.TreeSitterInputEdit
import com.itsaky.androidide.treesitter.api.TreeSitterQueryCapture
import com.itsaky.androidide.treesitter.api.safeExecQueryCursor
import com.itsaky.androidide.treesitter.string.UTF16StringFactory
import com.itsaky.androidide.utils.ILogger
import io.github.rosemoe.sora.data.ObjectAllocator
import io.github.rosemoe.sora.editor.ts.spans.TsSpanFactory
import io.github.rosemoe.sora.lang.analysis.StyleReceiver
import io.github.rosemoe.sora.lang.styling.CodeBlock
import io.github.rosemoe.sora.lang.styling.Styles
import io.github.rosemoe.sora.text.ContentReference
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import java.util.concurrent.CancellationException
import java.util.concurrent.LinkedBlockingQueue
/**
* @author Akash Yadav
*/
internal class TsAnalyzeWorker(
private val analyzer: TSAnalyzeManager,
private val languageSpec: TsLanguageSpec,
private val theme: TsTheme,
private val styles: Styles,
private val reference: ContentReference,
private val spanFactory: TsSpanFactory
) {
companion object {
private const val TS_PARSE_DELAY_MS = 500L
private val log = ILogger.newInstance("TsAnalyzeWorker")
}
var stylesReceiver: StyleReceiver? = null
private val messageChannel = LinkedBlockingQueue<Message<*>>()
private var analyzerJob: Job? = null
private var parseJob: Job? = null
private var tree: TSTree? = null
private val localText = UTF16StringFactory.newString()
private val parser = TSParser.create().also {
it.language = languageSpec.language
}
fun init(init: Init) {
messageChannel.offer(init)
}
fun onMod(mod: Mod) {
messageChannel.offer(mod)
}
fun stop() {
analyzerJob?.cancel(CancellationException("Analyzer is being rerun"))
messageChannel.clear()
parser.close()
tree?.close()
localText.close()
}
fun start(scope: CoroutineScope) {
analyzerJob = scope.launch(Dispatchers.Default) {
while (isActive) {
processNextMessage()
}
}.also { job ->
job.invokeOnCompletion { error ->
if (error != null) {
log.error("Analyzer job failed", error)
} else {
log.info("Analyzer job completed")
}
}
}
}
private suspend fun CoroutineScope.processNextMessage() {
val message = withContext(Dispatchers.IO) {
messageChannel.take()
}
try {
when (message) {
is Init -> doInit(message)
is Mod -> doMod(message)
}
} catch (err: Throwable) {
log.error("AnalyzeWorker[lang=${languageSpec.language.name}, pendingMsgs=${messageChannel.size}] crashed", err)
}
}
private fun doInit(init: Init) {
if (parser.isParsing) {
parser.requestCancellation()
}
check(localText.isEmpty()) {
"'Init' must be the first message to TsAnalyzeWorker"
}
localText.append(init.data)
tree = parser.parseString(localText)
updateStyles()
}
private fun CoroutineScope.doMod(mod: Mod) {
check(localText.isNotEmpty()) {
"'Init' must be the first message to TsAnalyzeWorker"
}
val textMod = mod.data
val edit = textMod.edit
val newText = textMod.changedText
val oldTree = tree!!
oldTree.edit(edit)
if (newText == null) {
localText.delete(textMod.start, textMod.end)
} else {
if (textMod.start == localText.length) {
localText.append(newText)
} else {
localText.insert(textMod.start, newText)
}
}
(edit as? TreeSitterInputEdit?)?.recycle()
if (messageChannel.isNotEmpty()) {
// don't bother to parse if there are other messages to process
return
}
if (parser.isParsing) {
parser.requestCancellation()
}
parseJob?.also {
if (it.isActive) {
it.cancel(CancellationException("Parse is not needed and must be cancelled"))
}
}
parseJob = launch {
delay(TS_PARSE_DELAY_MS)
tree = parser.parseString(oldTree, localText)
oldTree.close()
updateStyles()
}.also {
it.invokeOnCompletion { error ->
if (error != null && error !is CancellationException) {
log.error("Failed to parse source", error)
}
}
}
}
private fun updateStyles() {
if (messageChannel.isNotEmpty()) {
// more message need to be processed
return
}
val tree = this.tree!!
val scopedVariables = TsScopedVariables(tree, localText, languageSpec)
val oldTree = (styles.spans as? LineSpansGenerator?)?.tree
val copied = tree.copy()
styles.spans = LineSpansGenerator(
copied,
reference.lineCount,
reference.reference,
theme,
languageSpec,
scopedVariables,
spanFactory
)
val oldBlocks = styles.blocks
updateCodeBlocks()
oldBlocks?.also { ObjectAllocator.recycleBlockLines(it) }
stylesReceiver?.setStyles(analyzer, styles) {
oldTree?.close()
}
stylesReceiver?.updateBracketProvider(analyzer, TsBracketPairs(copied, languageSpec))
}
private fun updateCodeBlocks() {
if (languageSpec.blocksQuery.patternCount == 0
|| !languageSpec.blocksQuery.canAccess()
|| tree?.canAccess() != true
) {
return
}
val blocks = mutableListOf<CodeBlock>()
TSQueryCursor.create().use { cursor ->
cursor.safeExecQueryCursor(
query = languageSpec.blocksQuery,
tree = tree,
recycleNodeAfterUse = true,
matchCondition = null,
whileTrue = null,
onClosedOrEdited = { blocks.clear() },
debugName = "TsAnalyzeManager.updateCodeBlocks()"
) { match ->
if (!languageSpec.blocksPredicator.doPredicate(
languageSpec.predicates,
localText,
match
)
) {
return@safeExecQueryCursor
}
match.captures.forEach { capture ->
val block = ObjectAllocator.obtainBlockLine()
var node = capture.node
val start = node.startPoint
block.startLine = start.row
block.startColumn = start.column / 2
val end = if (languageSpec.blocksQuery.getCaptureNameForId(capture.index)
.endsWith(".marked")
) {
// Goto last terminal element
while (node.childCount > 0) {
node = node.getChild(node.childCount - 1)
}
node.startPoint
} else {
node.endPoint
}
block.endLine = end.row
block.endColumn = end.column / 2
if (block.endLine - block.startLine > 1) {
blocks.add(block)
}
(capture as? TreeSitterQueryCapture?)?.recycle()
}
}
}
val distinct = blocks.asSequence().distinct().toMutableList()
styles.blocks = distinct
styles.finishBuilding()
}
}
internal interface Message<T> {
val data: T
}
internal data class Init(override val data: String) : Message<String>
internal data class Mod(override val data: TextMod) : Message<TextMod>
internal data class TextMod(
val start: Int,
val end: Int,
val edit: TSInputEdit,
val changedText: String?
)
@@ -61,7 +61,7 @@ open class TsLanguage(
protected var tsTheme = TsThemeBuilder(languageSpec.tsQuery).apply { themeDescription() }.theme
open val analyzer by lazy {
TsAnalyzeManager(languageSpec, tsTheme)
TSAnalyzeManager(languageSpec, tsTheme)
}
/**
@@ -18,6 +18,7 @@
package com.itsaky.androidide.editor.language.treesitter
import com.itsaky.androidide.editor.schemes.LanguageScheme
import io.github.rosemoe.sora.editor.ts.TSAnalyzeManager
import io.github.rosemoe.sora.editor.ts.TsAnalyzeManager
import io.github.rosemoe.sora.editor.ts.TsLanguageSpec
import io.github.rosemoe.sora.editor.ts.TsTheme
@@ -31,7 +32,7 @@ import io.github.rosemoe.sora.lang.styling.Styles
class TreeSitterAnalyzeManager(
languageSpec: TsLanguageSpec,
theme: TsTheme
) : TsAnalyzeManager(languageSpec, theme) {
) : TSAnalyzeManager(languageSpec, theme) {
override var styles: Styles = Styles()
set(value) {
+1 -1
View File
@@ -5,7 +5,7 @@ gradle-tooling = "8.3"
junit-jupiter = "5.10.0"
kotlin = "1.9.10"
kotlin-coroutines = "1.7.3"
tree-sitter = "4.0.0-internal.1+branch.dev.commit.c54287e.timestamp.20231015110809-SNAPSHOT"
tree-sitter = "4.0.0-internal.1+branch.dev.commit.0b69c66.timestamp.20231023090855-SNAPSHOT"
editor = "0.22.1-f7ee449-SNAPSHOT"
glide = "4.16.0"
androidx-vectordrawable = "1.1.0"
@@ -84,6 +84,9 @@ object TSJavaParser : IJavaParser<TSParseResult> {
parser.reset()
val watch = StopWatch("[TreeSitter] Parsing")
val content = file.getCharContent(false).toString()
if (parser.isParsing) {
parser.requestCancellation()
}
val parseTree = parser.parseString(content)
watch.log()