From cafec5ea878426a26f6dd10cd45c1fe0023ff04c Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Tue, 21 Jun 2022 19:17:49 +0530 Subject: [PATCH] Implemented incremental analyzer for CPP --- .../androidide/language/cpp/CppAnalyzer.java | 138 +++++------ .../androidide/language/cpp/CppLanguage.java | 224 +++++++++--------- .../BaseIncrementalAnalyzeManager.java | 17 +- .../androidide/language/java/JavaAnalyzer.kt | 10 +- 4 files changed, 182 insertions(+), 207 deletions(-) diff --git a/app/src/main/java/com/itsaky/androidide/language/cpp/CppAnalyzer.java b/app/src/main/java/com/itsaky/androidide/language/cpp/CppAnalyzer.java index 368fb2dc7..6b93e628d 100644 --- a/app/src/main/java/com/itsaky/androidide/language/cpp/CppAnalyzer.java +++ b/app/src/main/java/com/itsaky/androidide/language/cpp/CppAnalyzer.java @@ -16,67 +16,54 @@ import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TYPE_NA import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forComment; import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forKeyword; import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forString; -import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.get; -import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.withoutCompletion; +import static io.github.rosemoe.sora.lang.styling.TextStyle.makeStyle; +import com.itsaky.androidide.language.incremental.BaseIncrementalAnalyzeManager; +import com.itsaky.androidide.language.incremental.IncrementalToken; +import com.itsaky.androidide.language.incremental.LineState; import com.itsaky.androidide.lexers.cpp.CPP14Lexer; -import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE; -import com.itsaky.androidide.utils.CharSequenceReader; import com.itsaky.androidide.utils.ILogger; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CodePointCharStream; -import org.antlr.v4.runtime.Token; - -import java.io.IOException; import java.lang.Override; -import java.lang.Void; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; -import io.github.rosemoe.sora.lang.analysis.SimpleAnalyzeManager; -import io.github.rosemoe.sora.lang.styling.MappedSpans; -import io.github.rosemoe.sora.lang.styling.Styles; +import io.github.rosemoe.sora.lang.analysis.AsyncIncrementalAnalyzeManager; +import io.github.rosemoe.sora.lang.styling.CodeBlock; +import io.github.rosemoe.sora.lang.styling.Span; +import io.github.rosemoe.sora.text.Content; -public class CppAnalyzer extends SimpleAnalyzeManager { +public class CppAnalyzer extends BaseIncrementalAnalyzeManager { private static final ILogger LOG = ILogger.newInstance("CppAnalyzer"); + public CppAnalyzer() { + super(CPP14Lexer.class); + } + @Override - protected Styles analyze(StringBuilder text, Delegate delegate) { - final var styles = new Styles(); - final var colors = new MappedSpans.Builder(); - final CodePointCharStream stream; - try { - stream = CharStreams.fromReader(new CharSequenceReader(text)); - } catch (IOException e) { - LOG.error("Unable to read text to analyze", e); - return styles; - } + public List computeBlocks( + final Content text, + final AsyncIncrementalAnalyzeManager.CodeBlockAnalyzeDelegate + delegate) { + return Collections.emptyList(); + } - final var lexer = new CPP14Lexer(stream); - // final var parser = new CPP14Parser(new CommonTokenStream(lexer)); - Token token; - int line, column, lastLine = 1, type; - boolean isFirst = true; - - while (!delegate.isCancelled()) { - token = lexer.nextToken(); - if (token == null) { - break; - } - - line = token.getLine() - 1; - column = token.getCharPositionInLine(); - type = token.getType(); - - if (type == CPP14Lexer.EOF) { - lastLine = line; - break; - } + @Override + protected List generateSpans(final LineTokenizeResult tokens) { + final var spans = new ArrayList(); + var first = true; + for (int i = 0; i < tokens.tokens.size(); i++) { + final var token = tokens.tokens.get(i); + final var type = token.getType(); + final var offset = token.getStartIndex(); switch (type) { case Whitespace: - if (isFirst) { - colors.addNormalIfNull(); + if (first) { + spans.add(Span.obtain(offset, makeStyle(TEXT_NORMAL))); + first = false; } break; case Alignas: @@ -141,7 +128,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager { case Volatile: case While: case Void: - colors.addIfNeeded(line, column, forKeyword()); + spans.add(Span.obtain(offset, forKeyword())); break; case Bool: case Char: @@ -153,7 +140,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager { case Long: case Short: case Wchar: - colors.addIfNeeded(line, column, get(TYPE_NAME)); + spans.add(Span.obtain(offset, makeStyle(TYPE_NAME))); break; case LeftParen: case LeftBracket: @@ -202,7 +189,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager { case Doublecolon: case DotStar: case Ellipsis: - colors.addIfNeeded(line, column, get(OPERATOR)); + spans.add(Span.obtain(offset, makeStyle(OPERATOR))); break; case IntegerLiteral: case DecimalLiteral: @@ -212,51 +199,40 @@ public class CppAnalyzer extends SimpleAnalyzeManager { case Integersuffix: case CharacterLiteral: case FloatingLiteral: - case StringLiteral: case BooleanLiteral: case Nullptr: case True_: case False_: - colors.addIfNeeded(line, column, get(LITERAL)); - break; - case BlockComment: - colors.addIfNeeded(line, column, forComment()); - break; - case LineComment: - var commentType = SchemeAndroidIDE.COMMENT; - - // highlight special line comments - var commentText = token.getText(); - if (commentText.length() > 2) { - commentText = commentText.substring(2); - commentText = commentText.trim(); - - if ("todo".equalsIgnoreCase(commentText.substring(0, 4))) { - commentType = SchemeAndroidIDE.TODO_COMMENT; - } else if ("fixme".equalsIgnoreCase(commentText.substring(0, 5))) { - commentType = SchemeAndroidIDE.FIXME_COMMENT; - } - } - - colors.addIfNeeded(line, column, withoutCompletion(commentType)); + spans.add(Span.obtain(offset, makeStyle(LITERAL))); break; + case StringLiteral: case MultiLineMacro: case Directive: - colors.addIfNeeded(line, column, forString()); + spans.add(Span.obtain(offset, forString())); + break; + case BlockComment: + spans.add(Span.obtain(offset, forComment())); + break; + case LineComment: + handleLineCommentSpan(token, spans, offset); break; - default: - colors.addIfNeeded(line, column, get(TEXT_NORMAL)); + spans.add(Span.obtain(offset, makeStyle(TEXT_NORMAL))); break; } - - isFirst = false; } + return null; + } - colors.determine(lastLine); + @Override + protected int[][] getMultilineTokenStartEndTypes() { + final var start = new int[] {Div, Star}; + final var end = new int[] {Star, Div}; + return new int[][] {start, end}; + } - styles.spans = colors.build(); - - return styles; + @Override + protected void handleIncompleteToken(final IncrementalToken token) { + token.type = BlockComment; } } diff --git a/app/src/main/java/com/itsaky/androidide/language/cpp/CppLanguage.java b/app/src/main/java/com/itsaky/androidide/language/cpp/CppLanguage.java index ebde8b69f..a51801cb1 100644 --- a/app/src/main/java/com/itsaky/androidide/language/cpp/CppLanguage.java +++ b/app/src/main/java/com/itsaky/androidide/language/cpp/CppLanguage.java @@ -22,16 +22,13 @@ import android.os.Bundle; import androidx.annotation.NonNull; import com.itsaky.androidide.language.IDELanguage; - -import com.itsaky.androidide.language.java.JavaLanguage; -import com.itsaky.androidide.utils.CharSequenceReader; +import com.itsaky.androidide.lexers.cpp.CPP14Lexer; import com.itsaky.androidide.utils.ILogger; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.Token; import java.io.StringReader; -import com.itsaky.androidide.lexers.cpp.CPP14Lexer; import io.github.rosemoe.sora.lang.analysis.AnalyzeManager; import io.github.rosemoe.sora.lang.completion.CompletionCancelledException; @@ -45,123 +42,118 @@ import io.github.rosemoe.sora.widget.SymbolPairMatch; public class CppLanguage extends IDELanguage { - private static final ILogger LOG = ILogger.newInstance("CppLanguage"); - private CppAnalyzer analyzer; - private final NewlineHandler[] newlineHandlers = new NewlineHandler[] {new BraceHandler()}; + private static final ILogger LOG = ILogger.newInstance("CppLanguage"); + private CppAnalyzer analyzer; + private final NewlineHandler[] newlineHandlers = new NewlineHandler[] {new BraceHandler()}; - public CppLanguage() { - analyzer = new CppAnalyzer(); - } + public CppLanguage() { + analyzer = new CppAnalyzer(); + } - @Override - public int getIndentAdvance(@NonNull String line) { - try { - CPP14Lexer lexer = new CPP14Lexer(CharStreams.fromReader(new StringReader(line))); - Token token; - int advance = 0; - while (((token = lexer.nextToken()) != null && token.getType() != token.EOF)) { - switch (token.getType()) { - case CPP14Lexer.LeftBrace: - advance++; - break; - case CPP14Lexer.RightBrace: - advance--; - break; - } - } - advance = Math.max(0, advance); - return advance * getTabSize(); - } catch (Throwable e) { - LOG.error("Error calculating indent advance", e); + @Override + public int getIndentAdvance(@NonNull String line) { + try { + CPP14Lexer lexer = new CPP14Lexer(CharStreams.fromReader(new StringReader(line))); + Token token; + int advance = 0; + while (((token = lexer.nextToken()) != null && token.getType() != token.EOF)) { + switch (token.getType()) { + case CPP14Lexer.LeftBrace: + advance++; + break; + case CPP14Lexer.RightBrace: + advance--; + break; } - return 0; + } + advance = Math.max(0, advance); + return advance * getTabSize(); + } catch (Throwable e) { + LOG.error("Error calculating indent advance", e); + } + return 0; + } + + @Override + public SymbolPairMatch getSymbolPairs() { + return new CppSymbolPairs(); + } + + @NonNull + @Override + public AnalyzeManager getAnalyzeManager() { + return analyzer; + } + + @Override + public int getInterruptionLevel() { + return INTERRUPTION_LEVEL_STRONG; + } + + @Override + public void requireAutoComplete( + @NonNull ContentReference content, + @NonNull CharPosition position, + @NonNull CompletionPublisher publisher, + @NonNull Bundle extraArguments) + throws CompletionCancelledException { + + // completer.complete(content, position, publisher, extraArguments); + } + + @Override + public int getIndentAdvance(@NonNull ContentReference content, int line, int column) { + return getIndentAdvance(content.getLine(line).substring(0, column)); + } + + @Override + public CharSequence format(CharSequence content) { + return content; + } + + @Override + public NewlineHandler[] getNewlineHandlers() { + return newlineHandlers; + } + + @Override + public void destroy() { + analyzer = null; + } + + private static class CppSymbolPairs extends SymbolPairMatch { + public CppSymbolPairs() { + super.putPair('{', new Replacement("{}", 1)); + super.putPair('(', new Replacement("()", 1)); + super.putPair('[', new Replacement("[]", 1)); + super.putPair('"', new Replacement("\"\"", 1)); + super.putPair('\'', new Replacement("''", 1)); + super.putPair('<', new Replacement("<>", 1)); + } + } + + class BraceHandler implements NewlineHandler { + + @Override + public boolean matchesRequirement(String beforeText, String afterText) { + beforeText = beforeText.trim(); + afterText = afterText.trim(); + return beforeText.endsWith("{") && afterText.startsWith("}"); } @Override - public SymbolPairMatch getSymbolPairs() { - return new CppSymbolPairs(); - } - - @NonNull - @Override - public AnalyzeManager getAnalyzeManager() { - return analyzer; - } - - @Override - public int getInterruptionLevel() { - return INTERRUPTION_LEVEL_STRONG; - } - - @Override - public void requireAutoComplete( - @NonNull ContentReference content, - @NonNull CharPosition position, - @NonNull CompletionPublisher publisher, - @NonNull Bundle extraArguments) - throws CompletionCancelledException { - - // completer.complete(content, position, publisher, extraArguments); - } - - @Override - public int getIndentAdvance(@NonNull ContentReference content, int line, int column) { - return getIndentAdvance(content.getLine(line).substring(0, column)); - } - - @Override - public CharSequence format(CharSequence content) { - return content; - } - - @Override - public NewlineHandler[] getNewlineHandlers() { - return newlineHandlers; - } - - @Override - public void destroy() { - analyzer = null; - } - - private static class CppSymbolPairs extends SymbolPairMatch { - public CppSymbolPairs() { - super.putPair('{', new Replacement("{}", 1)); - super.putPair('(', new Replacement("()", 1)); - super.putPair('[', new Replacement("[]", 1)); - super.putPair('"', new Replacement("\"\"", 1)); - super.putPair('\'', new Replacement("''", 1)); - super.putPair('<', new Replacement("<>", 1)); - } - } - - class BraceHandler implements NewlineHandler { - - @Override - public boolean matchesRequirement(String beforeText, String afterText) { - beforeText = beforeText.trim(); - afterText = afterText.trim(); - return beforeText.endsWith("{") && afterText.startsWith("}"); - } - - @Override - public NewlineHandleResult handleNewline(String beforeText, String afterText, int tabSize) { - int count = TextUtils.countLeadingSpaceCount(beforeText, tabSize); - int advanceBefore = getIndentAdvance(beforeText); - int advanceAfter = getIndentAdvance(afterText); - String text; - StringBuilder sb = - new StringBuilder("\n") - .append( - TextUtils.createIndent( - count + advanceBefore, tabSize, useTab())) - .append('\n') - .append( - text = - TextUtils.createIndent( - count + advanceAfter, tabSize, useTab())); - int shiftLeft = text.length() + 1; - return new NewlineHandleResult(sb, shiftLeft); - } + public NewlineHandleResult handleNewline(String beforeText, String afterText, int tabSize) { + int count = TextUtils.countLeadingSpaceCount(beforeText, tabSize); + int advanceBefore = getIndentAdvance(beforeText); + int advanceAfter = getIndentAdvance(afterText); + String text; + StringBuilder sb = + new StringBuilder("\n") + .append(TextUtils.createIndent(count + advanceBefore, tabSize, useTab())) + .append('\n') + .append(text = TextUtils.createIndent(count + advanceAfter, tabSize, useTab())); + int shiftLeft = text.length() + 1; + return new NewlineHandleResult(sb, shiftLeft); } + } } diff --git a/app/src/main/java/com/itsaky/androidide/language/incremental/BaseIncrementalAnalyzeManager.java b/app/src/main/java/com/itsaky/androidide/language/incremental/BaseIncrementalAnalyzeManager.java index 2a5676feb..3e5b43136 100644 --- a/app/src/main/java/com/itsaky/androidide/language/incremental/BaseIncrementalAnalyzeManager.java +++ b/app/src/main/java/com/itsaky/androidide/language/incremental/BaseIncrementalAnalyzeManager.java @@ -63,9 +63,9 @@ public abstract class BaseIncrementalAnalyzeManager private final int[] multilineEndTypes; private static final ILogger LOG = ILogger.newInstance("BaseIncrementalAnalyzeManager"); - public BaseIncrementalAnalyzeManager(final Lexer lexer) { + public BaseIncrementalAnalyzeManager(final Class lexer) { Objects.requireNonNull(lexer, "Cannot create analyzer manager for null lexer"); - this.lexer = lexer; + this.lexer = createLexerInstance(lexer); var multilineTokenTypes = getMultilineTokenStartEndTypes(); verifyMultilineTypes(multilineTokenTypes); @@ -74,6 +74,19 @@ public abstract class BaseIncrementalAnalyzeManager this.multilineEndTypes = multilineTokenTypes[1]; } + @NonNull + private Lexer createLexerInstance(final Class lexer) { + try { + final var constructor = lexer.getConstructor(CharStream.class); + if (!constructor.isAccessible()) { + constructor.setAccessible(true); + } + return constructor.newInstance(createStream("")); + } catch (Throwable err) { + throw new RuntimeException("Unable to create Lexer instance", err); + } + } + private void verifyMultilineTypes(@NonNull final int[][] types) { Preconditions.checkState( types.length == 2, "There must be exact two inner int[] in multiline token types"); diff --git a/app/src/main/java/com/itsaky/androidide/language/java/JavaAnalyzer.kt b/app/src/main/java/com/itsaky/androidide/language/java/JavaAnalyzer.kt index f2f3eeb77..a6f07df49 100644 --- a/app/src/main/java/com/itsaky/androidide/language/java/JavaAnalyzer.kt +++ b/app/src/main/java/com/itsaky/androidide/language/java/JavaAnalyzer.kt @@ -21,27 +21,22 @@ import com.itsaky.androidide.language.incremental.IncrementalToken import com.itsaky.androidide.language.incremental.LineState import com.itsaky.androidide.lexers.java.JavaLexer import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.ANNOTATION -import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.COMMENT -import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.FIXME_COMMENT import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.LITERAL import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.OPERATOR import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TEXT_NORMAL -import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TODO_COMMENT import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TYPE_NAME import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forComment import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forKeyword import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forString -import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.withoutCompletion import com.itsaky.androidide.utils.ILogger import io.github.rosemoe.sora.lang.analysis.IncrementalAnalyzeManager.LineTokenizeResult import io.github.rosemoe.sora.lang.styling.CodeBlock import io.github.rosemoe.sora.lang.styling.Span import io.github.rosemoe.sora.lang.styling.TextStyle.makeStyle import io.github.rosemoe.sora.text.Content -import org.antlr.v4.runtime.CharStreams /** @author Akash Yadav */ -class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromString(""))) { +class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer::class.java) { private val log = ILogger.newInstance(javaClass.simpleName) @@ -58,7 +53,6 @@ class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromStr for (token in tokens.tokens) { val type = token.type val offset = token.startIndex - val tokenLength = token.text.length when (type) { JavaLexer.WS -> { if (first) { @@ -190,7 +184,7 @@ class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromStr return spans } - + override fun getMultilineTokenStartEndTypes(): Array { val start = intArrayOf(JavaLexer.DIV, JavaLexer.MUL) val end = intArrayOf(JavaLexer.MUL, JavaLexer.DIV)