Implemented incremental analyzer for CPP

This commit is contained in:
Akash Yadav
2022-06-21 19:17:49 +05:30
parent 9892516dc3
commit cafec5ea87
4 changed files with 182 additions and 207 deletions
@@ -16,67 +16,54 @@ import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TYPE_NA
import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forComment;
import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forKeyword;
import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forString;
import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.get;
import static com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.withoutCompletion;
import static io.github.rosemoe.sora.lang.styling.TextStyle.makeStyle;
import com.itsaky.androidide.language.incremental.BaseIncrementalAnalyzeManager;
import com.itsaky.androidide.language.incremental.IncrementalToken;
import com.itsaky.androidide.language.incremental.LineState;
import com.itsaky.androidide.lexers.cpp.CPP14Lexer;
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE;
import com.itsaky.androidide.utils.CharSequenceReader;
import com.itsaky.androidide.utils.ILogger;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.Token;
import java.io.IOException;
import java.lang.Override;
import java.lang.Void;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import io.github.rosemoe.sora.lang.analysis.SimpleAnalyzeManager;
import io.github.rosemoe.sora.lang.styling.MappedSpans;
import io.github.rosemoe.sora.lang.styling.Styles;
import io.github.rosemoe.sora.lang.analysis.AsyncIncrementalAnalyzeManager;
import io.github.rosemoe.sora.lang.styling.CodeBlock;
import io.github.rosemoe.sora.lang.styling.Span;
import io.github.rosemoe.sora.text.Content;
public class CppAnalyzer extends SimpleAnalyzeManager<Void> {
public class CppAnalyzer extends BaseIncrementalAnalyzeManager {
private static final ILogger LOG = ILogger.newInstance("CppAnalyzer");
public CppAnalyzer() {
super(CPP14Lexer.class);
}
@Override
protected Styles analyze(StringBuilder text, Delegate<Void> delegate) {
final var styles = new Styles();
final var colors = new MappedSpans.Builder();
final CodePointCharStream stream;
try {
stream = CharStreams.fromReader(new CharSequenceReader(text));
} catch (IOException e) {
LOG.error("Unable to read text to analyze", e);
return styles;
}
public List<CodeBlock> computeBlocks(
final Content text,
final AsyncIncrementalAnalyzeManager<LineState, IncrementalToken>.CodeBlockAnalyzeDelegate
delegate) {
return Collections.emptyList();
}
final var lexer = new CPP14Lexer(stream);
// final var parser = new CPP14Parser(new CommonTokenStream(lexer));
Token token;
int line, column, lastLine = 1, type;
boolean isFirst = true;
while (!delegate.isCancelled()) {
token = lexer.nextToken();
if (token == null) {
break;
}
line = token.getLine() - 1;
column = token.getCharPositionInLine();
type = token.getType();
if (type == CPP14Lexer.EOF) {
lastLine = line;
break;
}
@Override
protected List<Span> generateSpans(final LineTokenizeResult<LineState, IncrementalToken> tokens) {
final var spans = new ArrayList<Span>();
var first = true;
for (int i = 0; i < tokens.tokens.size(); i++) {
final var token = tokens.tokens.get(i);
final var type = token.getType();
final var offset = token.getStartIndex();
switch (type) {
case Whitespace:
if (isFirst) {
colors.addNormalIfNull();
if (first) {
spans.add(Span.obtain(offset, makeStyle(TEXT_NORMAL)));
first = false;
}
break;
case Alignas:
@@ -141,7 +128,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager<Void> {
case Volatile:
case While:
case Void:
colors.addIfNeeded(line, column, forKeyword());
spans.add(Span.obtain(offset, forKeyword()));
break;
case Bool:
case Char:
@@ -153,7 +140,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager<Void> {
case Long:
case Short:
case Wchar:
colors.addIfNeeded(line, column, get(TYPE_NAME));
spans.add(Span.obtain(offset, makeStyle(TYPE_NAME)));
break;
case LeftParen:
case LeftBracket:
@@ -202,7 +189,7 @@ public class CppAnalyzer extends SimpleAnalyzeManager<Void> {
case Doublecolon:
case DotStar:
case Ellipsis:
colors.addIfNeeded(line, column, get(OPERATOR));
spans.add(Span.obtain(offset, makeStyle(OPERATOR)));
break;
case IntegerLiteral:
case DecimalLiteral:
@@ -212,51 +199,40 @@ public class CppAnalyzer extends SimpleAnalyzeManager<Void> {
case Integersuffix:
case CharacterLiteral:
case FloatingLiteral:
case StringLiteral:
case BooleanLiteral:
case Nullptr:
case True_:
case False_:
colors.addIfNeeded(line, column, get(LITERAL));
break;
case BlockComment:
colors.addIfNeeded(line, column, forComment());
break;
case LineComment:
var commentType = SchemeAndroidIDE.COMMENT;
// highlight special line comments
var commentText = token.getText();
if (commentText.length() > 2) {
commentText = commentText.substring(2);
commentText = commentText.trim();
if ("todo".equalsIgnoreCase(commentText.substring(0, 4))) {
commentType = SchemeAndroidIDE.TODO_COMMENT;
} else if ("fixme".equalsIgnoreCase(commentText.substring(0, 5))) {
commentType = SchemeAndroidIDE.FIXME_COMMENT;
}
}
colors.addIfNeeded(line, column, withoutCompletion(commentType));
spans.add(Span.obtain(offset, makeStyle(LITERAL)));
break;
case StringLiteral:
case MultiLineMacro:
case Directive:
colors.addIfNeeded(line, column, forString());
spans.add(Span.obtain(offset, forString()));
break;
case BlockComment:
spans.add(Span.obtain(offset, forComment()));
break;
case LineComment:
handleLineCommentSpan(token, spans, offset);
break;
default:
colors.addIfNeeded(line, column, get(TEXT_NORMAL));
spans.add(Span.obtain(offset, makeStyle(TEXT_NORMAL)));
break;
}
isFirst = false;
}
return null;
}
colors.determine(lastLine);
@Override
protected int[][] getMultilineTokenStartEndTypes() {
final var start = new int[] {Div, Star};
final var end = new int[] {Star, Div};
return new int[][] {start, end};
}
styles.spans = colors.build();
return styles;
@Override
protected void handleIncompleteToken(final IncrementalToken token) {
token.type = BlockComment;
}
}
@@ -22,16 +22,13 @@ import android.os.Bundle;
import androidx.annotation.NonNull;
import com.itsaky.androidide.language.IDELanguage;
import com.itsaky.androidide.language.java.JavaLanguage;
import com.itsaky.androidide.utils.CharSequenceReader;
import com.itsaky.androidide.lexers.cpp.CPP14Lexer;
import com.itsaky.androidide.utils.ILogger;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.Token;
import java.io.StringReader;
import com.itsaky.androidide.lexers.cpp.CPP14Lexer;
import io.github.rosemoe.sora.lang.analysis.AnalyzeManager;
import io.github.rosemoe.sora.lang.completion.CompletionCancelledException;
@@ -45,123 +42,118 @@ import io.github.rosemoe.sora.widget.SymbolPairMatch;
public class CppLanguage extends IDELanguage {
private static final ILogger LOG = ILogger.newInstance("CppLanguage");
private CppAnalyzer analyzer;
private final NewlineHandler[] newlineHandlers = new NewlineHandler[] {new BraceHandler()};
private static final ILogger LOG = ILogger.newInstance("CppLanguage");
private CppAnalyzer analyzer;
private final NewlineHandler[] newlineHandlers = new NewlineHandler[] {new BraceHandler()};
public CppLanguage() {
analyzer = new CppAnalyzer();
}
public CppLanguage() {
analyzer = new CppAnalyzer();
}
@Override
public int getIndentAdvance(@NonNull String line) {
try {
CPP14Lexer lexer = new CPP14Lexer(CharStreams.fromReader(new StringReader(line)));
Token token;
int advance = 0;
while (((token = lexer.nextToken()) != null && token.getType() != token.EOF)) {
switch (token.getType()) {
case CPP14Lexer.LeftBrace:
advance++;
break;
case CPP14Lexer.RightBrace:
advance--;
break;
}
}
advance = Math.max(0, advance);
return advance * getTabSize();
} catch (Throwable e) {
LOG.error("Error calculating indent advance", e);
@Override
public int getIndentAdvance(@NonNull String line) {
try {
CPP14Lexer lexer = new CPP14Lexer(CharStreams.fromReader(new StringReader(line)));
Token token;
int advance = 0;
while (((token = lexer.nextToken()) != null && token.getType() != token.EOF)) {
switch (token.getType()) {
case CPP14Lexer.LeftBrace:
advance++;
break;
case CPP14Lexer.RightBrace:
advance--;
break;
}
return 0;
}
advance = Math.max(0, advance);
return advance * getTabSize();
} catch (Throwable e) {
LOG.error("Error calculating indent advance", e);
}
return 0;
}
@Override
public SymbolPairMatch getSymbolPairs() {
return new CppSymbolPairs();
}
@NonNull
@Override
public AnalyzeManager getAnalyzeManager() {
return analyzer;
}
@Override
public int getInterruptionLevel() {
return INTERRUPTION_LEVEL_STRONG;
}
@Override
public void requireAutoComplete(
@NonNull ContentReference content,
@NonNull CharPosition position,
@NonNull CompletionPublisher publisher,
@NonNull Bundle extraArguments)
throws CompletionCancelledException {
// completer.complete(content, position, publisher, extraArguments);
}
@Override
public int getIndentAdvance(@NonNull ContentReference content, int line, int column) {
return getIndentAdvance(content.getLine(line).substring(0, column));
}
@Override
public CharSequence format(CharSequence content) {
return content;
}
@Override
public NewlineHandler[] getNewlineHandlers() {
return newlineHandlers;
}
@Override
public void destroy() {
analyzer = null;
}
private static class CppSymbolPairs extends SymbolPairMatch {
public CppSymbolPairs() {
super.putPair('{', new Replacement("{}", 1));
super.putPair('(', new Replacement("()", 1));
super.putPair('[', new Replacement("[]", 1));
super.putPair('"', new Replacement("\"\"", 1));
super.putPair('\'', new Replacement("''", 1));
super.putPair('<', new Replacement("<>", 1));
}
}
class BraceHandler implements NewlineHandler {
@Override
public boolean matchesRequirement(String beforeText, String afterText) {
beforeText = beforeText.trim();
afterText = afterText.trim();
return beforeText.endsWith("{") && afterText.startsWith("}");
}
@Override
public SymbolPairMatch getSymbolPairs() {
return new CppSymbolPairs();
}
@NonNull
@Override
public AnalyzeManager getAnalyzeManager() {
return analyzer;
}
@Override
public int getInterruptionLevel() {
return INTERRUPTION_LEVEL_STRONG;
}
@Override
public void requireAutoComplete(
@NonNull ContentReference content,
@NonNull CharPosition position,
@NonNull CompletionPublisher publisher,
@NonNull Bundle extraArguments)
throws CompletionCancelledException {
// completer.complete(content, position, publisher, extraArguments);
}
@Override
public int getIndentAdvance(@NonNull ContentReference content, int line, int column) {
return getIndentAdvance(content.getLine(line).substring(0, column));
}
@Override
public CharSequence format(CharSequence content) {
return content;
}
@Override
public NewlineHandler[] getNewlineHandlers() {
return newlineHandlers;
}
@Override
public void destroy() {
analyzer = null;
}
private static class CppSymbolPairs extends SymbolPairMatch {
public CppSymbolPairs() {
super.putPair('{', new Replacement("{}", 1));
super.putPair('(', new Replacement("()", 1));
super.putPair('[', new Replacement("[]", 1));
super.putPair('"', new Replacement("\"\"", 1));
super.putPair('\'', new Replacement("''", 1));
super.putPair('<', new Replacement("<>", 1));
}
}
class BraceHandler implements NewlineHandler {
@Override
public boolean matchesRequirement(String beforeText, String afterText) {
beforeText = beforeText.trim();
afterText = afterText.trim();
return beforeText.endsWith("{") && afterText.startsWith("}");
}
@Override
public NewlineHandleResult handleNewline(String beforeText, String afterText, int tabSize) {
int count = TextUtils.countLeadingSpaceCount(beforeText, tabSize);
int advanceBefore = getIndentAdvance(beforeText);
int advanceAfter = getIndentAdvance(afterText);
String text;
StringBuilder sb =
new StringBuilder("\n")
.append(
TextUtils.createIndent(
count + advanceBefore, tabSize, useTab()))
.append('\n')
.append(
text =
TextUtils.createIndent(
count + advanceAfter, tabSize, useTab()));
int shiftLeft = text.length() + 1;
return new NewlineHandleResult(sb, shiftLeft);
}
public NewlineHandleResult handleNewline(String beforeText, String afterText, int tabSize) {
int count = TextUtils.countLeadingSpaceCount(beforeText, tabSize);
int advanceBefore = getIndentAdvance(beforeText);
int advanceAfter = getIndentAdvance(afterText);
String text;
StringBuilder sb =
new StringBuilder("\n")
.append(TextUtils.createIndent(count + advanceBefore, tabSize, useTab()))
.append('\n')
.append(text = TextUtils.createIndent(count + advanceAfter, tabSize, useTab()));
int shiftLeft = text.length() + 1;
return new NewlineHandleResult(sb, shiftLeft);
}
}
}
@@ -63,9 +63,9 @@ public abstract class BaseIncrementalAnalyzeManager
private final int[] multilineEndTypes;
private static final ILogger LOG = ILogger.newInstance("BaseIncrementalAnalyzeManager");
public BaseIncrementalAnalyzeManager(final Lexer lexer) {
public BaseIncrementalAnalyzeManager(final Class<? extends Lexer> lexer) {
Objects.requireNonNull(lexer, "Cannot create analyzer manager for null lexer");
this.lexer = lexer;
this.lexer = createLexerInstance(lexer);
var multilineTokenTypes = getMultilineTokenStartEndTypes();
verifyMultilineTypes(multilineTokenTypes);
@@ -74,6 +74,19 @@ public abstract class BaseIncrementalAnalyzeManager
this.multilineEndTypes = multilineTokenTypes[1];
}
@NonNull
private Lexer createLexerInstance(final Class<? extends Lexer> lexer) {
try {
final var constructor = lexer.getConstructor(CharStream.class);
if (!constructor.isAccessible()) {
constructor.setAccessible(true);
}
return constructor.newInstance(createStream(""));
} catch (Throwable err) {
throw new RuntimeException("Unable to create Lexer instance", err);
}
}
private void verifyMultilineTypes(@NonNull final int[][] types) {
Preconditions.checkState(
types.length == 2, "There must be exact two inner int[] in multiline token types");
@@ -21,27 +21,22 @@ import com.itsaky.androidide.language.incremental.IncrementalToken
import com.itsaky.androidide.language.incremental.LineState
import com.itsaky.androidide.lexers.java.JavaLexer
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.ANNOTATION
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.COMMENT
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.FIXME_COMMENT
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.LITERAL
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.OPERATOR
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TEXT_NORMAL
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TODO_COMMENT
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.TYPE_NAME
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forComment
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forKeyword
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.forString
import com.itsaky.androidide.syntax.colorschemes.SchemeAndroidIDE.withoutCompletion
import com.itsaky.androidide.utils.ILogger
import io.github.rosemoe.sora.lang.analysis.IncrementalAnalyzeManager.LineTokenizeResult
import io.github.rosemoe.sora.lang.styling.CodeBlock
import io.github.rosemoe.sora.lang.styling.Span
import io.github.rosemoe.sora.lang.styling.TextStyle.makeStyle
import io.github.rosemoe.sora.text.Content
import org.antlr.v4.runtime.CharStreams
/** @author Akash Yadav */
class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromString(""))) {
class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer::class.java) {
private val log = ILogger.newInstance(javaClass.simpleName)
@@ -58,7 +53,6 @@ class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromStr
for (token in tokens.tokens) {
val type = token.type
val offset = token.startIndex
val tokenLength = token.text.length
when (type) {
JavaLexer.WS -> {
if (first) {
@@ -190,7 +184,7 @@ class JavaAnalyzer : BaseIncrementalAnalyzeManager(JavaLexer(CharStreams.fromStr
return spans
}
override fun getMultilineTokenStartEndTypes(): Array<IntArray> {
val start = intArrayOf(JavaLexer.DIV, JavaLexer.MUL)
val end = intArrayOf(JavaLexer.MUL, JavaLexer.DIV)