diff --git a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/RulesDsl.java b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/RulesDsl.java index 06f45978..6fbf5fd8 100644 --- a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/RulesDsl.java +++ b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/RulesDsl.java @@ -57,6 +57,4 @@ public class RulesDsl { undefined.removeAll(defined); return undefined; } - - } diff --git a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Lexer.java b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Lexer.java index b71721df..7c4ba517 100644 --- a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Lexer.java +++ b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Lexer.java @@ -9,7 +9,7 @@ import java.util.function.Predicate; import static it.cavallium.warppi.math.rules.dsl.frontend.TokenType.*; /** - * Converts the source string to a list of tokens. + * Converts the source String to a list of {@link Token}s. */ public class Lexer { private static final Map KEYWORDS; @@ -34,11 +34,28 @@ public class Lexer { private int curPosition = 0; private UnexpectedCharacters unexpectedCharacters = null; + /** + * Constructs a Lexer that will split the given source code into {@link Token}s. + * + * @param source a String containing the DSL source code to process. + * @param errorReporter a Consumer used to report each DslError that the + * Lexer finds within the source string. + */ public Lexer(final String source, final Consumer errorReporter) { this.source = source; this.errorReporter = errorReporter; } + /** + * Runs the Lexer. + *

+ * This method can only be called once per instance. + * + * @return the list of Tokens extracted from the source string. + * If any errors are reported, this list should not be considered to represent a valid set of DSL rules, + * but it can still be parsed to potentially find additional errors (which may allow the user to fix more + * errors before having to rerun the Lexer). + */ public List lex() { while (!atEnd()) { startOfLexeme = curPosition; diff --git a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Parser.java b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Parser.java index 4254f7fc..8c606665 100644 --- a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Parser.java +++ b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Parser.java @@ -17,7 +17,7 @@ import java.util.function.Function; import static it.cavallium.warppi.math.rules.dsl.frontend.TokenType.*; /** - * Converts a list of tokens to a list of PatternRules. + * Converts a list of {@link Token}s to a list of {@link PatternRule}s. */ public class Parser { private static final Map RULE_TYPES = MapFactory.fromEntries( @@ -29,7 +29,7 @@ public class Parser { private final List tokens; private final Consumer errorReporter; - private int current = 0; + private int currentIndex = 0; // For error reporting private Map> ruleSubFunctionIdentifiers; @@ -37,11 +37,29 @@ public class Parser { private final IdentityHashMap>> subFunctionIdentifiers = new IdentityHashMap<>(); + /** + * Constructs a Parser that will produce a list of {@link PatternRule}s from the the given list of {@link Token}s. + * + * @param tokens the list of Tokens to process. + * @param errorReporter a Consumer used to report each DslError that the + * Parser finds within the source string. + */ public Parser(final List tokens, final Consumer errorReporter) { this.tokens = tokens; this.errorReporter = errorReporter; } + /** + * Runs the Parser. + *

+ * This method can only be called once per instance. + * + * @return the list of all valid PatternRules constructed from the given tokens. + * If any errors are reported, this list should not be considered to represent a valid set of DSL rules, + * but each rule can still be analyzed to look for undefined sub-functions in replacement patterns and + * report them (which may allow the user to fix more errors before having to rerun the Lexer + * and Parser). + */ public List parse() { return rules(); } @@ -264,42 +282,42 @@ public class Parser { final Token matched = match(expectedType); if (matched == null) { throw new SyntaxException( - new UnexpectedToken(tokens.get(current), expectedType) + new UnexpectedToken(tokens.get(currentIndex), expectedType) ); } return matched; } private Token match(final TokenType expectedType) { - final Token curToken = tokens.get(current); + final Token curToken = tokens.get(currentIndex); if (curToken.type != expectedType) { return null; } - current++; + currentIndex++; return curToken; } private void synchronizeTo(final Set types) { - while (!atEnd() && !types.contains(tokens.get(current).type)) { - current++; + while (!atEnd() && !types.contains(tokens.get(currentIndex).type)) { + currentIndex++; } } private Token pop() throws SyntaxException { - final Token curToken = tokens.get(current); + final Token curToken = tokens.get(currentIndex); if (atEnd()) { throw new SyntaxException(new UnexpectedToken(curToken)); // Avoid popping EOF } - current++; + currentIndex++; return curToken; } private Token peek() { - return tokens.get(current); + return tokens.get(currentIndex); } private boolean atEnd() { - return tokens.get(current).type == EOF; + return tokens.get(currentIndex).type == EOF; } @FunctionalInterface diff --git a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Token.java b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Token.java index 9de4bb81..942a043b 100644 --- a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Token.java +++ b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Token.java @@ -2,14 +2,37 @@ package it.cavallium.warppi.math.rules.dsl.frontend; import java.util.Objects; +/** + * Represents a single token extracted from DSL source code. + *

+ * Tokens are produced by the {@link Lexer} and consumed by the {@link Parser}. + */ public class Token { /** The type of the token. */ public final TokenType type; - /** The source string which corresponds to the token. */ + /** + * The part of the source code which corresponds to the token. + *

+ * Some types of token always have the same lexemes (for example, PLUS is always represented by + * "+"), while others have variable lexemes (like IDENTIFIER, which may correspond to any + * valid identifier). + *

+ * As a special case, tokens of type EOF (which signal the end of the source code) have empty lexemes + * (""). Such tokens only exist to simplify the parser code, by allowing the end of the input to be + * treated like any other token (which is especially useful for error handling, because an unexpected end of input + * just becomes an "unexpected token" error). + */ public final String lexeme; /** The index at which the token starts in the source string. */ public final int position; + /** + * Constructs a Token. + * + * @param type the type of the token. + * @param lexeme the part of the source string which corresponds to the token. + * @param position the index at which the token starts in the source string. + */ public Token(final TokenType type, final String lexeme, final int position) { this.type = type; this.lexeme = lexeme; diff --git a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/TokenType.java b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/TokenType.java index 55aee670..769eb45b 100644 --- a/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/TokenType.java +++ b/core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/TokenType.java @@ -1,5 +1,8 @@ package it.cavallium.warppi.math.rules.dsl.frontend; +/** + * Specifies the type of a Token. + */ public enum TokenType { EOF, // Separators and grouping