Define the representation for tokens

core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/Token.java

@@ -0,0 +1,37 @@
+package it.cavallium.warppi.math.rules.dsl.frontend;
+
+import java.util.Objects;
+
+public class Token {
+	/** The type of the token. */
+	public final TokenType type;
+	/** The source string which corresponds to the token. */
+	public final String lexeme;
+	/** The index at which the token starts in the source string. */
+	public final int position;
+
+	public Token(final TokenType type, final String lexeme, final int position) {
+		this.type = type;
+		this.lexeme = lexeme;
+		this.position = position;
+	}
+
+	@Override
+	public String toString() {
+		return String.format("%s(\"%s\")@%d", type, lexeme, position);
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (!(o instanceof Token)) {
+			return false;
+		}
+		Token other = (Token) o;
+		return type == other.type && lexeme.equals(other.lexeme) && position == other.position;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(type, lexeme, position);
+	}
+}

core/src/main/java/it/cavallium/warppi/math/rules/dsl/frontend/TokenType.java

@@ -0,0 +1,15 @@
+package it.cavallium.warppi.math.rules.dsl.frontend;
+
+public enum TokenType {
+	EOF,
+	// Separators and grouping
+	COLON, ARROW, COMMA, LEFT_PAREN, RIGHT_PAREN, LEFT_BRACKET, RIGHT_BRACKET,
+	// Operators
+	EQUALS, PLUS, MINUS, PLUS_MINUS, TIMES, DIVIDE, POWER,
+	// Rule types
+	REDUCTION, EXPANSION, CALCULATION, EXISTENCE,
+	// Functions
+	ARCCOS, ARCSIN, ARCTAN, COS, SIN, TAN, ROOT, SQRT, LOG,
+	// Literals
+	UNDEFINED, PI, E, NUMBER, IDENTIFIER,
+}