Merge pull request #6 from adriantodt/feature/v4

Version 4

build.gradle.kts

@@ -1,11 +1,11 @@
 plugins {
-    kotlin("multiplatform") version "1.5.31"
+    kotlin("multiplatform") version "1.6.0-RC2"
     `maven-publish`
     id("org.jetbrains.dokka") version "1.5.31"
 }
 
 group = "com.github.adriantodt"
-version = "3.2.0"
+version = "4.0.0"
 
 repositories {
     mavenCentral()
@@ -69,7 +69,18 @@ kotlin {
     }
 }
 
+
 tasks {
+    dokkaHtml.configure {
+        dokkaSourceSets {
+            configureEach {
+                if (name == "commonNonJvmMain") {
+                    displayName.set("common (non-jvm)")
+                }
+                includes.from("dokka_modules.md")
+            }
+        }
+    }
     register<Jar>("dokkaJar") {
         from(dokkaHtml)
         dependsOn(dokkaHtml)

dokka_modules.md

@@ -0,0 +1,4 @@
+# Module tartar
+
+A trie-based lexical analysis and pratt-parsing library,
+leveraging Kotlin Multiplatform and DSL patterns.

src/commonMain/kotlin/com/github/adriantodt/tartar/api/dsl/CharPredicate.kt

@@ -4,6 +4,7 @@ package com.github.adriantodt.tartar.api.dsl
  * Represents a predicate (boolean-valued function) of one [Char]-valued argument.
  *
  * @since 3.0
+ * @author AdrianTodt
  */
 public fun interface CharPredicate {
     /**

src/commonMain/kotlin/com/github/adriantodt/tartar/api/dsl/GrammarDSL.kt

@@ -3,6 +3,7 @@ package com.github.adriantodt.tartar.api.dsl
 import com.github.adriantodt.tartar.api.grammar.Grammar
 import com.github.adriantodt.tartar.api.grammar.InfixParselet
 import com.github.adriantodt.tartar.api.grammar.PrefixParselet
+import com.github.adriantodt.tartar.api.parser.Token
 
 /**
  * A builder of [Grammars][com.github.adriantodt.tartar.api.parser.Grammar], as a domain-specific language (DSL).
@@ -11,44 +12,44 @@ import com.github.adriantodt.tartar.api.grammar.PrefixParselet
  * @param E The grammar's expression result.
  * @author AdrianTodt
  */
-public interface GrammarDSL<T, E> {
+public interface GrammarDSL<T, K : Token<T>, E> {
     /**
      * Imports all parselets from other grammars.
 
      * @param override If set to true, imported parselets overrides existing ones. If false, they throw.
      * @param grammars The grammars to import.
      */
-    public fun import(override: Boolean = false, vararg grammars: Grammar<T, E>)
+    public fun import(override: Boolean = false, vararg grammars: Grammar<T, K, E>)
 
     /**
      * Registers a prefix parselets into the grammar.
      * @param type The token type to associate the parselet with.
      * @param parselet The prefix parselet to register.
      * @param override If set to true, imported parselets overrides existing ones. If false, they throw.
      */
-    public fun prefix(type: T, parselet: PrefixParselet<T, E>, override: Boolean = false)
+    public fun prefix(type: T, parselet: PrefixParselet<T, K, E>, override: Boolean = false)
 
     /**
      * Registers a prefix parselets into the grammar.
      * @param type The token type to associate the parselet with.
      * @param override If set to true, imported parselets overrides existing ones. If false, they throw.
      * @param block The code to execute when the type matches.
      */
-    public fun prefix(type: T, override: Boolean = false, block: PrefixFunction<T, E>)
+    public fun prefix(type: T, override: Boolean = false, block: PrefixFunction<T, K, E>)
 
     /**
      * Registers a infix parselets into the grammar.
      * @param type The token type to associate the parselet with.
      * @param parselet The infix parselet to register.
      * @param override If set to true, imported parselets overrides existing ones. If false, they throw.
      */
-    public fun infix(type: T, parselet: InfixParselet<T, E>, override: Boolean = false)
+    public fun infix(type: T, parselet: InfixParselet<T, K, E>, override: Boolean = false)
 
     /**
      * Registers a infix parselets into the grammar.
      * @param type The token type to associate the parselet with.
      * @param override If set to true, imported parselets overrides existing ones. If false, they throw.
      * @param block The code to execute when the type matches.
      */
-    public fun infix(type: T, precedence: Int, override: Boolean = false, block: InfixFunction<T, E>)
+    public fun infix(type: T, precedence: Int, override: Boolean = false, block: InfixFunction<T, K, E>)
 }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/dsl/GrammarFunctionTypes.kt

@@ -1,23 +1,22 @@
 package com.github.adriantodt.tartar.api.dsl
 
 import com.github.adriantodt.tartar.api.parser.ParserContext
-import com.github.adriantodt.tartar.api.parser.Token
 
 /**
  * Function used by [GrammarDSL] to configure a
- * [Prefix Parselet][com.github.adriantodt.tartar.api.parser.PrefixParselet]
+ * [Prefix Parselet][com.github.adriantodt.tartar.api.grammar.PrefixParselet]
  * in a functional way.
  */
-public typealias PrefixFunction<T, E> = ParserContext<T, E>.(token: Token<T>) -> E
+public typealias PrefixFunction<T, K, E> = ParserContext<T, K, E>.(token: K) -> E
 
 /**
  * Function used by [GrammarDSL] to configure a
- * [Infix Parselet][com.github.adriantodt.tartar.api.parser.InfixParselet]
+ * [Infix Parselet][com.github.adriantodt.tartar.api.grammar.InfixParselet]
  * in a functional way.
  */
-public typealias InfixFunction<T, E> = ParserContext<T, E>.(left: E, token: Token<T>) -> E
+public typealias InfixFunction<T, K, E> = ParserContext<T, K, E>.(left: E, token: K) -> E
 
 /**
  * Function which receives a [LexerDSL] as its receiver.
  */
-public typealias GrammarConfig<T, E> = GrammarDSL<T, E>.() -> Unit
+public typealias GrammarConfig<T, K, E> = GrammarDSL<T, K, E>.() -> Unit

src/commonMain/kotlin/com/github/adriantodt/tartar/api/dsl/ParserFunctionTypes.kt

@@ -5,4 +5,4 @@ import com.github.adriantodt.tartar.api.parser.ParserContext
 /**
  * Function which receives a [ParserContext] as its receiver, and returns a result.
  */
-public typealias ParserFunction<T, E, R> = ParserContext<T, E>.() -> R
+public typealias ParserFunction<T, K, E, R> = ParserContext<T, K, E>.() -> R

src/commonMain/kotlin/com/github/adriantodt/tartar/api/grammar/Grammar.kt

@@ -1,6 +1,7 @@
 package com.github.adriantodt.tartar.api.grammar
 
 import com.github.adriantodt.tartar.api.dsl.GrammarConfig
+import com.github.adriantodt.tartar.api.parser.Token
 import com.github.adriantodt.tartar.impl.GrammarBuilder
 import kotlin.jvm.JvmStatic
 
@@ -13,9 +14,9 @@ import kotlin.jvm.JvmStatic
  * @param infix A map of prefix parsers for each token type.
  * @author An Tran
  */
-public data class Grammar<T, E>(
-    public val prefix: Map<T, PrefixParselet<T, E>>,
-    public val infix: Map<T, InfixParselet<T, E>>
+public data class Grammar<T, K : Token<T>, E>(
+    public val prefix: Map<T, PrefixParselet<T, K, E>>,
+    public val infix: Map<T, InfixParselet<T, K, E>>
 ) {
     public companion object {
         /**
@@ -28,8 +29,8 @@ public data class Grammar<T, E>(
          * @author AdrianTodt
          */
         @JvmStatic
-        public fun <T, E> create(block: GrammarConfig<T, E>): Grammar<T, E> {
-            return GrammarBuilder<T, E>().apply(block).build()
+        public fun <T, K: Token<T>, E> create(block: GrammarConfig<T, K, E>): Grammar<T, K, E> {
+            return GrammarBuilder<T, K, E>().apply(block).build()
         }
     }
 }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/grammar/InfixParselet.kt

@@ -10,7 +10,7 @@ import com.github.adriantodt.tartar.api.parser.Token
  * @param E The grammar's expression result.
  * @author An Tran
  */
-public interface InfixParselet<T, E> {
+public interface InfixParselet<T, K: Token<T>, E> {
     /**
      * This infix parser's precedence.
      */
@@ -19,5 +19,5 @@ public interface InfixParselet<T, E> {
     /**
      * This infix parser's parsing implementation.
      */
-    public fun parse(ctx: ParserContext<T, E>, left: E, token: Token<T>): E
+    public fun parse(ctx: ParserContext<T, K, E>, left: E, token: K): E
 }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/grammar/PrefixParselet.kt

@@ -10,9 +10,9 @@ import com.github.adriantodt.tartar.api.parser.Token
  * @param E The grammar's expression result.
  * @author An Tran
  */
-public interface PrefixParselet<T, E> {
+public interface PrefixParselet<T, K: Token<T>, E> {
     /**
      * This prefix parser's parsing implementation.
      */
-    public fun parse(ctx: ParserContext<T, E>, token: Token<T>): E
+    public fun parse(ctx: ParserContext<T, K, E>, token: K): E
 }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/parser/Parser.kt

@@ -15,11 +15,11 @@ import kotlin.jvm.JvmStatic
  * @param R The parser's result.
  * @author AdrianTodt
  */
-public interface Parser<T, E, R> {
+public interface Parser<T, K : Token<T>, E, R> {
     /**
      * The [Grammar] of this pratt-parser.
      */
-    public val grammar: Grammar<T, E>
+    public val grammar: Grammar<T, K, E>
 
     /**
      * Parses tokens with this pratt-parser and returns the computed result.
@@ -28,7 +28,7 @@ public interface Parser<T, E, R> {
      * @param tokens A list of tokens, probably created with [Lexer].
      * @return The computed result.
      */
-    public fun parse(source: Source, tokens: List<Token<T>>): R
+    public fun parse(source: Source, tokens: List<K>): R
 
     /**
      * Parses tokens from a source, using a specified lexer, with this pratt-parser, and returns the computed result.
@@ -37,7 +37,7 @@ public interface Parser<T, E, R> {
      * @param lexer A lexer to parse the source.
      * @return The computed result.
      */
-    public fun parse(source: Source, lexer: Lexer<Token<T>>): R {
+    public fun parse(source: Source, lexer: Lexer<K>): R {
         return parse(source, lexer.parseToList(source))
     }
 
@@ -54,7 +54,10 @@ public interface Parser<T, E, R> {
          * @author AdrianTodt
          */
         @JvmStatic
-        public fun <T, E, R> create(grammar: Grammar<T, E>, block: ParserFunction<T, E, R>): Parser<T, E, R> {
+        public fun <T, K : Token<T>, E, R> create(
+            grammar: Grammar<T, K, E>,
+            block: ParserFunction<T, K, E, R>
+        ): Parser<T, K, E, R> {
             return ParserImpl(grammar, block)
         }
     }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/parser/ParserContext.kt

@@ -6,11 +6,11 @@ import com.github.adriantodt.tartar.api.lexer.Source
 /**
  * A parsing context, created by a [Parser.parse] call, which exposes an interface for pratt-parsing.
  *
- * @param T The parser's (and grammar's) token type.
- * @param E The parser's (and grammar's) expression result.
+ * @param T The parser's (and underlying grammar's) token type.
+ * @param E The parser's (and underlying grammar's) expression result.
  * @author AdrianTodt
  */
-public interface ParserContext<T, E> {
+public interface ParserContext<T, K: Token<T>, E> {
     /**
      * The source of this grammar's tokens.
      */
@@ -19,7 +19,7 @@ public interface ParserContext<T, E> {
     /**
      * The grammar of this parser's context.
      */
-    public val grammar: Grammar<T, E>
+    public val grammar: Grammar<T, K, E>
 
     /**
      * The current index in the list of tokens.
@@ -39,7 +39,7 @@ public interface ParserContext<T, E> {
     /**
      * Creates a child parser context with the specified grammar.
      */
-    public fun withGrammar(grammar: Grammar<T, E>): ParserContext<T, E>
+    public fun withGrammar(grammar: Grammar<T, K, E>): ParserContext<T, K, E>
 
     /**
      * Parses the expression using this parser's grammar.
@@ -49,17 +49,19 @@ public interface ParserContext<T, E> {
     /**
      * Parses the expression using another grammar.
      */
-    public fun Grammar<T, E>.parseExpression(precedence: Int = 0): E = withGrammar(grammar).parseExpression(precedence)
+    public fun Grammar<T, K, E>.parseExpression(precedence: Int = 0): E {
+        return withGrammar(grammar).parseExpression(precedence)
+    }
 
     /**
      * Eats the current token, advancing the index by one.
      */
-    public fun eat(): Token<T>
+    public fun eat(): K
 
     /**
      * Eats the current token, advancing the index by one. Throws a [SyntaxException] if the token type doesn't match.
      */
-    public fun eat(type: T): Token<T>
+    public fun eat(type: T): K
 
     /**
      * Equivalent to [nextIs], but eats the current token if true.
@@ -74,12 +76,12 @@ public interface ParserContext<T, E> {
     /**
      * Move the index backwards one token and returns it.
      */
-    public fun back(): Token<T>
+    public fun back(): K
 
     /**
      * Peeks a token a distance far away of the reader.
      */
-    public fun peek(distance: Int = 0): Token<T>
+    public fun peek(distance: Int = 0): K
 
     /**
      * Peeks the next token and if the token types are equal, returns true.
@@ -94,7 +96,7 @@ public interface ParserContext<T, E> {
     /**
      * Peeks the tokens ahead until a token of any of the types is found.
      */
-    public fun peekAheadUntil(vararg type: T): List<Token<T>>
+    public fun peekAheadUntil(vararg type: T): List<K>
 
     /**
      * Skips tokens ahead until a token of any of the types is found.

src/commonMain/kotlin/com/github/adriantodt/tartar/api/parser/SourceParser.kt

@@ -15,7 +15,10 @@ import com.github.adriantodt.tartar.api.lexer.Source
  * @param parser The underlying parser.
  * @author AdrianTodt
  */
-public data class SourceParser<T, E, R>(public val lexer: Lexer<Token<T>>, public val parser: Parser<T, E, R>) {
+public data class SourceParser<T, K : Token<T>, E, R>(
+    public val lexer: Lexer<K>,
+    public val parser: Parser<T, K, E, R>
+) {
     /**
      * Parses tokens from a source, using the bundled lexer, with this pratt-parser, and returns the computed result.
      *
@@ -39,11 +42,11 @@ public data class SourceParser<T, E, R>(public val lexer: Lexer<Token<T>>, publi
          * @return A configured Parser.
          * @author AdrianTodt
          */
-        public fun <T, E, R> create(
-            lexer: Lexer<Token<T>>,
-            grammar: Grammar<T, E>,
-            block: ParserFunction<T, E, R>
-        ): SourceParser<T, E, R> {
+        public fun <T, K: Token<T>, E, R> create(
+            lexer: Lexer<K>,
+            grammar: Grammar<T, K, E>,
+            block: ParserFunction<T, K, E, R>
+        ): SourceParser<T, K, E, R> {
             return SourceParser(lexer, Parser.create(grammar, block))
         }
     }

src/commonMain/kotlin/com/github/adriantodt/tartar/api/parser/StringToken.kt

@@ -0,0 +1,42 @@
+package com.github.adriantodt.tartar.api.parser
+
+import com.github.adriantodt.tartar.api.lexer.Section
+
+/**
+ * A subclass of [Token] which holds a [String] as its [value].
+ *
+ * @param T The type of the token.
+ * @param type The type of the token.
+ * @param value The value of this token.
+ * @param section The section of this token.
+ * @author An Tran, AdrianTodt
+ *
+ * @see Token
+ */
+public class StringToken<out T>(type: T, public val value: String, section: Section) : Token<T>(type, section) {
+    /**
+     * Returns a string representation of the token.
+     */
+    override fun toString(): String {
+        if (value.isNotEmpty()) {
+            return "$type[$value] $section"
+        }
+        return super.toString()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+        if (!super.equals(other)) return false
+
+        other as StringToken<*>
+
+        if (value != other.value) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return 31 * super.hashCode() + value.hashCode()
+    }
+}