/// Scanning
let letter = CharacterSet.letters.parse(as:StringToken("letter"))

for token in [letter].tokenize("Hello"){
	print(token)
}

/// Choices
let punctuation = [".",",","!","?"].choice.parse(as: StringToken("punctuation"))

for token in [letter, punctuation].tokenize("Hello!"){
    print(token)
}

/// Skipping
let space = CharacterSet.whitespaces.skip()

for token in [letter, punctuation, space].tokenize("Hello, World!"){
    print(token)
}

let word = letter.require(.oneOrMore).parse(as: StringToken("word"))

for token in [word, punctuation, space].tokenize("Hello, World!"){
    print(token)
}

// Sequences
let properNoun = [CharacterSet.uppercaseLetters, CharacterSet.lowercaseLetters.require(.zeroOrMore)].sequence.parse(as: StringToken("properNoun"))
let classifiedWord = [properNoun,word.parse(as: StringToken("other"))].choice.parse(as: StringToken("word"))

print("Word classification")
for token in [classifiedWord, punctuation, space].tokenize("Jon was here!"){
    print(token)
}

do {
    print(try [[classifiedWord, punctuation, space].choice].parse("Jon was here!"))

} catch let error as ProcessingError {
    print(error.debugDescription)
}

root 
    word 
	    properNoun - 'Jon'
    word 
	    other - 'was'
    word 
	    other - 'here'
    punctuation - '!'

struct Word : Decodable, CustomStringConvertible {
    let properNoun : String?
    let other : String?
    
    var description: String {
        return properNoun != nil ? "properNoun: \(properNoun!)" : "other: \(other!)"
    }
}

struct Sentence : Decodable {
    let words : [Word]
    let punctuation : String
}

do {
    let words = [classifiedWord, space.require(.zeroOrMore)].sequence.require(.oneOrMore).parse(as:StringToken("words"))
    let sentence = try [ [words, punctuation ].sequence ].build("Jon was here!", as: Sentence.self)

    print(sentence)
} catch let error as ProcessingError {
    print(error.debugDescription)
} catch {
    print(error)
}

.build("Jon was here!", as: Sentence.self)

do {
    let sentence = Sentence.build("Hello Jon!")
} catch {
    print(error)
}

grammar Sentence

punctuation = "." | "," | "!" | "?"

properNoun  = .uppercaseLetter .lowercaseLetter*
other       = .letter+

word        = properNoun | other
words       = (word -.whitespace+)+

sentence    = words punctuation

stlrc generate -g Sentence.stlr -l swiftIR -ot ./

import Foundation
import OysterKit

/// Intermediate Representation of the grammar
internal enum SentenceTokens : Int, TokenType, CaseIterable, Equatable {
    typealias T = SentenceTokens

    /// The tokens defined by the grammar
    case `punctuation`, `properNoun`, `other`, `word`, `words`, `sentence`

    /// The rule for the token
    var rule : Rule {
	switch self {
	    /// punctuation
	    case .punctuation:
		return [".", ",", "!", "?"].choice.reference(.structural(token: self))

	    /// properNoun
	    case .properNoun:
		return [CharacterSet.uppercaseLetters,    CharacterSet.lowercaseLetters.require(.zeroOrMore)].sequence.reference(.structural(token: self))

	    /// other
	    case .other:
		return CharacterSet.letters.require(.oneOrMore).reference(.structural(token: self))

	    /// word
	    case .word:
		return [T.properNoun.rule,T.other.rule].choice.reference(.structural(token: self))

	    /// words
	    case .words:
		return [T.word.rule, -CharacterSet.whitespaces.require(.oneOrMore)].sequence.require(.oneOrMore).reference(.structural(token: self))

	    /// sentence
	    case .sentence:
		return [T.words.rule,T.punctuation.rule].sequence.reference(.structural(token: self))
	}
    }

    /// Create a language that can be used for parsing etc
    public static var generatedRules: [Rule] {
	return [T.sentence.rule]
    }
}

public struct Sentence : Codable {

    // Punctuation
    public enum Punctuation : Swift.String, Codable, CaseIterable {
	case period = ".",comma = ",",ping = "!",questionMark = "?"
    }

    // Word
    public enum Word : Swift.String, Codable, CaseIterable {
	case properNoun,other
    }

    public typealias Words = [Word] 

    /// Sentence 
    public struct Sentence : Codable {
	public let words: Words
	public let punctuation: Punctuation
    }
    public let sentence : Sentence
    
    /**
     Parses the supplied string using the generated grammar into a new instance of
     the generated data structure

     - Parameter source: The string to parse
     - Returns: A new instance of the data-structure
     */
    public static func build(_ source : Swift.String) throws ->Sentence{
	let root = HomogenousTree(with: StringToken("root"), matching: source, children: [try AbstractSyntaxTreeConstructor().build(source, using: Sentence.generatedLanguage)])
	// print(root.description)
	return try ParsingDecoder().decode(Sentence.self, using: root)
    }

    public static var generatedLanguage : Grammar {return SentenceTokens.generatedRules}
}