Lexer.h

Go to the documentation of this file.

 
#pragma once
 
#include <memory>
#include <sstream>
 
#include <ext/iostream>
 
#include <alib/string>
 
#include <exception/CommonException.h>
 
#include <lexer/CharSequence.h>
 
namespace cli {
 
class Lexer {
public:
    enum class Hint {
        NONE,
        TYPE,
        FILE
    };
 
private:
    CharSequence m_source;
    Hint m_hint;
 
public:
    enum class TokenType : unsigned {
        IDENTIFIER,
        UNSIGNED,
        DOUBLE,
        STRING,
 
        LESS_THAN,
        LESS_THAN_OR_EQUAL,
        MORE_THAN,
        MORE_THAN_OR_EQUAL,
        EQUAL,
        NOT_EQUAL,
 
        LEFT_PAREN,
        RIGHT_PAREN,
        LEFT_BRACE,
        RIGHT_BRACE,
        LEFT_BRACKET,
        RIGHT_BRACKET,
 
        DOLLAR_SIGN,
        AT_SIGN,
        AMPERSAND_SIGN,
        PIPE_SIGN,
        CARET_SIGN,
        COLON_SIGN,
        SEMICOLON_SIGN,
        MINUS_SIGN,
        PLUS_SIGN,
        SLASH_SIGN,
        ASTERISK_SIGN,
        TILDE_SIGN,
        EXCLAMATION_SIGN,
        PERCENTAGE_SIGN,
        HASH_SIGN,
 
        AND,
        OR,
        DEC,
        INC,
        ASSIGN,
 
        COMMA,
        DOT,
 
        FILE,
        TYPE,
        ERROR,
        EOT,
        EOS
    };
 
    friend bool operator < ( TokenType first, TokenType second ) {
        return static_cast < unsigned > ( first ) < static_cast < unsigned > ( second );
    }
 
    static std::string tokenTypeToString ( TokenType type ) {
        switch ( type ) {
        case TokenType::IDENTIFIER :
            return "identifier";
        case TokenType::UNSIGNED :
            return "unsigned";
        case TokenType::DOUBLE :
            return "double";
        case TokenType::STRING :
            return "string";
 
        case TokenType::LESS_THAN :
            return "less_than";
        case TokenType::LESS_THAN_OR_EQUAL :
            return "less_than_or_equal";
        case TokenType::MORE_THAN :
            return "more_than";
        case TokenType::MORE_THAN_OR_EQUAL :
            return "more_or_equal_operator";
        case TokenType::EQUAL :
            return "equal";
        case TokenType::NOT_EQUAL :
            return "not_equal";
 
        case TokenType::LEFT_PAREN :
            return "left_paren";
        case TokenType::RIGHT_PAREN :
            return "right_paren";
        case TokenType::LEFT_BRACE :
            return "left_brace";
        case TokenType::RIGHT_BRACE :
            return "right_brace";
        case TokenType::LEFT_BRACKET :
            return "left_bracket";
        case TokenType::RIGHT_BRACKET :
            return "right_bracket";
 
        case TokenType::DOLLAR_SIGN :
            return "dolar_sign";
        case TokenType::AT_SIGN :
            return "at_sign";
        case TokenType::AMPERSAND_SIGN :
            return "ampersand_sign";
        case TokenType::PIPE_SIGN :
            return "pipe_sign";
        case TokenType::CARET_SIGN :
            return "caret_sign";
        case TokenType::COLON_SIGN :
            return "colon_sign";
        case TokenType::SEMICOLON_SIGN :
            return "semicolon_sign";
        case TokenType::MINUS_SIGN :
            return "minus_sign";
        case TokenType::PLUS_SIGN :
            return "plus_sign";
        case TokenType::SLASH_SIGN :
            return "slash_sign";
        case TokenType::ASTERISK_SIGN :
            return "asterisk_sign";
        case TokenType::TILDE_SIGN :
            return "tilde_sign";
        case TokenType::EXCLAMATION_SIGN :
            return "exclemation_sign";
        case TokenType::PERCENTAGE_SIGN :
            return "percentage_sign";
        case TokenType::HASH_SIGN :
            return "hash_sign";
 
        case TokenType::AND:
            return "and";
        case TokenType::OR:
            return "or";
        case TokenType::DEC:
            return "dec";
        case TokenType::INC:
            return "inc";
        case TokenType::ASSIGN:
            return "assign";
 
        case TokenType::COMMA :
            return "comma";
        case TokenType::DOT :
            return "dot";
 
        case TokenType::FILE :
            return "file";
        case TokenType::TYPE :
            return "type";
        case TokenType::ERROR :
            return "error";
        case TokenType::EOT :
            return "eot";
        case TokenType::EOS :
            return "eos";
        default:
            throw exception::CommonException ( "Unhandled case in Lexer::tokenTypeToString" );
        }
    }
 
    static TokenType is_kw ( const std::string & /* identifier */ ) {
        return TokenType::IDENTIFIER;
    }
 
    struct Token {
        std::string m_value;
        std::string m_raw;
        TokenType m_type;
 
        size_t m_line = 0;
        size_t m_position = 0;
 
        size_t m_raw_line = 0;
        size_t m_raw_position = 0;
 
        friend std::ostream & operator << ( std::ostream & out, const Token & token ) {
            out << Lexer::tokenTypeToString ( token.m_type );
 
            if ( token.m_value.empty ( ) )
                return out;
 
            switch ( token.m_type ) {
            case TokenType::IDENTIFIER :
            case TokenType::UNSIGNED :
            case TokenType::DOUBLE :
                return out << ": " << token.m_value;
            case TokenType::STRING :
                return out << ": \"" << token.m_value << "\"";
            default:
                return out;
            }
        }
 
        bool operator < ( const Token & token ) const {
            return std::tie ( m_type, m_value ) < std::tie ( token.m_type, token.m_value ); // m_raw omitted intentionally
        }
 
    };
 
    explicit Lexer ( CharSequence source ) : m_source ( std::move ( source ) ), m_hint ( Hint::NONE ) {
        m_source.advance ( true );
    }
 
    Token nextToken ( bool readNextLine = false );
 
    void putback ( Token && token ) {
        m_source.putback ( std::move ( token.m_raw ), token.m_raw_line, token.m_raw_position );
    }
 
    void setHint ( Hint hint ) {
        m_hint = hint;
    }
};
 
} /* namespace cli */