tokens.oc

//* Definitions for Tokens

import std::span::{ Span, Location }
import std::libc::calloc

struct Token {
    type: TokenType
    span: Span
    text: str
    suffix: &Token
    seen_newline: bool

    //* Comment occuring *before* this token
    comment: str
    comment_loc: Location
}

def Token::new(type: TokenType, span: Span, text: str): &Token {
    let tok = calloc(1, sizeof(Token)) as &Token
    *tok = Token(type, span, text, suffix: null, seen_newline: false, comment: null, comment_loc: Location::default())
    return tok
}

def Token::from_type(type: TokenType, span: Span): &Token => Token::new(type, span, "")

def Token::from_ident(text: str, span: Span): &Token {
    let type = TokenType::from_text(text)
    return Token::new(type, span, text)
}

def Token::str(&this): str => `{.span.str()}: {.type.str()}`


def Token::is_word(this): bool => match .type {
    TokenType::Identifier => true
    else => .type as u64 > TokenType::BEGIN_KEYWORDS as u64
}

enum TokenType {
    // Other tokens
    AtSign
    Ampersand
    Backtick
    Caret
    CharLiteral
    CloseCurly
    CloseParen
    CloseSquare
    Colon
    ColonColon
    Comma
    Dot
    EOF
    Equals
    EqualEquals
    Exclamation
    FatArrow
    FloatLiteral
    FormatStringLiteral
    GreaterThan
    GreaterThanEquals
    Identifier
    IntLiteral
    LessThan
    LessThanEquals
    Line
    Minus
    MinusEquals
    MinusMinus
    NotEquals
    OpenCurly
    OpenParen
    OpenSquare
    Percent
    Plus
    PlusEquals
    PlusPlus
    Question
    Semicolon
    Slash
    SlashEquals
    Star
    StarEquals
    StringLiteral
    Tilde

    //* Not an actual token, just here for convenience
    Newline

    BEGIN_KEYWORDS

    // Keywords
    And
    As
    Assert
    Break
    Const
    Continue
    Def
    Defer
    Else
    Enum
    Extern
    False
    For
    Fn
    If
    Let
    Match
    Namespace
    Null
    Not
    Or
    Return
    SizeOf
    Struct
    True
    Then
    Union
    Import
    Void
    Yield
    While
}

def TokenType::from_text(text: str): TokenType => match text {
    "and" => TokenType::And
    "as" => TokenType::As
    "assert" => TokenType::Assert
    "break" => TokenType::Break
    "const" => TokenType::Const
    "continue" => TokenType::Continue
    "def" => TokenType::Def
    "defer" => TokenType::Defer
    "else" => TokenType::Else
    "enum" => TokenType::Enum
    "extern" => TokenType::Extern
    "false" => TokenType::False
    "for" => TokenType::For
    "fn" => TokenType::Fn
    "if" => TokenType::If
    "let" => TokenType::Let
    "match" => TokenType::Match
    "namespace" => TokenType::Namespace
    "not" => TokenType::Not
    "null" => TokenType::Null
    "or" => TokenType::Or
    "return" => TokenType::Return
    "sizeof" => TokenType::SizeOf
    "struct" => TokenType::Struct
    "true" => TokenType::True
    "then" => TokenType::Then
    "union" => TokenType::Union
    "import" => TokenType::Import
    "void" => TokenType::Void
    "yield" => TokenType::Yield
    "while" => TokenType::While
    else => TokenType::Identifier
}

def TokenType::str(this): str => match this {
    // Keywords
    And => "and"
    As => "as"
    Assert => "assert"
    Break => "break"
    Const => "const"
    Continue => "continue"
    Def => "def"
    Defer => "defer"
    Else => "else"
    Enum => "enum"
    Extern => "extern"
    False => "false"
    For => "for"
    Fn => "fn"
    If => "if"
    Let => "let"
    Match => "match"
    Namespace => "namespace"
    Not => "not"
    Null => "null"
    Or => "or"
    Return => "return"
    SizeOf => "sizeof"
    Struct => "struct"
    True => "true"
    Then => "then"
    Union => "union"
    Import => "import"
    Void => "void"
    Yield => "yield"
    While => "while"

    // Others
    else => .dbg()
}