1 #include "source_file/token.h"
6 umap<TokenType, str> token_type_names{
7 {TokenType::COMMENTS,
"COMMENTS" },
8 {TokenType::KEYWORD,
"KEYWORD" },
9 {TokenType::INT,
"INT" },
10 {TokenType::FLOAT,
"FLOAT" },
11 {TokenType::ID,
"ID" },
12 {TokenType::CHAR,
"CHAR" },
13 {TokenType::STRING,
"STRING" },
14 {TokenType::PUNCTUATION,
"PUNCTUATION"},
15 {TokenType::RELOP,
"RELOP" },
16 {TokenType::UOP,
"UOP" },
17 {TokenType::BOP,
"BOP" },
21 const vector<str> KEYWORDS{
22 "for",
"while",
"if",
"else",
"fn",
"var",
"continue",
"break",
"let",
"struct",
"union",
23 "switch",
"case",
"return",
"pub",
"extern",
"import",
"as",
"true",
"false",
"package",
26 const vector<char> PUNCTUATIONS{
'~',
'!',
'#',
'%',
'^',
'&',
'*',
'(',
')',
'-',
'=',
'+',
'[',
']',
'{',
27 '}',
'\\',
'|',
';',
':',
'\'',
'"',
',',
'.',
'<',
'>',
'/',
'?',
'@'};
29 const vector<str> TERMINAL_TOKENS{
";",
"}",
")",
":",
",",
"]"};
32 const vector<char> OP{
'~',
'!',
'%',
'^',
'&',
'*',
'-',
'=',
'+',
'|',
'<',
'>',
'/',
'.'};
34 const vector<str> OP_ALL{
"==",
"!=",
">=",
"<=",
">",
"<",
"&&",
"||",
"~",
"%=",
"%",
"^=",
"^",
"&=",
"&",
"+=",
35 "+",
"-=",
"-",
"*=",
"*",
"/=",
"/",
"|=",
"|",
"<<=",
"<<",
">>=",
">>",
"!=",
"."};
37 umap<str, TokenType> OPERATION_VALUE_TYPE_MAP{
38 pair(
"==", TokenType::RELOP),
39 pair(
"!=", TokenType::RELOP),
40 pair(
">=", TokenType::RELOP),
41 pair(
"<=", TokenType::RELOP),
42 pair(
">", TokenType::RELOP),
43 pair(
"<", TokenType::RELOP),
44 pair(
"&&", TokenType::RELOP),
45 pair(
"||", TokenType::RELOP),
47 pair(
"~", TokenType::UOP),
48 pair(
"!", TokenType::UOP),
50 pair(
"%=", TokenType::BOP),
51 pair(
"%", TokenType::BOP),
52 pair(
"^=", TokenType::BOP),
53 pair(
"^", TokenType::BOP),
54 pair(
"&=", TokenType::BOP),
55 pair(
"&", TokenType::BOP),
56 pair(
"+=", TokenType::BOP),
57 pair(
"+", TokenType::BOP),
58 pair(
"-=", TokenType::BOP),
59 pair(
"-", TokenType::BOP),
60 pair(
"*=", TokenType::BOP),
61 pair(
"*", TokenType::BOP),
62 pair(
"/=", TokenType::BOP),
63 pair(
"/", TokenType::BOP),
64 pair(
"|=", TokenType::BOP),
65 pair(
"|", TokenType::BOP),
66 pair(
"<<=", TokenType::BOP),
67 pair(
"<<", TokenType::BOP),
68 pair(
">>=", TokenType::BOP),
69 pair(
">>", TokenType::BOP),
70 pair(
"!=", TokenType::BOP),
71 pair(
",", TokenType::BOP),
72 pair(
".", TokenType::BOP),
73 pair(
"=", TokenType::BOP)};
75 Token::Token(TokenType tokenType, uint32_t line, uint32_t col, str value, SourceFile *src)
76 : _type(tokenType), _value(std::move(value)), _line(line), _col(col), _src(src) {}
78 TokenType Token::get_type()
const {
return _type; }
80 void Token::set_type(TokenType type) { _type = type; }
82 const str &Token::get_value()
const {
return _value; }
84 str Token::get_source_line()
const {
return _src->get_line(_line); }
86 bool Token::is_unsigned()
const {
return _is_unsigned; }
88 void Token::set_is_unsigned(
bool is_unsigned) { _is_unsigned = is_unsigned; }
90 uint32_t Token::get_line()
const {
return _line; }
92 uint32_t Token::get_col()
const {
return _col; }
94 SourceSpan Token::GetSourceSpan(
const Token &start,
const Token &end) {
95 TAN_ASSERT(start._src == end._src);
96 return SourceSpan(start._src, start._line, start._col, end._line, end._col);
99 SrcLoc Token::GetSrcLoc(
const Token *tok) {
return SrcLoc(tok->get_line(), tok->get_col(), tok->src()); }
101 SourceFile *Token::src()
const {
return _src; }