Pasha/LXC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed raw-enums

Browse Source
This commit is contained in:
Pasha Bibko
2025-08-24 21:17:50 +01:00
parent 2427d67269
commit 025a1ee0b4
9 changed files with 315 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
lexer/inc/Token.h
View File

@@ -4,92 +4,81 @@
namespace PashaBibko::LXC::Lexer
{
namespace TokenClass
enum class TokenClass : unsigned short
{
// Bitmask for different token classes //
enum ClassMask : unsigned short
{
// Mathematical and logic operators //
Operator = 1 << (1 + 8),
// Mathematical and logic operators //
Operator = 1 << (1 + 8),
// Special words defined by the compiler //
Keyword = 1 << (2 + 8),
// Special words defined by the compiler //
Keyword = 1 << (2 + 8),
// Words such as literals and identifiers //
UserDefined = 1 << (3 + 8),
// Words such as literals and identifiers //
UserDefined = 1 << (3 + 8),
// Symbols in the source like (? , . ! <) //
Symbols = 1 << (4 + 8),
// Symbols in the source like (? , . ! <) //
Symbols = 1 << (4 + 8),
// Tokens not defined by previous classes //
Misc = 1 << (5 + 8)
};
// Tokens not defined by previous classes //
Misc = 1 << (5 + 8)
};
struct LexerContext;
// Enum of token type organised by their token class //
enum class TokenType : unsigned short
{
// === Operators === //
Add = (unsigned short)TokenClass::Operator,
Sub,
Mul,
Div,
Mod,
Eql,
// === Keywords === //
For = (unsigned short)TokenClass::Keyword,
While,
If,
ElseIf,
Else,
Return,
FunctionDef,
// === User defined === //
StringLiteral = (unsigned short)TokenClass::UserDefined,
NumLiteral,
Identifier,
// === Symbols === //
Assign = (unsigned short)TokenClass::Symbols,
Colon,
CloseBracket,
OpenBracket,
CloseBrace,
OpenBrace,
CloseParen,
OpenParen,
CloseCrocodile,
OpenCrocodile,
Comma,
// === Misc === //
End_of_file = (unsigned short)TokenClass::Misc,
UNDEFINED = 65535 // Invalid token type (max number)
};
// Data type for storing the output of the lexer //
class Token final
{
public:
// Enum of token type organised by their token class //
enum TokenType : unsigned short
{
// === Operators === //
Add = TokenClass::Operator,
Sub,
Mul,
Div,
Mod,
Eql,
// === Keywords === //
For = TokenClass::Keyword,
While,
If,
ElseIf,
Else,
Return,
FunctionDef,
// === User defined === //
StringLiteral = TokenClass::UserDefined,
NumLiteral,
Identifier,
// === Symbols === //
Assign = TokenClass::Symbols,
Colon,
CloseBracket,
OpenBracket,
CloseBrace,
OpenBrace,
CloseParen,
OpenParen,
CloseCrocodile,
OpenCrocodile,
Comma,
// === Misc === //
End_of_file = TokenClass::Misc,
UNDEFINED = 65535 // Invalid token type (max number)
};
// Util function calculating wether a token is of a given class //
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type)
template<TokenClass mask> static constexpr bool IsTypeClass(TokenType type)
{
using T = std::underlying_type_t<TokenType>;
return static_cast<T>(type) & static_cast<T>(mask);
@@ -132,7 +121,7 @@ namespace PashaBibko::LXC::Lexer
};
// Function for converting token types to their equivalent C-Strings //
const char* TokenTypeToCStr(Token::TokenType type);
const char* TokenTypeToCStr(TokenType type);
// Typedef for the output type of how the Lexer outputs //
typedef std::vector<Token> LexerOutput;

60
lexer/src/Lexer.cpp
View File

@@ -38,46 +38,46 @@ namespace PashaBibko::LXC::Internal
c == '>' || c == ':';
}
static const std::unordered_map<std::string_view, Lexer::Token::TokenType> operatorMap =
static const std::unordered_map<std::string_view, Lexer::TokenType> operatorMap =
{
{ "+", Lexer::Token::Add },
{ "-", Lexer::Token::Sub },
{ "*", Lexer::Token::Mul },
{ "/", Lexer::Token::Div },
{ "%", Lexer::Token::Mod },
{ "+", Lexer::TokenType::Add },
{ "-", Lexer::TokenType::Sub },
{ "*", Lexer::TokenType::Mul },
{ "/", Lexer::TokenType::Div },
{ "%", Lexer::TokenType::Mod },
{ "==", Lexer::Token::Eql },
{ "==", Lexer::TokenType::Eql },
{ "=", Lexer::Token::Assign }
{ "=", Lexer::TokenType::Assign }
};
static const std::unordered_map<char, Lexer::Token::TokenType> symbolMap =
static const std::unordered_map<char, Lexer::TokenType> symbolMap =
{
{ ',', Lexer::Token::Comma },
{ ':', Lexer::Token::Colon },
{ ',', Lexer::TokenType::Comma },
{ ':', Lexer::TokenType::Colon },
{ '[', Lexer::Token::CloseBracket },
{ ']', Lexer::Token::OpenBracket },
{ '[', Lexer::TokenType::CloseBracket },
{ ']', Lexer::TokenType::OpenBracket },
{ '}', Lexer::Token::CloseBrace },
{ '{', Lexer::Token::OpenBrace },
{ '}', Lexer::TokenType::CloseBrace },
{ '{', Lexer::TokenType::OpenBrace },
{ ')', Lexer::Token::CloseParen },
{ '(', Lexer::Token::OpenParen },
{ ')', Lexer::TokenType::CloseParen },
{ '(', Lexer::TokenType::OpenParen },
{ '>', Lexer::Token::CloseCrocodile },
{ '<', Lexer::Token::OpenCrocodile }
{ '>', Lexer::TokenType::CloseCrocodile },
{ '<', Lexer::TokenType::OpenCrocodile }
};
static const std::unordered_map<std::string_view, Lexer::Token::TokenType> keywords =
static const std::unordered_map<std::string_view, Lexer::TokenType> keywords =
{
{ "for", Lexer::Token::For },
{ "while", Lexer::Token::While },
{ "if", Lexer::Token::If },
{ "elif", Lexer::Token::ElseIf },
{ "else", Lexer::Token::Else },
{ "return", Lexer::Token::Return },
{ "func", Lexer::Token::FunctionDef },
{ "for", Lexer::TokenType::For },
{ "while", Lexer::TokenType::While },
{ "if", Lexer::TokenType::If },
{ "elif", Lexer::TokenType::ElseIf },
{ "else", Lexer::TokenType::Else },
{ "return", Lexer::TokenType::Return },
{ "func", Lexer::TokenType::FunctionDef },
};
}
@@ -126,7 +126,7 @@ namespace PashaBibko::LXC::Lexer
// Creates the token (if at the end of the string literal) //
if (!trackers.inStrLiteral)
ctx.out.emplace_back(ctx, trackers.sectionStart + 1, (unsigned short)(ctx.index - trackers.sectionStart - 1), Token::StringLiteral);
ctx.out.emplace_back(ctx, trackers.sectionStart + 1, (unsigned short)(ctx.index - trackers.sectionStart - 1), TokenType::StringLiteral);
} else if (trackers.inStrLiteral) {}
@@ -140,7 +140,7 @@ namespace PashaBibko::LXC::Lexer
// Checks for the end of the number literal to create the token //
if (!Internal::IsNumeric(next)) _UNLIKELY
{
ctx.out.emplace_back(ctx, trackers.sectionStart, (unsigned short)(ctx.index - trackers.sectionStart + 1), Token::NumLiteral);
ctx.out.emplace_back(ctx, trackers.sectionStart, (unsigned short)(ctx.index - trackers.sectionStart + 1), TokenType::NumLiteral);
trackers.inNumLiteral = false;
}
}
@@ -158,7 +158,7 @@ namespace PashaBibko::LXC::Lexer
// Finds out if the word is a keyword or not //
std::string_view fullWord(ctx.source.data() + trackers.sectionStart, ctx.index - trackers.sectionStart + 1);
auto it = Internal::keywords.find(fullWord);
Token::TokenType tType = (it != Internal::keywords.end()) ? it->second : Token::Identifier;
TokenType tType = (it != Internal::keywords.end()) ? it->second : TokenType::Identifier;
ctx.out.emplace_back(ctx, trackers.sectionStart, (unsigned short)(ctx.index - trackers.sectionStart + 1), tType);
trackers.inIdentifier = false;

60
lexer/src/Token.cpp
View File

@@ -55,46 +55,46 @@ namespace PashaBibko::LXC::Lexer
// Helper macro for converting type to string //
#define TOKEN_TYPE_CASE(type) case type: return #type;
const char* TokenTypeToCStr(Token::TokenType type)
const char* TokenTypeToCStr(TokenType type)
{
switch (type)
{
// All the different types of tokens //
TOKEN_TYPE_CASE(Token::Add);
TOKEN_TYPE_CASE(Token::Sub);
TOKEN_TYPE_CASE(Token::Mul);
TOKEN_TYPE_CASE(Token::Div);
TOKEN_TYPE_CASE(Token::Mod);
TOKEN_TYPE_CASE(TokenType::Add);
TOKEN_TYPE_CASE(TokenType::Sub);
TOKEN_TYPE_CASE(TokenType::Mul);
TOKEN_TYPE_CASE(TokenType::Div);
TOKEN_TYPE_CASE(TokenType::Mod);
TOKEN_TYPE_CASE(Token::Eql);
TOKEN_TYPE_CASE(TokenType::Eql);
TOKEN_TYPE_CASE(Token::For);
TOKEN_TYPE_CASE(Token::While);
TOKEN_TYPE_CASE(Token::If);
TOKEN_TYPE_CASE(Token::ElseIf);
TOKEN_TYPE_CASE(Token::Else);
TOKEN_TYPE_CASE(Token::Return);
TOKEN_TYPE_CASE(TokenType::For);
TOKEN_TYPE_CASE(TokenType::While);
TOKEN_TYPE_CASE(TokenType::If);
TOKEN_TYPE_CASE(TokenType::ElseIf);
TOKEN_TYPE_CASE(TokenType::Else);
TOKEN_TYPE_CASE(TokenType::Return);
TOKEN_TYPE_CASE(Token::FunctionDef);
TOKEN_TYPE_CASE(TokenType::FunctionDef);
TOKEN_TYPE_CASE(Token::StringLiteral);
TOKEN_TYPE_CASE(Token::NumLiteral);
TOKEN_TYPE_CASE(Token::Identifier);
TOKEN_TYPE_CASE(TokenType::StringLiteral);
TOKEN_TYPE_CASE(TokenType::NumLiteral);
TOKEN_TYPE_CASE(TokenType::Identifier);
TOKEN_TYPE_CASE(Token::Assign);
TOKEN_TYPE_CASE(Token::Colon);
TOKEN_TYPE_CASE(Token::CloseBracket);
TOKEN_TYPE_CASE(Token::OpenBracket);
TOKEN_TYPE_CASE(Token::CloseBrace);
TOKEN_TYPE_CASE(Token::OpenBrace);
TOKEN_TYPE_CASE(Token::CloseParen);
TOKEN_TYPE_CASE(Token::OpenParen);
TOKEN_TYPE_CASE(Token::CloseCrocodile);
TOKEN_TYPE_CASE(Token::OpenCrocodile);
TOKEN_TYPE_CASE(Token::Comma);
TOKEN_TYPE_CASE(TokenType::Assign);
TOKEN_TYPE_CASE(TokenType::Colon);
TOKEN_TYPE_CASE(TokenType::CloseBracket);
TOKEN_TYPE_CASE(TokenType::OpenBracket);
TOKEN_TYPE_CASE(TokenType::CloseBrace);
TOKEN_TYPE_CASE(TokenType::OpenBrace);
TOKEN_TYPE_CASE(TokenType::CloseParen);
TOKEN_TYPE_CASE(TokenType::OpenParen);
TOKEN_TYPE_CASE(TokenType::CloseCrocodile);
TOKEN_TYPE_CASE(TokenType::OpenCrocodile);
TOKEN_TYPE_CASE(TokenType::Comma);
TOKEN_TYPE_CASE(Token::End_of_file);
TOKEN_TYPE_CASE(Token::UNDEFINED);
TOKEN_TYPE_CASE(TokenType::End_of_file);
TOKEN_TYPE_CASE(TokenType::UNDEFINED);
// When the case has not been defined yet //
default: