Cleaned up Lexer

This commit is contained in:
Pasha Bibko
2025-05-07 18:10:15 +01:00
parent 6783564f10
commit 98fc4589ab
11 changed files with 246 additions and 203 deletions

42
Lexer/inc/Constants.h Normal file
View File

@@ -0,0 +1,42 @@
#include <LX-Common.h>
#include <Lexer.h>
namespace LX
{
// All the keywords the lexer currently supports with their token-enum equivalents //
static const std::unordered_map<std::string, Token::TokenType> keywords =
{
{ "for" , Token::FOR },
{ "while" , Token::WHILE },
{ "if" , Token::IF },
{ "else" , Token::ELSE },
{ "elif" , Token::ELIF },
{ "func" , Token::FUNCTION },
{ "return" , Token::RETURN },
{ "int" , Token::INT_DEC }
};
// All the symbols supported by the lexer //
static const std::unordered_map<char, Token::TokenType> symbols =
{
{ '{', Token::OPEN_BRACKET },
{ '}', Token::CLOSE_BRACKET },
{ '[', Token::OPEN_BRACE },
{ ']', Token::CLOSE_BRACE },
{ '(', Token::OPEN_PAREN },
{ ')', Token::CLOSE_PAREN },
{ ',', Token::COMMA },
{ '=', Token::ASSIGN }
};
// All the single-char operators currently supported by the lexer with their token-enum equivalents //
// TODO: Support multi-char operators such as: ==, -> +=, &&
static const std::unordered_map<char, Token::TokenType> operators =
{
{ '+', Token::ADD },
{ '-', Token::SUB },
{ '*', Token::MUL },
{ '/', Token::DIV }
};
}

View File

@@ -11,7 +11,7 @@ namespace LX
{
GENERATE_LX_ERROR_REQUIRED_FUNCTION_DECLARATIONS;
InvalidCharInSource(const LexerInfo& info, const std::string& source, const std::string _file);
InvalidCharInSource(const LexerInfo& info, const std::string _file);
std::string lineContents;
std::string file;

View File

@@ -7,6 +7,11 @@ namespace LX
// Struct to store the current information of the lexer //
struct LexerInfo
{
// Constructor to set the constants //
LexerInfo(const std::string& _source)
: source(_source), len(_source.length())
{}
// Current trackers of where in the source it is //
std::streamsize line = 1; // <- Lines start on 1 (probably because of non-programmer's)
@@ -19,6 +24,11 @@ namespace LX
std::streamsize startOfNumberLiteral = 0;
std::streamsize startOfStringLiteral = 0;
// Information about the source //
const std::string& source;
const std::streamsize len;
// Different flags of the lexer //
// Stored as a bitset to minimse memory allocated //
// - Basically no difference, because only one exists at any given time //