Pasha/LXC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added basic lexer

Browse Source
This commit is contained in:
Pasha Bibko
2025-07-20 16:15:58 +01:00
parent f5bb46788c
commit 903b4da7df
4 changed files with 103 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Lexer/inc/Lexer.h
View File

@@ -11,10 +11,10 @@ namespace LXC::Lexer
// Trackers for the Lexer itself //
const std::string& source;
size_t index;
__int32 index;
LexerOutput out;
const size_t len;
const __int32 len;
// Trackers for where the Lexer is within the user version of source //
unsigned short column;

21
Lexer/inc/Token.h
View File

@@ -4,9 +4,6 @@
namespace LXC::Lexer
{
// Foward declaration to allow it passing to the Token class //
struct LexerContext;
namespace TokenClass
{
// Bitmask for different token classes //
@@ -29,6 +26,8 @@ namespace LXC::Lexer
};
};
struct LexerContext;
// Data type for storing the output of the lexer //
class Token final
{
@@ -56,8 +55,7 @@ namespace LXC::Lexer
// === User defined === //
String_Literal = TokenClass::UserDefined,
Int_Literal,
Float_Literal,
Num_Literal,
Identifier,
// === Symbols === //
@@ -86,8 +84,8 @@ namespace LXC::Lexer
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; }
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; }
// Constructor to set the data of the token //
Token(const LexerContext& context, const unsigned short _length, TokenType _type);
// Constructor to set the data of the token for more complex token types //
Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type);
// Deconstructor to clean up the allocated memory //
~Token();
@@ -102,15 +100,12 @@ namespace LXC::Lexer
// The length of the token //
const unsigned short length;
// The line the token is on (starts on 1) //
const unsigned short line;
// The index on the line (starts on 1) //
const unsigned short column;
// Start index of the token //
const unsigned __int32 index;
private:
// The data of the token //
const char* contents;
char* contents;
};
// Typedef for the output type of how the Lexer outputs //