Pasha/MIRROR-LX-OriginalRepo
1
0
Fork 0
mirror of https://github.com/PashaBibko/LX.git synced 2026-04-04 09:59:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Changed how tokens are represented in memory

Browse Source 
Also made string-literal tokens now include the quotes as part of their contents.
This commit is contained in:
Pasha Bibko
2025-04-22 10:55:35 +01:00
parent 4b47e803ce
commit c95d91b867
6 changed files with 40 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
common/Lexer.h
View File

@@ -62,6 +62,9 @@ namespace LX
// Data type to store a more computer readable version of files
struct __declspec(novtable) Token final
{
//
static std::string* source;
// Enum to hold the type of the token //
enum TokenType : short
{
@@ -87,13 +90,12 @@ namespace LX
UNDEFINED = -1
};
// Constructor of the tokens to set their info //
Token(const TokenType _type, const LexerInfo& info, std::string _contents, std::streamsize _length);
Token(const TokenType _type, const LexerInfo& info, std::streamsize _length);
// Contents of the token (may be empty if not needed) //
// Const to avoid external changes //
const std::string contents;
//
std::string GetContents() const;
// Type of the token //
// Const to avoid external changes //
@@ -102,7 +104,7 @@ namespace LX
// Index in the source of the token //
const std::streamsize index;
// The length of the token on the line, may be different to the length of contents //
// The length of the token on the line //
const std::streamsize length;
// The line the token is located on //