Pasha/MIRROR-LX-OriginalRepo
1
0
Fork 0
mirror of https://github.com/PashaBibko/LX.git synced 2026-04-04 01:49:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved lexer debugging

Browse Source
This commit is contained in:
Pasha Bibko
2025-04-21 14:23:47 +01:00
parent a560c53c58
commit 49e4bba510
15 changed files with 405 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
common/Lexer.h
View File

@@ -28,13 +28,17 @@ namespace LX
struct InvalidCharInSource
{
std::streamsize index;
std::streamsize line;
std::string lineContents;
char invalid;
};
// Data type to store a more computer readable version of files
struct __declspec(novtable) Token final
{
// Enum to hold the type of the token
// Enum to hold the type of the token //
enum TokenType : short
{
// General tokens //
@@ -60,18 +64,27 @@ namespace LX
UNDEFINED = -1
};
// Constructor of the tokens to set their info
Token(const TokenType _type, std::string _contents);
// Constructor of the tokens to set their info //
Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length);
// Contents of the token (may be empty if not needed)
// Const to avoid external changes
// Contents of the token (may be empty if not needed) //
// Const to avoid external changes //
const std::string contents;
// Type of the token
// Const to avoid external changes
// Type of the token //
// Const to avoid external changes //
const TokenType type;
// The line where the token is located in the source //
const std::streamsize line;
// Index on the line where the token starts //
const std::streamsize index;
// The length of the token on the line, may be different to the length of contents //
const std::streamsize length;
};
// Lexer function to take in a file and output a vector of tokens
// Lexer function to take in a file and output a vector of tokens //
const std::vector<Token> LexicalAnalyze(std::ifstream& src, std::ofstream* log);
}