Pasha/LXC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tabs -> Spaces

Browse Source
This commit is contained in:
Pasha Bibko
2025-07-21 17:39:43 +01:00
parent 24fde1b770
commit 5bfeb75536
10 changed files with 514 additions and 515 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
Common/File.h
View File

@@ -5,98 +5,98 @@
namespace LXC::Util namespace LXC::Util
{ {
// Error returned when Util::ReadFile runs into errors // // Error returned when Util::ReadFile runs into errors //
struct FileReadError final struct FileReadError final
{ {
// Different reasons why the error can occur // // Different reasons why the error can occur //
enum Reason enum Reason
{ {
FileNotFound, FileNotFound,
PermissionDenied, PermissionDenied,
NotAFile NotAFile
}; };
// Constructor to pass arguments to the struct // // Constructor to pass arguments to the struct //
FileReadError(const std::filesystem::path& _path, Reason _reason) FileReadError(const std::filesystem::path& _path, Reason _reason)
: path(_path), reason(_reason) : path(_path), reason(_reason)
{} {}
// Error information // // Error information //
const std::filesystem::path path; const std::filesystem::path path;
const Reason reason; const Reason reason;
// Turns the error into a c-string // // Turns the error into a c-string //
inline static const char* const ReasonStr(Reason reason) inline static const char* const ReasonStr(Reason reason)
{ {
static const char* reasons[] = static const char* reasons[] =
{ {
"File cannot be found", "File cannot be found",
"File reading permissions are denied", "File reading permissions are denied",
"Not a file" "Not a file"
}; };
return reasons[reason]; return reasons[reason];
} }
}; };
// Util function to read a file as quick as possible with error handling // // Util function to read a file as quick as possible with error handling //
inline ReturnVal<std::string, FileReadError> ReadFile(const std::filesystem::path& filepath) inline ReturnVal<std::string, FileReadError> ReadFile(const std::filesystem::path& filepath)
{ {
// Checks the file exists // // Checks the file exists //
if (!std::filesystem::exists(filepath)) if (!std::filesystem::exists(filepath))
return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::FileNotFound); return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::FileNotFound);
// Checks it is a regular file // // Checks it is a regular file //
if (!std::filesystem::is_regular_file(filepath)) if (!std::filesystem::is_regular_file(filepath))
return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::NotAFile); return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::NotAFile);
// Checks it can open the file // // Checks it can open the file //
std::ifstream file(filepath, std::ios::binary | std::ios::ate); std::ifstream file(filepath, std::ios::binary | std::ios::ate);
if (!file) if (!file)
return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::PermissionDenied); return FunctionFail<FileReadError>(std::filesystem::absolute(filepath), FileReadError::PermissionDenied);
// Copies the file to the output string // // Copies the file to the output string //
const std::streamsize len = file.tellg(); const std::streamsize len = file.tellg();
file.seekg(0, std::ios::beg); file.seekg(0, std::ios::beg);
std::string contents(len, '\0'); std::string contents(len, '\0');
file.read(&contents[0], len); file.read(&contents[0], len);
return contents; return contents;
} }
// Struct to hold a position within a file // // Struct to hold a position within a file //
struct FileLocation struct FileLocation
{ {
unsigned short col; unsigned short col;
unsigned short line; unsigned short line;
}; };
// Finds the location of a given index within a file // // Finds the location of a given index within a file //
inline bool GetFileLocationAtIndex(FileLocation& location, const std::string& file, __int32 index) inline bool GetFileLocationAtIndex(FileLocation& location, const std::string& file, __int32 index)
{ {
// Resets location // // Resets location //
location.line = 1; location.line = 1;
location.col = 1; location.col = 1;
// Returns false if outside the bounds // // Returns false if outside the bounds //
if (index < 0 || index > file.length()) if (index < 0 || index > file.length())
return false; return false;
// Finds the location // // Finds the location //
__int32 localIndex = 0; __int32 localIndex = 0;
while (localIndex != index) while (localIndex != index)
{ {
if (file[localIndex] == '\n') if (file[localIndex] == '\n')
{ {
location.line += 1; location.line += 1;
location.col = 0; location.col = 0;
} }
location.col++; location.col++;
localIndex++; localIndex++;
} }
return true; return true;
} }
} }

220
Common/IO.h
View File

@@ -8,137 +8,137 @@
namespace LXC::Util namespace LXC::Util
{ {
// Enum to translate to the Win32 code for the colors // // Enum to translate to the Win32 code for the colors //
enum Color : WORD enum Color : WORD
{ {
DEFAULT = 0x07, DEFAULT = 0x07,
BLACK = 0x00, BLACK = 0x00,
BLUE = 0x01, BLUE = 0x01,
GREEN = 0x02, GREEN = 0x02,
AQUA = 0x03, AQUA = 0x03,
RED = 0x04, RED = 0x04,
PURPLE = 0x05, PURPLE = 0x05,
YELLOW = 0x06, YELLOW = 0x06,
LIGHT_GRAY = 0x07, LIGHT_GRAY = 0x07,
LIGHT_BLUE = 0x09, LIGHT_BLUE = 0x09,
LIGHT_GREEN = 0x0a, LIGHT_GREEN = 0x0a,
LIGHT_AQUA = 0x0b, LIGHT_AQUA = 0x0b,
LIGHT_RED = 0x0c, LIGHT_RED = 0x0c,
LIGHT_PURPLE = 0x0d, LIGHT_PURPLE = 0x0d,
LIGHT_YELLOW = 0x0e, LIGHT_YELLOW = 0x0e,
WHITE = 0x0f WHITE = 0x0f
}; };
} }
namespace LXC::Internal namespace LXC::Internal
{ {
// Checks if a type can be outputted to std::ostream // // Checks if a type can be outputted to std::ostream //
template<typename T> concept Logable = requires(std::ostream& os, T t) template<typename T> concept Logable = requires(std::ostream& os, T t)
{ {
// I have no idea what this part does at all // // I have no idea what this part does at all //
{ os << t } -> std::same_as<std::ostream&>; { os << t } -> std::same_as<std::ostream&>;
}; };
// Checks if a list of types can be outputted to std::ostream // // Checks if a list of types can be outputted to std::ostream //
template<typename... Args> concept AllLogable = (Logable<Args> && ...); template<typename... Args> concept AllLogable = (Logable<Args> && ...);
// Checks if the type has a custom log method // // Checks if the type has a custom log method //
template <typename T> concept HasLogStrFunc = requires(T obj) template <typename T> concept HasLogStrFunc = requires(T obj)
{ {
// Checks the type has a function LogStr that returns a string // // Checks the type has a function LogStr that returns a string //
{ obj.LogStr() } -> std::same_as<std::string>; { obj.LogStr() } -> std::same_as<std::string>;
}; };
// Returns a reference to the log file // // Returns a reference to the log file //
inline std::ofstream& Log() inline std::ofstream& Log()
{ {
static std::ofstream sLog; static std::ofstream sLog;
return sLog; return sLog;
} }
// Base function for all derived Print() functions // // Base function for all derived Print() functions //
template<Util::Color col, bool newLine, typename... Args> template<Util::Color col, bool newLine, typename... Args>
inline void WriteImpl(std::ostream& os, Args&&... args) inline void WriteImpl(std::ostream& os, Args&&... args)
{ {
static HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE); static HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
if constexpr(col != Util::Color::DEFAULT) if constexpr(col != Util::Color::DEFAULT)
SetConsoleTextAttribute(hConsole, static_cast<WORD>(col)); SetConsoleTextAttribute(hConsole, static_cast<WORD>(col));
(os << ... << std::forward<Args>(args)); (os << ... << std::forward<Args>(args));
if constexpr (newLine) if constexpr (newLine)
os << std::endl; os << std::endl;
if constexpr (col != Util::Color::DEFAULT) if constexpr (col != Util::Color::DEFAULT)
SetConsoleTextAttribute(hConsole, static_cast<WORD>(Util::Color::DEFAULT)); SetConsoleTextAttribute(hConsole, static_cast<WORD>(Util::Color::DEFAULT));
} }
} }
namespace LXC::Util namespace LXC::Util
{ {
// Prints arguments to the console with the given color // // Prints arguments to the console with the given color //
template<Color col, typename... Args> template<Color col, typename... Args>
requires Internal::AllLogable<Args...> requires Internal::AllLogable<Args...>
inline void PrintAs(Args&&... args) inline void PrintAs(Args&&... args)
{ {
Internal::WriteImpl<col, false>(std::cout, std::forward<Args>(args)...); Internal::WriteImpl<col, false>(std::cout, std::forward<Args>(args)...);
} }
// Prints arguments to the console // // Prints arguments to the console //
template<typename... Args> template<typename... Args>
requires Internal::AllLogable<Args...> requires Internal::AllLogable<Args...>
inline void Print(Args&&... args) inline void Print(Args&&... args)
{ {
Internal::WriteImpl<Util::Color::DEFAULT, false>(std::cout, std::forward<Args>(args)...); Internal::WriteImpl<Util::Color::DEFAULT, false>(std::cout, std::forward<Args>(args)...);
} }
// Prints arguments to the console with a new-line character at the end // // Prints arguments to the console with a new-line character at the end //
template<typename... Args> template<typename... Args>
requires Internal::AllLogable<Args...> requires Internal::AllLogable<Args...>
inline void PrintLn(Args&&... args) inline void PrintLn(Args&&... args)
{ {
Internal::WriteImpl<Util::Color::DEFAULT, true>(std::cout, std::forward<Args>(args)...); Internal::WriteImpl<Util::Color::DEFAULT, true>(std::cout, std::forward<Args>(args)...);
} }
// Logs all the arguments to the file (automatically flushes) // // Logs all the arguments to the file (automatically flushes) //
template<typename... Args> template<typename... Args>
requires Internal::AllLogable<Args...> && (sizeof...(Args) > 1) requires Internal::AllLogable<Args...> && (sizeof...(Args) > 1)
inline void Log(Args&&... args) inline void Log(Args&&... args)
{ {
std::ofstream& log = Internal::Log(); std::ofstream& log = Internal::Log();
if (log.is_open()) _UNLIKELY if (log.is_open()) _UNLIKELY
Internal::WriteImpl<Util::Color::DEFAULT, true>(log, R"([LXC] ")", std::forward<Args>(args)..., '"'); Internal::WriteImpl<Util::Color::DEFAULT, true>(log, R"([LXC] ")", std::forward<Args>(args)..., '"');
} }
// Logs a singular argument to the log, calls Log() if it can on the object // // Logs a singular argument to the log, calls Log() if it can on the object //
template<typename T> template<typename T>
requires Internal::HasLogStrFunc<T> || Internal::Logable<T> requires Internal::HasLogStrFunc<T> || Internal::Logable<T>
inline void Log(T arg) inline void Log(T arg)
{ {
std::ofstream& log = Internal::Log(); std::ofstream& log = Internal::Log();
if (log.is_open()) _UNLIKELY if (log.is_open()) _UNLIKELY
{ {
if constexpr (Internal::HasLogStrFunc<T>) if constexpr (Internal::HasLogStrFunc<T>)
Internal::WriteImpl<Util::Color::DEFAULT, true>(log, "[LXC] ", '{', arg.LogStr(), '}'); Internal::WriteImpl<Util::Color::DEFAULT, true>(log, "[LXC] ", '{', arg.LogStr(), '}');
else else
Internal::WriteImpl<Util::Color::DEFAULT, true>(log, "[LXC] ", '"', arg, '"'); Internal::WriteImpl<Util::Color::DEFAULT, true>(log, "[LXC] ", '"', arg, '"');
} }
} }
// Intitalises the log with the given file name // // Intitalises the log with the given file name //
inline void CreateLog(const std::filesystem::path& path) inline void CreateLog(const std::filesystem::path& path)
{ {
// Opens the log file with the given path // // Opens the log file with the given path //
std::ofstream& log = Internal::Log(); std::ofstream& log = Internal::Log();
log.open(path); log.open(path);
// Assigns a function to close the log file on program exit // // Assigns a function to close the log file on program exit //
std::atexit([]() std::atexit([]()
{ {
std::ofstream& log = Internal::Log(); std::ofstream& log = Internal::Log();
log.close(); log.close();
}); });
} }
} }

8
Common/OS.h
View File

@@ -2,9 +2,9 @@
// Platform specific includes // // Platform specific includes //
#ifdef _WIN32 #ifdef _WIN32
#define NOMINMAX #define NOMINMAX
#define WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN
#include <Windows.h> #include <Windows.h>
#else #else
#error "Code is currently only supported on Win32" #error "Code is currently only supported on Win32"
#endif // _WIN32 #endif // _WIN32

166
Common/Result.h
View File

@@ -7,110 +7,110 @@
namespace LXC::Util namespace LXC::Util
{ {
// Util function to stop/ the program // // Util function to stop/ the program //
inline void Stop() inline void Stop()
{ {
// Only checks for a debugger when compiled in Debug mode // // Only checks for a debugger when compiled in Debug mode //
#ifdef _DEBUG #ifdef _DEBUG
// Triggers a breakpoint so the debugger can work out where the program exits // // Triggers a breakpoint so the debugger can work out where the program exits //
if (IsDebuggerPresent()) if (IsDebuggerPresent())
DebugBreak(); DebugBreak();
#endif // _DEBUG #endif // _DEBUG
// Force exits the program // // Force exits the program //
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
} }
// Custom version of std::unexpected // // Custom version of std::unexpected //
template<typename ErrorType> struct FunctionFail final template<typename ErrorType> struct FunctionFail final
{ {
// Basic constructor to copy the error across // // Basic constructor to copy the error across //
explicit FunctionFail(ErrorType _err) : error(_err) explicit FunctionFail(ErrorType _err) : error(_err)
{ {
// Only checks for a debugger when compiled in Debug mode // // Only checks for a debugger when compiled in Debug mode //
#ifdef _DEBUG #ifdef _DEBUG
// Triggers a breakpoint when a debugger is attached as a function has failed // // Triggers a breakpoint when a debugger is attached as a function has failed //
if (IsDebuggerPresent()) if (IsDebuggerPresent())
DebugBreak(); DebugBreak();
#endif // _DEBUG #endif // _DEBUG
} }
// Constructs the FunctionFail with the error itself // // Constructs the FunctionFail with the error itself //
template<typename... Args> requires std::constructible_from<ErrorType, Args...> template<typename... Args> requires std::constructible_from<ErrorType, Args...>
explicit FunctionFail(Args&&... args) explicit FunctionFail(Args&&... args)
: error(std::forward<Args>(args)...) : error(std::forward<Args>(args)...)
{ {
// Only checks for a debugger when compiled in Debug mode // // Only checks for a debugger when compiled in Debug mode //
#ifdef _DEBUG #ifdef _DEBUG
// Triggers a breakpoint when a debugger is attached as a function has failed // // Triggers a breakpoint when a debugger is attached as a function has failed //
if (IsDebuggerPresent()) if (IsDebuggerPresent())
DebugBreak(); DebugBreak();
#endif // _DEBUG #endif // _DEBUG
} }
const ErrorType error; const ErrorType error;
}; };
// Custom version of std::expected // // Custom version of std::expected //
template<typename ResultType, typename ErrorType> template<typename ResultType, typename ErrorType>
requires (!std::same_as<ResultType, bool>) // ResultType being bool causes issues with operator overloads requires (!std::same_as<ResultType, bool>) // ResultType being bool causes issues with operator overloads
class ReturnVal final class ReturnVal final
{ {
public: public:
// Constructor for function sucess // // Constructor for function sucess //
ReturnVal(ResultType result) ReturnVal(ResultType result)
: m_Result(result), m_FunctionFailed(false) : m_Result(result), m_FunctionFailed(false)
{} {}
// Constructor for function fail // // Constructor for function fail //
ReturnVal(FunctionFail<ErrorType> error) ReturnVal(FunctionFail<ErrorType> error)
: m_Error(error.error), m_FunctionFailed(true) : m_Error(error.error), m_FunctionFailed(true)
{} {}
// Destructor // // Destructor //
~ReturnVal() {} ~ReturnVal() {}
// Different getters of the class // // Different getters of the class //
inline bool Failed() const { return m_FunctionFailed; } inline bool Failed() const { return m_FunctionFailed; }
inline bool Suceeded() const { return !m_FunctionFailed; } inline bool Suceeded() const { return !m_FunctionFailed; }
inline ResultType& Result() inline ResultType& Result()
{ {
if (Suceeded()) _LIKELY if (Suceeded()) _LIKELY
return m_Result; return m_Result;
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
} }
inline ErrorType& Error() inline ErrorType& Error()
{ {
if (Failed()) _LIKELY if (Failed()) _LIKELY
return m_Error; return m_Error;
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
} }
// Operator overloads // // Operator overloads //
operator bool() const { return !m_FunctionFailed; } operator bool() const { return !m_FunctionFailed; }
operator ResultType() { return Result(); } operator ResultType() { return Result(); }
private: private:
// Union to hold either the result or the error // // Union to hold either the result or the error //
union union
{ {
ResultType m_Result; ResultType m_Result;
ErrorType m_Error; ErrorType m_Error;
}; };
// Tracks what item is currently in the union // // Tracks what item is currently in the union //
bool m_FunctionFailed; bool m_FunctionFailed;
}; };
} }

92
LXC/LXC.cpp
View File

@@ -4,64 +4,64 @@
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
using namespace LXC; using namespace LXC;
// Creates the debug log // // Creates the debug log //
Util::CreateLog("LXC.log"); Util::CreateLog("LXC.log");
std::filesystem::path src = "example/example.lx"; std::filesystem::path src = "example/example.lx";
// Reads the given file to a string // // Reads the given file to a string //
Util::ReturnVal fileContents = Util::ReadFile(src); Util::ReturnVal fileContents = Util::ReadFile(src);
if (fileContents.Failed()) _UNLIKELY if (fileContents.Failed()) _UNLIKELY
{ {
// Stores the error for easier access // // Stores the error for easier access //
Util::FileReadError& err = fileContents.Error(); Util::FileReadError& err = fileContents.Error();
// Prints the error to the console // // Prints the error to the console //
Util::PrintAs<Util::WHITE>("[LXC]"); Util::PrintAs<Util::WHITE>("[LXC]");
Util::PrintAs<Util::LIGHT_RED>(" Error: "); Util::PrintAs<Util::LIGHT_RED>(" Error: ");
Util::PrintLn(Util::FileReadError::ReasonStr(err.reason), " [", std::filesystem::absolute(err.path), ']'); Util::PrintLn(Util::FileReadError::ReasonStr(err.reason), " [", std::filesystem::absolute(err.path), ']');
Util::Log("Opening source file failed. Stopping program."); Util::Log("Opening source file failed. Stopping program.");
Util::Stop(); Util::Stop();
} }
// Turns the file contents into a vector of tokens // // Turns the file contents into a vector of tokens //
Util::ReturnVal tokens = Lexer::TokenizeFile(fileContents); Util::ReturnVal tokens = Lexer::TokenizeFile(fileContents);
if (tokens.Failed()) _UNLIKELY if (tokens.Failed()) _UNLIKELY
{ {
// Stores the error for easier access // // Stores the error for easier access //
Lexer::LexerError& err = tokens.Error(); Lexer::LexerError& err = tokens.Error();
// Finds the file location of the error // // Finds the file location of the error //
Util::FileLocation location; Util::FileLocation location;
Util::GetFileLocationAtIndex(location, fileContents, err.index); Util::GetFileLocationAtIndex(location, fileContents, err.index);
// Prints the error to the console // // Prints the error to the console //
Util::PrintAs<Util::WHITE>("[LXC] "); Util::PrintAs<Util::WHITE>("[LXC] ");
Util::Print(src.filename().string(), '(', location.line, ',', location.col, ')'); Util::Print(src.filename().string(), '(', location.line, ',', location.col, ')');
Util::PrintAs<Util::LIGHT_RED>(" Error: "); Util::PrintAs<Util::LIGHT_RED>(" Error: ");
Util::Print(Lexer::LexerError::ReasonStr(err.reason)); Util::Print(Lexer::LexerError::ReasonStr(err.reason));
if (err.reason == Lexer::LexerError::InvalidCharacter) if (err.reason == Lexer::LexerError::InvalidCharacter)
Util::PrintLn(": {", fileContents.Result()[err.index], '}'); Util::PrintLn(": {", fileContents.Result()[err.index], '}');
if (err.reason == Lexer::LexerError::UnknownSymbolOrOperand) if (err.reason == Lexer::LexerError::UnknownSymbolOrOperand)
Util::PrintLn(": {", err.info, '}'); Util::PrintLn(": {", err.info, '}');
else else
Util::PrintLn(); Util::PrintLn();
Util::Log("Error occured in Lexer: ", Lexer::LexerError::ReasonStr(err.reason)); Util::Log("Error occured in Lexer: ", Lexer::LexerError::ReasonStr(err.reason));
Util::Stop(); Util::Stop();
} }
// Prints all of the tokens to the log // // Prints all of the tokens to the log //
for (const auto& token : tokens.Result()) for (const auto& token : tokens.Result())
{ {
Util::Log(token); Util::Log(token);
} }
return 0; return 0;
} }

4
Lexer/CMakeLists.txt
View File

@@ -4,8 +4,8 @@ file (GLOB LexerSources src/*.cpp inc/*.h)
add_library(Lexer STATIC ${LexerSources}) add_library(Lexer STATIC ${LexerSources})
target_include_directories ( target_include_directories (
Lexer PUBLIC Lexer PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}/inc ${CMAKE_CURRENT_SOURCE_DIR}/inc
) )
# Creates the precompiled header for the binary # # Creates the precompiled header for the binary #

88
Lexer/inc/Lexer.h
View File

@@ -4,57 +4,57 @@
namespace LXC::Lexer namespace LXC::Lexer
{ {
struct LexerContext final struct LexerContext final
{ {
// Constructor to set the information of the context // // Constructor to set the information of the context //
LexerContext(const std::string& _source); LexerContext(const std::string& _source);
// Trackers for the Lexer itself // // Trackers for the Lexer itself //
const std::string& source; const std::string& source;
__int32 index; __int32 index;
LexerOutput out; LexerOutput out;
const __int32 len; const __int32 len;
// Trackers for where the Lexer is within the user version of source // // Trackers for where the Lexer is within the user version of source //
unsigned short column; unsigned short column;
unsigned short line; unsigned short line;
}; };
struct LexerError final struct LexerError final
{ {
// Different reasons why the Lexer can fail // // Different reasons why the Lexer can fail //
enum Reason enum Reason
{ {
InvalidCharacter, InvalidCharacter,
UnterminatedStringLiteral, UnterminatedStringLiteral,
UnknownSymbolOrOperand UnknownSymbolOrOperand
}; };
// Constructor to pass arguments through to the struct // // Constructor to pass arguments through to the struct //
LexerError(Reason _reason, __int32 errorIndex, std::string _info = "") LexerError(Reason _reason, __int32 errorIndex, std::string _info = "")
: reason(_reason), index(errorIndex), info(_info) : reason(_reason), index(errorIndex), info(_info)
{} {}
// Turns the error into a c-string // // Turns the error into a c-string //
inline static const char* const ReasonStr(Reason reason) inline static const char* const ReasonStr(Reason reason)
{ {
static const char* reasons[] = static const char* reasons[] =
{ {
"Invalid character found in source", "Invalid character found in source",
"Unterminated string literal in source", "Unterminated string literal in source",
"Unknown symbol or operand in source" "Unknown symbol or operand in source"
}; };
return reasons[reason]; return reasons[reason];
} }
// Error information // // Error information //
const Reason reason; const Reason reason;
const __int32 index; const __int32 index;
const std::string info; const std::string info;
}; };
// Turns a file into a vector of tokens // // Turns a file into a vector of tokens //
Util::ReturnVal<LexerOutput, LexerError> TokenizeFile(const std::string& fileContents); Util::ReturnVal<LexerOutput, LexerError> TokenizeFile(const std::string& fileContents);
} }

156
Lexer/inc/Token.h
View File

@@ -4,112 +4,112 @@
namespace LXC::Lexer namespace LXC::Lexer
{ {
namespace TokenClass namespace TokenClass
{ {
// Bitmask for different token classes // // Bitmask for different token classes //
enum ClassMask : unsigned short enum ClassMask : unsigned short
{ {
// Mathematical and logic operators // // Mathematical and logic operators //
Operator = 1 << (1 + 8), Operator = 1 << (1 + 8),
// Special words defined by the compiler // // Special words defined by the compiler //
Keyword = 1 << (2 + 8), Keyword = 1 << (2 + 8),
// Words such as literals and identifiers // // Words such as literals and identifiers //
UserDefined = 1 << (3 + 8), UserDefined = 1 << (3 + 8),
// Symbols in the source like (? , . ! <) // // Symbols in the source like (? , . ! <) //
Symbols = 1 << (4 + 8), Symbols = 1 << (4 + 8),
// Tokens not defined by previous classes // // Tokens not defined by previous classes //
Misc = 1 << (5 + 8) Misc = 1 << (5 + 8)
}; };
}; };
struct LexerContext; struct LexerContext;
// Data type for storing the output of the lexer // // Data type for storing the output of the lexer //
class Token final class Token final
{ {
public: public:
// Enum of token type organised by their token class // // Enum of token type organised by their token class //
enum TokenType : unsigned short enum TokenType : unsigned short
{ {
// === Operators === // // === Operators === //
Add = TokenClass::Operator, Add = TokenClass::Operator,
Sub, Sub,
Mul, Mul,
Div, Div,
Mod, Mod,
// === Keywords === // // === Keywords === //
For = TokenClass::Keyword, For = TokenClass::Keyword,
While, While,
If, If,
ElseIf, ElseIf,
Else, Else,
Return, Return,
// === User defined === // // === User defined === //
StringLiteral = TokenClass::UserDefined, StringLiteral = TokenClass::UserDefined,
NumLiteral, NumLiteral,
Identifier, Identifier,
// === Symbols === // // === Symbols === //
Assign = TokenClass::Symbols, Assign = TokenClass::Symbols,
CloseBracket, CloseBracket,
OpenBracket, OpenBracket,
CloseBrace, CloseBrace,
OpenBrace, OpenBrace,
CloseParen, CloseParen,
OpenParen, OpenParen,
Comma, Comma,
// === Misc === // // === Misc === //
End_of_file = TokenClass::Misc, End_of_file = TokenClass::Misc,
UNDEFINED = 65535 // Invalid token type (max number) UNDEFINED = 65535 // Invalid token type (max number)
}; };
// Util function calculating wether a token is of a given class // // Util function calculating wether a token is of a given class //
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; } template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; }
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; } template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; }
// Constructor to set the data of the token for more complex token types // // Constructor to set the data of the token for more complex token types //
Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type); Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type);
// Deconstructor to clean up the allocated memory // // Deconstructor to clean up the allocated memory //
~Token(); ~Token();
// Getters for the c-string to stop it being reassigned (or deleted) // // Getters for the c-string to stop it being reassigned (or deleted) //
inline const char* const Str() const { return contents; } inline const char* const Str() const { return contents; }
// Outputs all the relevant infomration in a string for logging purposes // // Outputs all the relevant infomration in a string for logging purposes //
std::string LogStr() const; std::string LogStr() const;
// The type of the token // // The type of the token //
const TokenType type; const TokenType type;
// The length of the token // // The length of the token //
const unsigned short length; const unsigned short length;
// Start index of the token // // Start index of the token //
const unsigned __int32 index; const unsigned __int32 index;
private: private:
// The data of the token // // The data of the token //
char* contents; char* contents;
}; };
// Typedef for the output type of how the Lexer outputs // // Typedef for the output type of how the Lexer outputs //
typedef std::vector<Token> LexerOutput; typedef std::vector<Token> LexerOutput;
} }

136
Lexer/src/Token.cpp
View File

@@ -7,84 +7,84 @@
namespace LXC::Lexer namespace LXC::Lexer
{ {
// Constructor to assign the members of the token class // // Constructor to assign the members of the token class //
Token::Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type) : Token::Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type) :
type(_type), length(len), index(start), contents(nullptr) type(_type), length(len), index(start), contents(nullptr)
{ {
// Only user defined class tokens need to store c-string // // Only user defined class tokens need to store c-string //
if (Token::IsTypeClass<TokenClass::UserDefined>(type)) if (Token::IsTypeClass<TokenClass::UserDefined>(type))
{ {
// Copies the memory to a c-string // // Copies the memory to a c-string //
contents = new char[len + 1]; // +1 for null terminator contents = new char[len + 1]; // +1 for null terminator
std::memcpy(contents, ctx.source.data() + start, len); std::memcpy(contents, ctx.source.data() + start, len);
contents[len] = '\0'; contents[len] = '\0';
} }
} }
// Destructor to clean up the memory of the token that can be allocated // // Destructor to clean up the memory of the token that can be allocated //
Token::~Token() Token::~Token()
{ {
// Frees any allocated memory // // Frees any allocated memory //
//if (contents != nullptr) //if (contents != nullptr)
// delete[] contents; // delete[] contents;
contents = nullptr; contents = nullptr;
} }
// Helper macro for converting type to string // // Helper macro for converting type to string //
#define TOKEN_TYPE_CASE(type) case type: return #type; #define TOKEN_TYPE_CASE(type) case type: return #type;
static constexpr const char* TokenTypeToCStr(Token::TokenType type) static constexpr const char* TokenTypeToCStr(Token::TokenType type)
{ {
switch (type) switch (type)
{ {
// All the different types of tokens // // All the different types of tokens //
TOKEN_TYPE_CASE(Token::Add); TOKEN_TYPE_CASE(Token::Add);
TOKEN_TYPE_CASE(Token::Sub); TOKEN_TYPE_CASE(Token::Sub);
TOKEN_TYPE_CASE(Token::Mul); TOKEN_TYPE_CASE(Token::Mul);
TOKEN_TYPE_CASE(Token::Div); TOKEN_TYPE_CASE(Token::Div);
TOKEN_TYPE_CASE(Token::Mod); TOKEN_TYPE_CASE(Token::Mod);
TOKEN_TYPE_CASE(Token::For); TOKEN_TYPE_CASE(Token::For);
TOKEN_TYPE_CASE(Token::While); TOKEN_TYPE_CASE(Token::While);
TOKEN_TYPE_CASE(Token::If); TOKEN_TYPE_CASE(Token::If);
TOKEN_TYPE_CASE(Token::ElseIf); TOKEN_TYPE_CASE(Token::ElseIf);
TOKEN_TYPE_CASE(Token::Else); TOKEN_TYPE_CASE(Token::Else);
TOKEN_TYPE_CASE(Token::Return); TOKEN_TYPE_CASE(Token::Return);
TOKEN_TYPE_CASE(Token::StringLiteral); TOKEN_TYPE_CASE(Token::StringLiteral);
TOKEN_TYPE_CASE(Token::NumLiteral); TOKEN_TYPE_CASE(Token::NumLiteral);
TOKEN_TYPE_CASE(Token::Identifier); TOKEN_TYPE_CASE(Token::Identifier);
TOKEN_TYPE_CASE(Token::Assign); TOKEN_TYPE_CASE(Token::Assign);
TOKEN_TYPE_CASE(Token::CloseBracket); TOKEN_TYPE_CASE(Token::CloseBracket);
TOKEN_TYPE_CASE(Token::OpenBracket); TOKEN_TYPE_CASE(Token::OpenBracket);
TOKEN_TYPE_CASE(Token::CloseBrace); TOKEN_TYPE_CASE(Token::CloseBrace);
TOKEN_TYPE_CASE(Token::OpenBrace); TOKEN_TYPE_CASE(Token::OpenBrace);
TOKEN_TYPE_CASE(Token::CloseParen); TOKEN_TYPE_CASE(Token::CloseParen);
TOKEN_TYPE_CASE(Token::OpenParen); TOKEN_TYPE_CASE(Token::OpenParen);
TOKEN_TYPE_CASE(Token::Comma); TOKEN_TYPE_CASE(Token::Comma);
TOKEN_TYPE_CASE(Token::End_of_file); TOKEN_TYPE_CASE(Token::End_of_file);
TOKEN_TYPE_CASE(Token::UNDEFINED); TOKEN_TYPE_CASE(Token::UNDEFINED);
// When the case has not been defined yet // // When the case has not been defined yet //
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
} }
std::string LXC::Lexer::Token::LogStr() const std::string LXC::Lexer::Token::LogStr() const
{ {
// Output stream to log to // // Output stream to log to //
std::ostringstream os; std::ostringstream os;
os << std::setw(25) << std::left << TokenTypeToCStr(type) << " | "; os << std::setw(25) << std::left << TokenTypeToCStr(type) << " | ";
if (contents != nullptr) if (contents != nullptr)
os << std::setw(25) << std::left << std::string('"' + std::string(contents) + '"'); os << std::setw(25) << std::left << std::string('"' + std::string(contents) + '"');
else else
os << std::setw(25) << std::left << "EMPTY"; os << std::setw(25) << std::left << "EMPTY";
return os.str(); return os.str();
} }
} }

1
README.md
View File

@@ -1,2 +1 @@
# LXC # LXC