Pasha/MIRROR-LX-OriginalRepo
1
0
Fork 0
mirror of https://github.com/PashaBibko/LX.git synced 2026-04-03 17:39:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed Unexpected Token crashing the process

Browse Source
This commit is contained in:
Pasha Bibko
2025-05-07 16:53:10 +01:00
parent 4e78a9f6ae
commit 6783564f10
10 changed files with 53 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Common/Common.vcxproj
View File

@@ -151,14 +151,14 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="inc\Console.h" />
<ClInclude Include="inc\IO.h" />
<ClInclude Include="inc\Error.h" />
<ClInclude Include="inc\Logger.h" />
<ClInclude Include="inc\ThrowIf.h" />
<ClInclude Include="LX-Common.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\Console.cpp" />
<ClCompile Include="src\IO.cpp" />
<ClCompile Include="src\Error.cpp" />
<ClCompile Include="src\Logger.cpp" />
<ClCompile Include="src\pch.cpp">

4
Common/Common.vcxproj.filters
View File

@@ -14,7 +14,7 @@
<ClInclude Include="LX-Common.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="inc\Console.h">
<ClInclude Include="inc\IO.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="inc\Error.h">
@@ -34,7 +34,7 @@
<ClCompile Include="src\Logger.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\Console.cpp">
<ClCompile Include="src\IO.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\Error.cpp">

2
Common/LX-Common.h
View File

@@ -65,7 +65,7 @@
// Includes the rest of common //
#include <inc/Console.h>
#include <inc/Error.h>
#include <inc/Logger.h>
#include <inc/ThrowIf.h>
#include <inc/IO.h>

19
Common/inc/Console.h → Common/inc/IO.h
View File

@@ -23,6 +23,25 @@ namespace LX
// Prints a string to std::cout with a certain color using Win32 API //
extern "C" void COMMON_API PrintStringAsColor(const std::string& str, Color c);
inline std::string ReadFileToString(const std::filesystem::path& path, const std::string errorName = "input file path")
{
// Verifies the file path is valid //
ThrowIf<LX::InvalidFilePath>(std::filesystem::exists(path) == false, errorName, path);
// Opens the file //
std::ifstream file(path, std::ios::binary | std::ios::ate); // Opens in binary and at the end (microptimsation)
ThrowIf<LX::InvalidFilePath>(file.is_open() == false, errorName, path);
// Stores the length of the string and goes back to the beginning //
const std::streamsize len = file.tellg(); // tellg returns length because it was opened at the end
file.seekg(0, std::ios::beg);
// Transfers the file contents to the output //
std::string contents(len, '\0'); // Allocates an empty string which is the size of the file
file.read(&contents[0], len);
return contents;
}
// Util function for getting a line of the source at a given index (used for errors) //
inline std::string GetLineAtIndexOf(const std::string& src, const std::streamsize index) // <- Has to be inline because of C++ types
{

0
Common/src/Console.cpp → Common/src/IO.cpp
View File

7
IR-Generator/inc/Parser.h
View File

@@ -71,11 +71,10 @@ namespace LX
{
GENERATE_LX_ERROR_REQUIRED_FUNCTION_DECLARATIONS;
UnexpectedToken(Token::TokenType _expected, std::string _override, Token _got);
UnexpectedToken(Token::TokenType _expected, std::string _override, Token _got, const std::filesystem::path& _file);
//
static std::string* s_Source;
static std::filesystem::path* s_SourceFile;
const std::filesystem::path file;
// The token type that should be there //
Token::TokenType expected;
@@ -143,7 +142,7 @@ namespace LX
};
// Turns the tokens of a file into it's abstract syntax tree equivalent //
FileAST TurnTokensIntoAbstractSyntaxTree(std::vector<Token>& tokens);
FileAST TurnTokensIntoAbstractSyntaxTree(std::vector<Token>& tokens, const std::filesystem::path& path);
// Turns an abstract binary tree into LLVM intermediate representation //
void GenerateIR(FileAST& ast, const std::string& name, const std::filesystem::path& IRPath);

4
IR-Generator/src/Generator.cpp
View File

@@ -29,7 +29,7 @@ extern "C" int __declspec(dllexport) GenIR(const char* a_inpPath, const char* a_
std::vector<LX::Token>tokens = LX::LexicalAnalyze(inpPath);
// Turns the tokens into an AST //
LX::FileAST AST = LX::TurnTokensIntoAbstractSyntaxTree(tokens);
LX::FileAST AST = LX::TurnTokensIntoAbstractSyntaxTree(tokens, inpPath);
// Turns the AST into LLVM IR //
LX::GenerateIR(AST, inpPath.filename().string(), outPath);
@@ -72,7 +72,7 @@ extern "C" int __declspec(dllexport) GenIR(const char* a_inpPath, const char* a_
std::cout << "An error occured. Please report this on the github page.\n" << std::endl;
std::cout << e.what() << std::endl;
// Exit code -1 means an undefined error //
// Exit code -1 means an undefined error // But this isn't undefined and neither is LX::RuntimeError?
return -1;
}

19
Lexer/src/Lexer.cpp
View File

@@ -175,25 +175,6 @@ namespace LX
}
}
static std::string ReadFileToString(const std::filesystem::path& path)
{
// Verifies the file path is valid //
ThrowIf<LX::InvalidFilePath>(std::filesystem::exists(path) == false, "input file path", path);
// Opens the file //
std::ifstream file(path, std::ios::binary | std::ios::ate); // Opens in binary and at the end (microptimsation)
ThrowIf<LX::InvalidFilePath>(file.is_open() == false, "input file path", path);
// Stores the length of the string and goes back to the beginning //
const std::streamsize len = file.tellg(); // tellg returns length because it was opened at the end
file.seekg(0, std::ios::beg);
// Transfers the file contents to the output //
std::string contents(len, '\0'); // Allocates an empty string which is the size of the file
file.read(&contents[0], len);
return contents;
}
const std::vector<Token> LX::LexicalAnalyze(const std::filesystem::path& path)
{
// Logs that the file is being read //

34
Parser/src/Parser.cpp
View File

@@ -6,9 +6,6 @@
namespace LX
{
std::string* UnexpectedToken::s_Source = nullptr;
std::filesystem::path* UnexpectedToken::s_SourceFile = nullptr;
// Util function for working out if a token is a two sided operator //
static bool IsTwoSidedOperator(Token::TokenType t)
{
@@ -29,10 +26,13 @@ namespace LX
struct Parser
{
// Passes constructor args to members //
Parser(std::vector<Token>& _tokens)
: tokens(_tokens), index(0), len(_tokens.size()), scopeDepth(0)
Parser(std::vector<Token>& _tokens, const std::filesystem::path& path)
: tokens(_tokens), index(0), len(_tokens.size()), scopeDepth(0), file(path)
{}
// The file that the tokens were generated from //
const std::filesystem::path file;
// Tokens created by the lexer //
std::vector<Token>& tokens;
@@ -69,7 +69,7 @@ namespace LX
// TODO: Fix this //
case Token::CLOSE_BRACE:
ThrowIf<UnexpectedToken>(p.scopeDepth == 0, Token::UNDEFINED, "need a different error", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.scopeDepth == 0, Token::UNDEFINED, "need a different error", p.tokens[p.index], p.file);
p.scopeDepth--;
p.index++;
return nullptr;
@@ -92,7 +92,7 @@ namespace LX
{
// Parses the left hand side of the operation //
std::unique_ptr<AST::Node> lhs = ParsePrimary(p);
ThrowIf<UnexpectedToken>(lhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1]);
ThrowIf<UnexpectedToken>(lhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1], p.file);
// Stores the operator to pass into the AST node //
Token::TokenType op = p.tokens[p.index].type;
@@ -100,7 +100,7 @@ namespace LX
// Parses the right hand of the operation //
std::unique_ptr<AST::Node> rhs = ParseOperation(p);
ThrowIf<UnexpectedToken>(rhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1]);
ThrowIf<UnexpectedToken>(rhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1], p.file);
// Returns an AST node as all of the components combined together //
return std::make_unique<AST::Operation>(std::move(lhs), op, std::move(rhs));
@@ -137,7 +137,7 @@ namespace LX
p.index++;
// Checks for the variable name //
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index], p.file);
std::string name = p.tokens[p.index].GetContents();
p.index++; // <- Goes over the identifier token
@@ -152,7 +152,7 @@ namespace LX
// Gets the value to be assigned to the variable //
std::unique_ptr<AST::Node> defaultVal = ParsePrimary(p);
ThrowIf<UnexpectedToken>(defaultVal.get() == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1]);
ThrowIf<UnexpectedToken>(defaultVal.get() == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1], p.file);
return std::make_unique<AST::VariableDeclaration>(name);
}
@@ -170,7 +170,7 @@ namespace LX
if (p.tokens[p.index + 1].type == Token::ASSIGN)
{
// Gets the variable that is being assigned too //
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index], p.file);
std::string name = p.tokens[p.index].GetContents();
// Skips over the assign token and name of the variable //
@@ -195,19 +195,19 @@ namespace LX
std::unique_ptr<AST::Node> out = ParseVarAssignment(p);
// Checks it is valid before returning //
ThrowIf<UnexpectedToken>(out == nullptr, Token::UNDEFINED, "top level statement", p.tokens[p.index - 1]);
ThrowIf<UnexpectedToken>(out == nullptr, Token::UNDEFINED, "top level statement", p.tokens[p.index - 1], p.file);
return out;
}
// Turns the tokens of a file into it's abstract syntax tree equivalent //
FileAST TurnTokensIntoAbstractSyntaxTree(std::vector<Token>& tokens)
FileAST TurnTokensIntoAbstractSyntaxTree(std::vector<Token>& tokens, const std::filesystem::path& path)
{
// Logs the start of the parsing
Log::LogNewSection("Started parsing tokens");
// Creates the output storer and the parser //
FileAST output;
Parser p(tokens);
Parser p(tokens, path);
// Loops over the tokens and calls the correct parsing function //
// Which depends on their type and current state of the parser //
@@ -225,11 +225,11 @@ namespace LX
FunctionDefinition& func = output.functions.back();
// Assigns the function name //
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::IDENTIFIER, Token::IDENTIFIER, "", p.tokens[p.index], p.file);
func.name = p.tokens[p.index++].GetContents();
// Checks for opening paren '(' //
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::OPEN_PAREN, Token::OPEN_PAREN, "", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::OPEN_PAREN, Token::OPEN_PAREN, "", p.tokens[p.index], p.file);
p.index++;
// Loops over all the arguments of the function //
@@ -243,7 +243,7 @@ namespace LX
p.index++;
// Checks for opening bracket '{' //
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::OPEN_BRACKET, Token::OPEN_BRACKET, "", p.tokens[p.index]);
ThrowIf<UnexpectedToken>(p.tokens[p.index].type != Token::OPEN_BRACKET, Token::OPEN_BRACKET, "", p.tokens[p.index], p.file);
p.index++;
// Loops over the body until it reaches the end //

11
Parser/src/ParserErrors.cpp
View File

@@ -13,8 +13,8 @@ namespace LX
return "IR Generation Error";
}
UnexpectedToken::UnexpectedToken(Token::TokenType _expected, std::string _override, Token _got)
: expected(_expected), override(_override), got(_got)
UnexpectedToken::UnexpectedToken(Token::TokenType _expected, std::string _override, Token _got, const std::filesystem::path& _file)
: expected(_expected), override(_override), got(_got), file(_file)
{}
void UnexpectedToken::PrintToConsole() const
@@ -25,13 +25,16 @@ namespace LX
size_t lineNumberWidthInConsole = std::max(oss.str().size(), (size_t)3);
// Gets the line of the error //
std::string line = LX::GetLineAtIndexOf(*s_Source, got.index);
// As the file has been closed and the source has been deleted it needs to be reopened //
std::string fileContents = ReadFileToString(file);
std::string line = LX::GetLineAtIndexOf(fileContents, got.index);
// Prints the error to the console with the relevant info //
std::cout << "\n";
LX::PrintStringAsColor("Error: ", LX::Color::LIGHT_RED);
std::cout << "Incorrect syntax in ";
LX::PrintStringAsColor(s_SourceFile->filename().string(), LX::Color::WHITE);
LX::PrintStringAsColor(file.string(), LX::Color::WHITE);
std::cout << ", found ";
LX::PrintStringAsColor(LX::ToString(got.type).c_str(), LX::Color::WHITE);
std::cout << " expected: ";