mirror of
https://github.com/PashaBibko/LX.git
synced 2026-04-03 17:39:02 +00:00
Improved logging
Small errors with logging parser unexpected token errors. Tokens need to be re-written with how they store memory as it can be halved.
This commit is contained in:
@@ -128,6 +128,9 @@
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="src\Console.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="Console.h" />
|
||||
<ClInclude Include="Lexer.h" />
|
||||
@@ -135,9 +138,6 @@
|
||||
<ClInclude Include="Parser.h" />
|
||||
<ClInclude Include="Util.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="src\Console.cpp" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
|
||||
@@ -5,35 +5,17 @@
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Resource Files">
|
||||
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
|
||||
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="inclu">
|
||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="Lexer.h">
|
||||
<Filter>inclu</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="LLVM.h">
|
||||
<Filter>inclu</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="Parser.h">
|
||||
<Filter>inclu</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="Util.h">
|
||||
<Filter>inclu</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="Console.h">
|
||||
<Filter>inclu</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="src\Console.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="Console.h" />
|
||||
<ClInclude Include="Lexer.h" />
|
||||
<ClInclude Include="LLVM.h" />
|
||||
<ClInclude Include="Parser.h" />
|
||||
<ClInclude Include="Util.h" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -17,6 +17,22 @@ namespace LX
|
||||
struct InvalidInputFilePath {};
|
||||
struct InvalidOutputFilePath {};
|
||||
struct InvalidLogFilePath {};
|
||||
|
||||
// Util function for getting a line of the source at a given index (used for errors) //
|
||||
static std::string GetLineAtIndexOf(const std::string src, const std::streamsize index)
|
||||
{
|
||||
// Finds the start of the line //
|
||||
size_t start = src.rfind('\n', index);
|
||||
if (start == std::string::npos) { start = 0; } // None means first line
|
||||
else { start = start + 1; } // Skips new line char
|
||||
|
||||
// Finds the end of the line //
|
||||
size_t end = src.find('\n', index);
|
||||
if (end == std::string::npos) { end = src.size(); } // None means last line
|
||||
|
||||
// Returns the string between start and end //
|
||||
return src.substr(start, end - start);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
@@ -26,6 +42,9 @@ int main(int argc, char** argv)
|
||||
std::filesystem::path outPath;
|
||||
std::filesystem::path logPath;
|
||||
|
||||
// Creates the contents string outside of the try-catch so they can be used in errors //
|
||||
std::string contents;
|
||||
|
||||
// Creates the log-file out of the try-catch so it can be closed propely if an error is thrown //
|
||||
std::unique_ptr<std::ofstream> log = nullptr;
|
||||
|
||||
@@ -43,6 +62,12 @@ int main(int argc, char** argv)
|
||||
std::ifstream inpFile(inpPath, std::ios::binary | std::ios::ate); // Opens in binary at the end for microptimisation //
|
||||
LX::ThrowIf<LX::InvalidInputFilePath>(inpFile.is_open() == false);
|
||||
|
||||
// Copies the file into the string //
|
||||
const std::streamsize len = inpFile.tellg(); // Gets length of file because it was opened at the end
|
||||
inpFile.seekg(0, std::ios::beg); // Goes back to the beginning
|
||||
contents = std::string(len, '\0'); // Allocates all the space for the string
|
||||
inpFile.read(&contents[0], len); // Transfers file contents to string
|
||||
|
||||
// Opens / Creates the output file //
|
||||
std::ofstream outFile(outPath);
|
||||
LX::ThrowIf<LX::InvalidOutputFilePath>(outFile.is_open() == false);
|
||||
@@ -60,7 +85,7 @@ int main(int argc, char** argv)
|
||||
std::cout << std::filesystem::absolute(inpPath) << " -> " << std::filesystem::absolute(outPath) << std::endl;
|
||||
|
||||
// Create tokens out of the input file //
|
||||
std::vector<LX::Token>tokens = LX::LexicalAnalyze(inpFile, log.get());
|
||||
std::vector<LX::Token>tokens = LX::LexicalAnalyze(contents, len, log.get());
|
||||
LX::SafeFlush(log.get());
|
||||
|
||||
// Turns the tokens into an AST //
|
||||
@@ -144,14 +169,17 @@ int main(int argc, char** argv)
|
||||
oss << std::setw(3) << e.line;
|
||||
size_t lineNumberWidthInConsole = std::max(oss.str().size(), (size_t)3);
|
||||
|
||||
// Gets the line of the error //
|
||||
std::string line = LX::GetLineAtIndexOf(contents, e.index);
|
||||
|
||||
// Prints the error with the relevant information to the console //
|
||||
std::cout << "\n";
|
||||
LX::PrintStringAsColor("Error: ", LX::Color::LIGHT_RED);
|
||||
std::cout << "Invalid character found in ";
|
||||
LX::PrintStringAsColor(inpPath.filename().string(), LX::Color::WHITE);
|
||||
std::cout << ":\n";
|
||||
std::cout << "Line: " << std::setw(lineNumberWidthInConsole) << e.line << " | " << e.lineContents << "\n";
|
||||
std::cout << " " << std::setw(lineNumberWidthInConsole) << "" << " | " << std::setw(e.index);
|
||||
std::cout << "Line: " << std::setw(lineNumberWidthInConsole) << e.line << " | " << line << "\n";
|
||||
std::cout << " " << std::setw(lineNumberWidthInConsole) << "" << " | " << std::setw(e.col);
|
||||
LX::PrintStringAsColor("^", LX::Color::LIGHT_RED);
|
||||
std::cout << "\n";
|
||||
|
||||
@@ -159,6 +187,41 @@ int main(int argc, char** argv)
|
||||
return 5;
|
||||
}
|
||||
|
||||
catch (LX::UnexpectedToken& e)
|
||||
{
|
||||
// Calculates the length of the line number in the console so it is formatted correctly //
|
||||
std::ostringstream oss;
|
||||
oss << std::setw(3) << e.got.line;
|
||||
size_t lineNumberWidthInConsole = std::max(oss.str().size(), (size_t)3);
|
||||
|
||||
// Gets the line of the error //
|
||||
std::string line = LX::GetLineAtIndexOf(contents, e.got.index);
|
||||
|
||||
// Prints the error to the console with the relevant info //
|
||||
std::cout << "\n";
|
||||
LX::PrintStringAsColor("Error: ", LX::Color::LIGHT_RED);
|
||||
std::cout << "Incorrect syntax in ";
|
||||
LX::PrintStringAsColor(inpPath.filename().string(), LX::Color::WHITE);
|
||||
std::cout << ", found ";
|
||||
LX::PrintStringAsColor(LX::ToString(e.got.type), LX::Color::WHITE);
|
||||
std::cout << " expected: ";
|
||||
|
||||
// Allows the error to have a custom type that is printed to the console //
|
||||
if (e.expected == LX::Token::UNDEFINED) { LX::PrintStringAsColor(e.override, LX::Color::WHITE); }
|
||||
else { LX::PrintStringAsColor(LX::ToString(e.expected), LX::Color::WHITE); }
|
||||
std::cout << "\n";
|
||||
|
||||
// Prints the code with the error to the console //
|
||||
std::string errorSquiggle(e.got.length, '^');
|
||||
std::cout << "Line: " << std::setw(lineNumberWidthInConsole) << e.got.line << " | " << line << "\n";
|
||||
std::cout << " " << std::setw(lineNumberWidthInConsole) << "" << " | " << std::setw(e.got.column - 1) << "";
|
||||
LX::PrintStringAsColor(errorSquiggle, LX::Color::LIGHT_RED);
|
||||
std::cout << "\n";
|
||||
|
||||
// Returns Exit id of 6 so other process can be alerted of the error //
|
||||
return 6;
|
||||
}
|
||||
|
||||
// Catches any std errors, there should be none //
|
||||
catch (std::exception& e)
|
||||
{
|
||||
|
||||
@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.13.35931.197
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LX-Compiler", "LX-LLVM.vcxproj", "{CC37E36F-B3B3-41B0-A887-01E8EFE84994}"
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Frontend", "LX-LLVM.vcxproj", "{CC37E36F-B3B3-41B0-A887-01E8EFE84994}"
|
||||
ProjectSection(ProjectDependencies) = postProject
|
||||
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA} = {3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}
|
||||
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA} = {4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
<ProjectGuid>{cc37e36f-b3b3-41b0-a887-01e8efe84994}</ProjectGuid>
|
||||
<RootNamespace>LXLLVM</RootNamespace>
|
||||
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
|
||||
<ProjectName>Compiler-Frontend</ProjectName>
|
||||
<ProjectName>Frontend</ProjectName>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
|
||||
@@ -11,12 +11,11 @@
|
||||
|
||||
namespace LX
|
||||
{
|
||||
// Local macros cause im lazy //
|
||||
|
||||
// Helper macro for outputting token type //
|
||||
#define TOKEN_CASE(type) case type: return #type;
|
||||
|
||||
// Logging function to turn a tokentype enum val into it's string //
|
||||
static std::string ToString(Token::TokenType type)
|
||||
// Helper util function to translate a tokentype to it's enum val //
|
||||
static std::string ToStringNoFormat(Token::TokenType type)
|
||||
{
|
||||
switch (type)
|
||||
{
|
||||
@@ -40,51 +39,53 @@ namespace LX
|
||||
}
|
||||
}
|
||||
|
||||
static constexpr bool CanBePartOfNumberLiteral(const char c)
|
||||
// Logging function to turn a tokentype enum val into a nicely formatted string //
|
||||
std::string ToString(Token::TokenType type)
|
||||
{
|
||||
return (c == '.') || (c == 'f');
|
||||
// Gets the unformated version of the string //
|
||||
std::string unformatted = ToStringNoFormat(type);
|
||||
unformatted = unformatted.substr(7); // Removes the Token:: prefix
|
||||
|
||||
// Formats the string (turns to lowercase and replaces _ with a space //
|
||||
std::string formatted;
|
||||
|
||||
for (char current : unformatted)
|
||||
{
|
||||
// Adding 32 makes it lowercase due to how ASCII works //
|
||||
if ((current >= 'A' && current <= 'Z')) { formatted.push_back(current + 32); }
|
||||
|
||||
// Replaces _ with spaces //
|
||||
else if (current == '_') { formatted.push_back(' '); }
|
||||
|
||||
// Else adds the current character //
|
||||
else { formatted.push_back(current); }
|
||||
}
|
||||
|
||||
// Returns the formatted string //
|
||||
return formatted;
|
||||
}
|
||||
|
||||
// Stops use outside of the function //
|
||||
#undef TOKEN_CASE
|
||||
|
||||
// Helper function for dealing with floating-point number literals //
|
||||
static constexpr bool CanBePartOfNumberLiteral(const char c) { return (c == '.') || (c == 'f'); }
|
||||
|
||||
// Helper function to stop printing whitespace as pure whitespace //
|
||||
static std::string PrintChar(const char c)
|
||||
{
|
||||
switch (c)
|
||||
{
|
||||
// Stores them as pure string literals //
|
||||
case '\n': return R"(\n)";
|
||||
case '\t': return R"(\t)";
|
||||
case '\r': return R"(\r)";
|
||||
|
||||
// Else returns a string of length one with the char inside //
|
||||
default: return std::string(1, c);
|
||||
}
|
||||
}
|
||||
|
||||
// Struct to store the current information of the lexer //
|
||||
struct LexerInfo
|
||||
{
|
||||
// Current trackers of where in the source it is //
|
||||
|
||||
std::streamsize line = 1; // <- Lines start on 1 (probably because of non-programmer's)
|
||||
std::streamsize index = 0;
|
||||
std::streamsize column = 0; // <- Columns start on 1 (probably because of non-programmer's)
|
||||
|
||||
// Trackers for when a multi-char token started //
|
||||
|
||||
std::streamsize startOfWord = 0;
|
||||
std::streamsize startOfNumberLiteral = 0;
|
||||
std::streamsize startOfStringLiteral = 0;
|
||||
|
||||
// Different flags of the lexer //
|
||||
// Stored as a bitset to minimse memory allocated (basically no difference, because only one exists at any given time) //
|
||||
|
||||
bool isAlpha : 1 = false;
|
||||
bool isNumeric : 1 = false;
|
||||
bool inComment : 1 = false;
|
||||
bool inStringLiteral : 1 = false;
|
||||
bool isNextCharAlpha : 1 = false;
|
||||
bool isNextCharNumeric : 1 = false;
|
||||
bool wasLastCharAlpha : 1 = false;
|
||||
bool wasLastCharNumeric : 1 = false;
|
||||
bool lexingNumber : 1 = false;
|
||||
};
|
||||
|
||||
// All the keywords the lexer currently supports with their token-enum equivalents //
|
||||
static const std::unordered_map<std::string, Token::TokenType> keywords =
|
||||
{
|
||||
@@ -113,17 +114,17 @@ namespace LX
|
||||
// Checks the map for a check and if so adds it with its enum equivalent //
|
||||
if (auto keyword = keywords.find(word); keyword != keywords.end())
|
||||
{
|
||||
tokens.push_back({ keyword->second, "", info.line, info.column - (std::streamsize)word.size(), (std::streamsize)word.size()});
|
||||
tokens.push_back({ keyword->second, info, "", (std::streamsize)word.size() });
|
||||
}
|
||||
|
||||
// Else adds it as a type of IDENTIFIER //
|
||||
else
|
||||
{
|
||||
tokens.push_back({ Token::IDENTIFIER, word, info.line, info.column - (std::streamsize)word.size(), (std::streamsize)word.size()});
|
||||
tokens.push_back({ Token::IDENTIFIER, info, word, (std::streamsize)word.size() });
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<Token> LX::LexicalAnalyze(std::ifstream& src, std::ofstream* log)
|
||||
const std::vector<Token> LX::LexicalAnalyze(const std::string& contents, std::streamsize len, std::ofstream* log)
|
||||
{
|
||||
// Logs the start of the lexical analysis
|
||||
SafeLog(log, LOG_BREAK, "Started lexing file", LOG_BREAK);
|
||||
@@ -133,15 +134,6 @@ namespace LX
|
||||
std::vector<Token> tokens = {};
|
||||
tokens.reserve(0xFFFF);
|
||||
|
||||
// Turns the contents of the file into a string //
|
||||
|
||||
// Gets length of the file because it is opened at the end
|
||||
const std::streamsize len = src.tellg();
|
||||
src.seekg(0, std::ios::beg); // Goes back to the beginning
|
||||
|
||||
std::string contents(len, '\0'); // Preallocates all space needed
|
||||
src.read(&contents[0], len); // Transfers file to string
|
||||
|
||||
// Trackers for when the program is iterating over the file //
|
||||
LexerInfo info;
|
||||
|
||||
@@ -190,7 +182,7 @@ namespace LX
|
||||
{
|
||||
// Adds the string literal token to the token vector //
|
||||
std::string lit(contents.data() + info.startOfStringLiteral, info.index - info.startOfStringLiteral);
|
||||
tokens.push_back({ Token::STRING_LITERAL, lit, info.line, info.column - (std::streamsize)lit.length(), (std::streamsize)lit.length() });
|
||||
tokens.push_back({ Token::STRING_LITERAL, info, lit, (std::streamsize)lit.length() + 1 });
|
||||
|
||||
// Updates trackers //
|
||||
info.inStringLiteral = false;
|
||||
@@ -220,7 +212,7 @@ namespace LX
|
||||
{
|
||||
// Pushes the number to the token vector. Number literals are stored as string in the tokens //
|
||||
std::string num(contents.data() + info.startOfNumberLiteral, (unsigned __int64)(info.index + 1) - info.startOfNumberLiteral);
|
||||
tokens.push_back({ Token::NUMBER_LITERAL, num, info.line, info.column - (std::streamsize)num.size(), (std::streamsize)num.size() });
|
||||
tokens.push_back({ Token::NUMBER_LITERAL, info, num, (std::streamsize)num.size() });
|
||||
}
|
||||
|
||||
// Stores it is lexing a number literal //
|
||||
@@ -232,7 +224,7 @@ namespace LX
|
||||
{
|
||||
// Pushes the number to the token vector. Number literals are stored as string in the tokens //
|
||||
std::string num(contents.data() + info.startOfNumberLiteral, (unsigned __int64)(info.index + 1) - info.startOfNumberLiteral);
|
||||
tokens.push_back({ Token::NUMBER_LITERAL, num, info.line, info.column - (std::streamsize)num.size(), (std::streamsize)num.size() });
|
||||
tokens.push_back({ Token::NUMBER_LITERAL, info, num, (std::streamsize)num.size() });
|
||||
info.lexingNumber = false; // Stops storing it is lexing a number
|
||||
}
|
||||
|
||||
@@ -267,7 +259,7 @@ namespace LX
|
||||
// Operators (+, -, /, *) //
|
||||
else if (auto op = operators.find(current); op != operators.end())
|
||||
{
|
||||
tokens.push_back({ op->second, "", info.line, info.column, 1});
|
||||
tokens.push_back({ op->second, info, "", 1 });
|
||||
}
|
||||
|
||||
// If it is here and not whitespace that means it's an invalid character //
|
||||
@@ -287,31 +279,20 @@ namespace LX
|
||||
info.line++;
|
||||
}
|
||||
|
||||
// Throws an error with all the relevant information //s
|
||||
// Throws an error with all the relevant information //
|
||||
else
|
||||
{
|
||||
// Finds the start of the line //
|
||||
size_t start = contents.rfind('\n', info.index);
|
||||
if (start == std::string::npos) { start = 0; } // std::npos means none was found so defaults to 1
|
||||
else { start = start + 1; } // Skips the new line character
|
||||
|
||||
// Finds the end of the line //
|
||||
size_t end = contents.find('\n', info.index);
|
||||
if (end == std::string::npos) { end = contents.size(); } // If it reaches the end with no /n it defaults to the length of the string
|
||||
|
||||
// The line where the invalid character is //
|
||||
std::string line = contents.substr(start, end - start);
|
||||
|
||||
// Throws an error to alert the user of the invalid character //
|
||||
throw InvalidCharInSource(info.column, info.line, line, contents[info.index]);
|
||||
throw InvalidCharInSource(info.column, info.line, info.index, contents[info.index]);
|
||||
}
|
||||
|
||||
// Log dumps A LOT of info //
|
||||
|
||||
#ifdef LOG_EVERYTHING
|
||||
|
||||
SafeLog
|
||||
(
|
||||
log,
|
||||
"Is Alpha: ", info.isAlpha,
|
||||
log, "Index: ", std::left, std::setw(3), info.index,
|
||||
" Is Alpha: ", info.isAlpha,
|
||||
" Is Numeric: ", info.isNumeric,
|
||||
" In Comment: ", info.inComment,
|
||||
" In String: ", info.inStringLiteral,
|
||||
@@ -322,6 +303,8 @@ namespace LX
|
||||
" Current: {", PrintChar(current), "}"
|
||||
);
|
||||
|
||||
#endif // LOG_EVERYTHING
|
||||
|
||||
// Updates trackers to their default state of a new character //
|
||||
|
||||
info.index++;
|
||||
@@ -334,18 +317,20 @@ namespace LX
|
||||
// Logs the tokens if logging is on //
|
||||
if (log != nullptr)
|
||||
{
|
||||
SafeLog(log, LOG_BREAK, "Tokens", LOG_BREAK);
|
||||
#ifdef LOG_EVERYTHING
|
||||
SafeLog(log, "\n"); // Puts a space when there is a lot in the log
|
||||
#endif // LOG_EVERYTHING
|
||||
|
||||
for (auto& token : tokens)
|
||||
{
|
||||
if (token.contents.empty() == false)
|
||||
{
|
||||
SafeLog(log, "{ Line: ", std::left, std::setw(3), token.line, ", Column: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, "} ", std::setw(30), ToString(token.type) + ":", "{", token.contents, "}");
|
||||
SafeLog(log, std::left, "{ Line: ", std::setw(3), token.line, ", Index: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, " } ", std::setw(30), ToStringNoFormat(token.type) + ":", "{", token.contents, "}");
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
SafeLog(log, "{ Line: ", std::left, std::setw(3), token.line, ", Column: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, "} ", ToString(token.type));
|
||||
SafeLog(log, std::left, "{ Line: ", std::setw(3), token.line, ", Index: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, " } ", ToStringNoFormat(token.type));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
namespace LX
|
||||
{
|
||||
// Passes the constructor args to the values //
|
||||
Token::Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length)
|
||||
: type(_type), contents(_contents), line(_line), index(_index), length(_length)
|
||||
Token::Token(const TokenType _type, const LexerInfo& info, std::string _contents, std::streamsize _length)
|
||||
: type(_type), contents(_contents), index(info.index - _length + 1), line(info.line), column(info.column - _length), length(_length)
|
||||
{}
|
||||
}
|
||||
|
||||
@@ -45,10 +45,8 @@ namespace LX
|
||||
case Token::NUMBER_LITERAL:
|
||||
return std::make_unique<AST::NumberLiteral>(p.tokens[p.index++].contents);
|
||||
|
||||
// Default just alerts the user of an error //
|
||||
// TODO: Actually make this error tell the user something useful //
|
||||
// Returns nullptr, the parsing function that recives that value will decide if that is valid //
|
||||
default:
|
||||
std::cout << "UNKNOWN TOKEN: " << p.tokens[p.index].type << std::endl;
|
||||
p.index++;
|
||||
return nullptr;
|
||||
}
|
||||
@@ -66,6 +64,7 @@ namespace LX
|
||||
{
|
||||
// Parses the left hand side of the operation //
|
||||
std::unique_ptr<AST::Node> lhs = ParsePrimary(p);
|
||||
ThrowIf<UnexpectedToken>(lhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1]);
|
||||
|
||||
// Stores the operator to pass into the AST node //
|
||||
Token::TokenType op = p.tokens[p.index].type;
|
||||
@@ -73,6 +72,7 @@ namespace LX
|
||||
|
||||
// Parses the right hand of the operation //
|
||||
std::unique_ptr<AST::Node> rhs = ParseOperation(p);
|
||||
ThrowIf<UnexpectedToken>(rhs == nullptr, Token::UNDEFINED, "value", p.tokens[p.index - 1]);
|
||||
|
||||
// Returns an AST node as all of the components combined together //
|
||||
return std::make_unique<AST::Operation>(std::move(lhs), op, std::move(rhs));
|
||||
|
||||
@@ -9,13 +9,9 @@ namespace std
|
||||
template<typename T1>
|
||||
struct char_traits;
|
||||
|
||||
template<typename T1, typename T2>
|
||||
class basic_ifstream;
|
||||
|
||||
template<typename T1, typename T2>
|
||||
class basic_ofstream;
|
||||
|
||||
using ifstream = basic_ifstream<char, char_traits<char>>;
|
||||
using ofstream = basic_ofstream<char, char_traits<char>>;
|
||||
}
|
||||
|
||||
@@ -27,14 +23,42 @@ namespace LX
|
||||
// Error type with index and character to alert the user that LX does not understand that symbol //
|
||||
struct InvalidCharInSource
|
||||
{
|
||||
std::streamsize index;
|
||||
std::streamsize col;
|
||||
std::streamsize line;
|
||||
|
||||
std::string lineContents;
|
||||
std::streamsize index;
|
||||
|
||||
char invalid;
|
||||
};
|
||||
|
||||
// Struct to store the current information of the lexer //
|
||||
struct LexerInfo
|
||||
{
|
||||
// Current trackers of where in the source it is //
|
||||
|
||||
std::streamsize line = 1; // <- Lines start on 1 (probably because of non-programmer's)
|
||||
std::streamsize index = 0;
|
||||
std::streamsize column = 0; // <- Columns start on 1 (probably because of non-programmer's)
|
||||
|
||||
// Trackers for when a multi-char token started //
|
||||
|
||||
std::streamsize startOfWord = 0;
|
||||
std::streamsize startOfNumberLiteral = 0;
|
||||
std::streamsize startOfStringLiteral = 0;
|
||||
|
||||
// Different flags of the lexer //
|
||||
// Stored as a bitset to minimse memory allocated (basically no difference, because only one exists at any given time) //
|
||||
|
||||
bool isAlpha : 1 = false;
|
||||
bool isNumeric : 1 = false;
|
||||
bool inComment : 1 = false;
|
||||
bool inStringLiteral : 1 = false;
|
||||
bool isNextCharAlpha : 1 = false;
|
||||
bool isNextCharNumeric : 1 = false;
|
||||
bool wasLastCharAlpha : 1 = false;
|
||||
bool wasLastCharNumeric : 1 = false;
|
||||
bool lexingNumber : 1 = false;
|
||||
};
|
||||
|
||||
// Data type to store a more computer readable version of files
|
||||
struct __declspec(novtable) Token final
|
||||
{
|
||||
@@ -65,7 +89,7 @@ namespace LX
|
||||
};
|
||||
|
||||
// Constructor of the tokens to set their info //
|
||||
Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length);
|
||||
Token(const TokenType _type, const LexerInfo& info, std::string _contents, std::streamsize _length);
|
||||
|
||||
// Contents of the token (may be empty if not needed) //
|
||||
// Const to avoid external changes //
|
||||
@@ -75,16 +99,22 @@ namespace LX
|
||||
// Const to avoid external changes //
|
||||
const TokenType type;
|
||||
|
||||
// The line where the token is located in the source //
|
||||
const std::streamsize line;
|
||||
|
||||
// Index on the line where the token starts //
|
||||
// Index in the source of the token //
|
||||
const std::streamsize index;
|
||||
|
||||
// The length of the token on the line, may be different to the length of contents //
|
||||
const std::streamsize length;
|
||||
|
||||
// The line the token is located on //
|
||||
const std::streamsize line;
|
||||
|
||||
// The column on the line where it is located //
|
||||
const std::streamsize column;
|
||||
};
|
||||
|
||||
// Logging function to turn a tokentype enum val into it's string //
|
||||
std::string ToString(Token::TokenType t);
|
||||
|
||||
// Lexer function to take in a file and output a vector of tokens //
|
||||
const std::vector<Token> LexicalAnalyze(std::ifstream& src, std::ofstream* log);
|
||||
const std::vector<Token> LexicalAnalyze(const std::string& contents, const std::streamsize len, std::ofstream* log);
|
||||
}
|
||||
|
||||
@@ -61,6 +61,20 @@ namespace LX
|
||||
// Thrown if there was an error during IR Generation //
|
||||
struct IRGenerationError {};
|
||||
|
||||
// Thrown if there was an unexpected (incorrect) token //
|
||||
struct UnexpectedToken
|
||||
{
|
||||
// The token type that should be there //
|
||||
Token::TokenType expected;
|
||||
|
||||
// If there are multiple expected types there is an option for a custom message //
|
||||
std::string override;
|
||||
|
||||
// What token was actually at that position //
|
||||
// Stored as Token not TokenType to store the location of it within the source //
|
||||
Token got;
|
||||
};
|
||||
|
||||
// Holds all needed info about a function //
|
||||
// Currently only holds the body but in the future will hold: name, params, namespace/class-member
|
||||
struct FunctionDefinition
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
#include <fstream>
|
||||
|
||||
// Defining this is only if you are at the point where you should be using a debugger //
|
||||
#define LOG_EVERYTHING
|
||||
|
||||
namespace LX
|
||||
{
|
||||
template<typename T, typename... Args>
|
||||
|
||||
Reference in New Issue
Block a user