Pasha/MIRROR-LX-OriginalRepo
1
0
Fork 0
mirror of https://github.com/PashaBibko/LX.git synced 2026-04-03 17:39:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved lexer debugging

Browse Source
This commit is contained in:
Pasha Bibko
2025-04-21 14:23:47 +01:00
parent a560c53c58
commit 49e4bba510
15 changed files with 405 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
Common/Common.vcxproj Normal file
View File

@@ -0,0 +1,144 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>17.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
<ProjectGuid>{3125ca11-9f6d-4a4f-afc1-37feb3bbd9fa}</ProjectGuid>
<RootNamespace>Common</RootNamespace>
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpp20</LanguageStandard>
<AdditionalIncludeDirectories>$(SolutionDir)common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="Console.h" />
<ClInclude Include="Lexer.h" />
<ClInclude Include="LLVM.h" />
<ClInclude Include="Parser.h" />
<ClInclude Include="Util.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\Console.cpp" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

39
Common/Common.vcxproj.filters Normal file
View File

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Resource Files">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
<Filter Include="inclu">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="Lexer.h">
<Filter>inclu</Filter>
</ClInclude>
<ClInclude Include="LLVM.h">
<Filter>inclu</Filter>
</ClInclude>
<ClInclude Include="Parser.h">
<Filter>inclu</Filter>
</ClInclude>
<ClInclude Include="Util.h">
<Filter>inclu</Filter>
</ClInclude>
<ClInclude Include="Console.h">
<Filter>inclu</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\Console.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

27
Common/Console.h Normal file
View File

@@ -0,0 +1,27 @@
#pragma once
#include <string>
namespace LX
{
enum class Color
{
BLACK = 0,
BLUE = 1,
GREEN = 2,
AQUA = 3,
RED = 4,
PURPLE = 5,
YELLOW = 6,
LIGHT_GRAY = 7,
LIGHT_BLUE = 9,
LIGHT_GREEN = 10,
LIGHT_AQUA = 11,
LIGHT_RED = 12,
LIGHT_PURPLE = 13,
LIGHT_YELLOW = 14,
WHITE = 15
};
void PrintStringAsColor(const std::string& str, Color c);
}

22
Common/src/Console.cpp Normal file
View File

@@ -0,0 +1,22 @@
#include <Console.h>
#include <Windows.h>
#include <iostream>
namespace LX
{
void PrintStringAsColor(const std::string& str, Color c)
{
// Gets a handle to the console //
HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
// Sets the color of the console to the desired color //
SetConsoleTextAttribute(hConsole, (WORD)c);
// Outputs the text //
std::cout << str;
// Resets the color //
SetConsoleTextAttribute(hConsole, (WORD)Color::LIGHT_GRAY);
}
}

14
LX-Compiler.sln
View File

@@ -5,6 +5,7 @@ VisualStudioVersion = 17.13.35931.197
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LX-Compiler", "LX-LLVM.vcxproj", "{CC37E36F-B3B3-41B0-A887-01E8EFE84994}"
ProjectSection(ProjectDependencies) = postProject
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA} = {3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA} = {4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}
{D6EAFB31-4AFD-4989-9522-D6609AC4ED64} = {D6EAFB31-4AFD-4989-9522-D6609AC4ED64}
EndProjectSection
@@ -15,6 +16,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Frontend-Modules", "Fronten
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Parser", "Parser\Parser.vcxproj", "{D6EAFB31-4AFD-4989-9522-D6609AC4ED64}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Util", "Util", "{0E83E43A-13FB-422F-B903-96E3E8F74DC2}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Common", "Common\Common.vcxproj", "{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@@ -47,6 +52,14 @@ Global
{D6EAFB31-4AFD-4989-9522-D6609AC4ED64}.Release|x64.Build.0 = Release|x64
{D6EAFB31-4AFD-4989-9522-D6609AC4ED64}.Release|x86.ActiveCfg = Release|Win32
{D6EAFB31-4AFD-4989-9522-D6609AC4ED64}.Release|x86.Build.0 = Release|Win32
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Debug|x64.ActiveCfg = Debug|x64
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Debug|x64.Build.0 = Debug|x64
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Debug|x86.ActiveCfg = Debug|Win32
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Debug|x86.Build.0 = Debug|Win32
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Release|x64.ActiveCfg = Release|x64
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Release|x64.Build.0 = Release|x64
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Release|x86.ActiveCfg = Release|Win32
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA}.Release|x86.Build.0 = Release|Win32
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -54,6 +67,7 @@ Global
GlobalSection(NestedProjects) = preSolution
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
{D6EAFB31-4AFD-4989-9522-D6609AC4ED64} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
{3125CA11-9F6D-4A4F-AFC1-37FEB3BBD9FA} = {0E83E43A-13FB-422F-B903-96E3E8F74DC2}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {411AFEC9-4075-4FCC-B95A-9288BC822E90}

7
LX-LLVM.vcxproj
View File

@@ -112,7 +112,7 @@
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>Lexer.lib;Parser.lib;Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalDependencies>Lexer.lib;Parser.lib;Common.lib;Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)\;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
@@ -139,11 +139,6 @@
<ItemGroup>
<ClCompile Include="Main.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="common\Parser.h" />
<ClInclude Include="common\Lexer.h" />
<ClInclude Include="common\Util.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>

17
LX-LLVM.vcxproj.filters
View File

@@ -1,22 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Common">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="common\Util.h">
<Filter>Common</Filter>
</ClInclude>
<ClInclude Include="common\Lexer.h">
<Filter>Common</Filter>
</ClInclude>
<ClInclude Include="common\Parser.h">
<Filter>Common</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="Main.cpp" />
</ItemGroup>

109
Lexer/src/Lexer.cpp
View File

@@ -7,6 +7,7 @@
#include <vector>
#include <iostream>
#include <iomanip>
namespace LX
{
@@ -32,12 +33,41 @@ namespace LX
TOKEN_CASE(Token::MUL);
TOKEN_CASE(Token::DIV);
TOKEN_CASE(Token::NUMBER_LITERAL);
TOKEN_CASE(Token::RETURN);
default:
return "Unknown: " + std::to_string(type);
}
}
// Struct to store the current information of the lexer //
struct LexerInfo
{
// Current trackers of where in the source it is //
std::streamsize line = 1; // <- Lines start on 1 (probably because of non-programmer's)
std::streamsize index = 0;
std::streamsize column = 0; // <- Columns start on 1 (probably because of non-programmer's)
// Trackers for when a multi-char token started //
std::streamsize startOfWord = 0;
std::streamsize startOfNumberLiteral = 0;
std::streamsize startOfStringLiteral = 0;
// Different flags of the lexer //
// Stored as a bitset to minimse memory allocated (basically no difference, because only one exists at any given time) //
bool isAlpha : 1 = false;
bool isNumeric : 1 = false;
bool inComment : 1 = false;
bool inStringLiteral : 1 = false;
bool isNextCharAlpha : 1 = false;
bool isNextCharNumeric : 1 = false;
bool wasLastCharAlpha : 1 = false;
bool wasLastCharNumeric : 1 = false;
};
// All the keywords the lexer currently supports with their token-enum equivalents //
static const std::unordered_map<std::string, Token::TokenType> keywords =
{
@@ -61,46 +91,21 @@ namespace LX
};
// Checks if the given word is a keyword before adding it to the tokens //
static void TokenizeWord(const std::string& word, std::vector<Token>& tokens)
static void TokenizeWord(const std::string& word, std::vector<Token>& tokens, LexerInfo& info)
{
// Checks the map for a check and if so adds it with its enum equivalent //
if (auto keyword = keywords.find(word); keyword != keywords.end())
{
tokens.push_back({ keyword->second, "" });
tokens.push_back({ keyword->second, "", info.line, info.column - (std::streamsize)word.size(), (std::streamsize)word.size()});
}
// Else adds it as a type of IDENTIFIER //
else
{
tokens.push_back({ Token::IDENTIFIER, word });
tokens.push_back({ Token::IDENTIFIER, word, info.line, info.column - (std::streamsize)word.size(), (std::streamsize)word.size()});
}
}
// Struct to store the current information of the lexer //
struct LexerInfo
{
// Current index within the lexer //
std::streamsize index = 0;
// Trackers for when a multi-char token started //
std::streamsize startOfWord = 0;
std::streamsize startOfNumberLiteral = 0;
std::streamsize startOfStringLiteral = 0;
// Different flags of the lexer //
// Stored as a bitset to minimse memory allocated (basically no difference, because only one exists at any given time) //
bool isAlpha : 1 = false;
bool isNumeric : 1 = false;
bool inComment : 1 = false;
bool inStringLiteral : 1 = false;
bool isNextCharAlpha : 1 = false;
bool isNextCharNumeric : 1 = false;
bool wasLastCharAlpha : 1 = false;
bool wasLastCharNumeric : 1 = false;
};
const std::vector<Token> LX::LexicalAnalyze(std::ifstream& src, std::ofstream* log)
{
// Logs the start of the lexical analysis
@@ -168,7 +173,7 @@ namespace LX
{
// Adds the string literal token to the token vector //
std::string lit(contents.data() + info.startOfStringLiteral, info.index - info.startOfStringLiteral);
tokens.push_back({ Token::STRING_LITERAL, lit });
tokens.push_back({ Token::STRING_LITERAL, lit, info.line, info.column - (std::streamsize)lit.length(), (std::streamsize)lit.length() });
// Updates trackers //
info.inStringLiteral = false;
@@ -197,7 +202,7 @@ namespace LX
if (info.isNextCharAlpha == false)
{
// Calls the function designed to handle the tokenisation of words //
TokenizeWord({ contents.data() + info.startOfWord, 1 }, tokens);
TokenizeWord({ contents.data() + info.startOfWord, 1 }, tokens, info);
}
}
@@ -205,7 +210,7 @@ namespace LX
else if (info.isAlpha == true && info.isNextCharAlpha == false)
{
// Calls the function designed to handle the tokenisation of words //
TokenizeWord({ contents.data() + info.startOfWord, (unsigned __int64)((info.index + 1) - info.startOfWord) }, tokens);
TokenizeWord({ contents.data() + info.startOfWord, (unsigned __int64)((info.index + 1) - info.startOfWord) }, tokens, info);
}
// During a word //
@@ -222,7 +227,7 @@ namespace LX
{
// Pushes the number to the token vector. Number literals are stored as string in the tokens //
std::string num(contents.data() + info.startOfNumberLiteral, (unsigned __int64)(info.index + 1) - info.startOfNumberLiteral);
tokens.push_back({ Token::NUMBER_LITERAL, num });
tokens.push_back({ Token::NUMBER_LITERAL, num, info.line, info.column - (std::streamsize)num.size(), (std::streamsize)num.size()});
}
}
@@ -231,7 +236,7 @@ namespace LX
{
// Pushes the number to the token vector. Number literals are stored as string in the tokens //
std::string num(contents.data() + info.startOfNumberLiteral, (unsigned __int64)(info.index + 1) - info.startOfNumberLiteral);
tokens.push_back({ Token::NUMBER_LITERAL, num });
tokens.push_back({ Token::NUMBER_LITERAL, num, info.line, info.column - (std::streamsize)num.size(), (std::streamsize)num.size()});
}
// During a number //
@@ -240,21 +245,49 @@ namespace LX
// Operators (+, -, /, *) //
else if (auto op = operators.find(current); op != operators.end())
{
tokens.push_back({ op->second, "" });
tokens.push_back({ op->second, "", info.line, info.column, 1});
}
// If it is here and not whitespace that means it's an invalid character //
else if (current == ' ' || current == '\t' || current == '\r' || current == '\n');
else if (current == ' ' || current == '\r');
// Skips over an extra 3 spaces as tabs SHOULD ALWAYS take up 4 spaces //
// Only for the column and not index //
else if (current == '\t')
{
info.column = info.column + 3;
}
// Increments the line number and resets the column on entering a new line //
else if (current == '\n')
{
info.column = 0;
info.line++;
}
else
{
// Finds the start of the line //
size_t start = contents.rfind('\n', info.index);
if (start == std::string::npos) { start = 0; } // std::npos means none was found so defaults to 1
else { start = start + 1; } // Skips the new line character
// Finds the end of the line //
size_t end = contents.find('\n', info.index);
if (end == std::string::npos) { end = contents.size(); } // If it reaches the end with no /n it defaults to the length of the string
// The line where the invalid character is //
std::string line = contents.substr(start, end - start);
// Throws an error to alert the user of the invalid character //
throw InvalidCharInSource(info.index, current);
throw InvalidCharInSource(info.column, info.line, line, contents[info.index]);
}
// Updates trackers to their default state of a new character //
info.index++;
info.column++;
info.wasLastCharAlpha = info.isAlpha;
info.wasLastCharNumeric = info.isNumeric;
}
@@ -266,12 +299,12 @@ namespace LX
{
if (token.contents.empty() == false)
{
SafeLog(log, ToString(token.type), ":\t", token.contents);
SafeLog(log, "{ Line: ", std::left, std::setw(3), token.line, ", Column: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, "} ", std::setw(30), ToString(token.type) + ":", "{", token.contents, "}");
}
else
{
SafeLog(log, ToString(token.type));
SafeLog(log, "{ Line: ", std::left, std::setw(3), token.line, ", Column: ", std::setw(3), token.index, ", Length: ", std::setw(2), token.length, "} ", ToString(token.type));
}
}
}

4
Lexer/src/Token.cpp
View File

@@ -6,7 +6,7 @@
namespace LX
{
// Passes the constructor args to the values //
Token::Token(const TokenType _type, std::string _contents)
: type(_type), contents(_contents)
Token::Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length)
: type(_type), contents(_contents), line(_line), index(_index), length(_length)
{}
}

58
Main.cpp
View File

@@ -4,6 +4,7 @@
#include <string>
#include <memory>
#include <Console.h>
#include <Parser.h>
#include <Lexer.h>
#include <Util.h>
@@ -20,44 +21,49 @@ namespace LX
int main(int argc, char** argv)
{
// Creates the file paths outside of the try-catch so they can be used in errors //
std::filesystem::path inpPath;
std::filesystem::path outPath;
// Creates the log-file out of the try-catch so it can be closed propely if an error is thrown //
std::unique_ptr<std::ofstream> log = nullptr;
try
{
// Checks there is the correct ammount of arguments
// Checks there is the correct ammount of arguments //
LX::ThrowIf<LX::IncorrectCommandLineArgs>((argc == 3 || argc == 4) == false);
// Turns the file paths into the C++ type for handling them
std::filesystem::path inpPath = argv[1];
std::filesystem::path outPath = argv[2];
// Turns the file paths into the C++ type for handling them //
inpPath = argv[1];
outPath = argv[2];
// Prints the full paths to the console to let the user know compiling is being done
// Prints the full paths to the console to let the user know compiling is being done //
std::cout << std::filesystem::absolute(inpPath) << " -> " << std::filesystem::absolute(outPath) << std::endl;
// Checks the input file exists and opens it
// Checks the input file exists and opens it //
LX::ThrowIf<LX::InvalidInputFilePath>(std::filesystem::exists(inpPath) == false);
std::ifstream inpFile(inpPath, std::ios::binary | std::ios::ate); // Opens in binary at the end for microptimisation
std::ifstream inpFile(inpPath, std::ios::binary | std::ios::ate); // Opens in binary at the end for microptimisation //
LX::ThrowIf<LX::InvalidInputFilePath>(inpFile.is_open() == false);
// Opens / Creates the output file
// Opens / Creates the output file //
std::ofstream outFile(outPath);
LX::ThrowIf<LX::InvalidOutputFilePath>(outFile.is_open() == false);
// Opens / Creates the log file
std::unique_ptr<std::ofstream> log = nullptr;
// Opens the log file (if there is one specified //
if (argc == 4)
{
log = std::make_unique<std::ofstream>(argv[3]);
LX::ThrowIf<LX::InvalidLogFilePath>(log->is_open() == false);
}
// Create tokens out of the input file
// Create tokens out of the input file //
std::vector<LX::Token>tokens = LX::LexicalAnalyze(inpFile, log.get());
// Turns the tokens into an AST
// Turns the tokens into an AST //
LX::FileAST AST = LX::TurnTokensIntoAbstractSyntaxTree(tokens, log.get());
//
LX::GenerateIR(AST);
// Turns the AST into LLVM IR //
LX::GenerateIR(AST, inpPath.filename().string());
// Returns success
return 0;
@@ -89,9 +95,27 @@ int main(int argc, char** argv)
catch (LX::InvalidCharInSource& e)
{
//
std::cout << "\nInvalid character found in source file: {" << e.invalid << "} at index: " << e.index << "\n";
// Adds the error to the log and closes it to save all the output //
LX::SafeLog(log.get(), LX::LOG_BREAK, "Error thrown from Lexer:\n\tInvalid character: ", e.invalid, " on line: ", e.line, LX::LOG_BREAK);
if (log != nullptr) { log->close(); }
// Calculates the length of the line number in the console so it is formatted correctly //
std::ostringstream oss;
oss << std::setw(3) << e.line;
size_t lineNumberWidthInConsole = std::max(oss.str().size(), (size_t)3);
// Prints the error with the relevant information to the console //
std::cout << "\n";
LX::PrintStringAsColor("Error: ", LX::Color::LIGHT_RED);
std::cout << "invalid character found in ";
LX::PrintStringAsColor(inpPath.filename().string(), LX::Color::WHITE);
std::cout << ":\n";
std::cout << "Line: " << std::setw(lineNumberWidthInConsole) << e.line << " | " << e.lineContents << "\n";
std::cout << " " << std::setw(lineNumberWidthInConsole) << "" << " | " << std::setw(e.index);
LX::PrintStringAsColor("^", LX::Color::LIGHT_RED);
std::cout << "\n";
// Returns Exit-id of 4 so other process can be alerted of the error //
return 4;
}

2
Parser/src/AST-Constructors.cpp
View File

@@ -11,7 +11,7 @@ namespace LX
// Reserves space for nodes (stops excess allocations) //
FunctionDefinition::FunctionDefinition()
: body{}
: body{}, name{}
{ body.reserve(32); }
// Reserves space for functions (stops excess allocations) //

8
Parser/src/GenIR.cpp
View File

@@ -8,7 +8,7 @@
namespace LX
{
// Tells the generator if the current node is allowed to be within a top-level context //
// TODO: Make this function do something other than return true
// TODO: Make this function do something other than return true //
static constexpr bool IsValidTopLevelNode(AST::Node::NodeType type)
{
return true;
@@ -20,7 +20,7 @@ namespace LX
// Creates the functions signature and return type //
llvm::FunctionType* retType = llvm::FunctionType::get(llvm::Type::getInt32Ty(LLVM.context), false); // <- Defaults to int currently
llvm::Function* func = llvm::Function::Create(retType, llvm::Function::ExternalLinkage, "main", LLVM.module); // Defaults to main currently
llvm::Function* func = llvm::Function::Create(retType, llvm::Function::ExternalLinkage, funcAST.name, LLVM.module);
llvm::BasicBlock* entry = llvm::BasicBlock::Create(LLVM.context, "entry", func);
LLVM.builder.SetInsertPoint(entry);
@@ -42,10 +42,10 @@ namespace LX
}
// Turns an abstract binary tree into LLVM intermediate representation //
void GenerateIR(FileAST& ast)
void GenerateIR(FileAST& ast, const std::string& name)
{
// Creates the LLVM variables needed for generating IR that are shared between functions //
InfoLLVM LLVM("add_itns");
InfoLLVM LLVM(name);
// Loops over the functions to generate their LLVM IR //
for (auto& func : ast.functions)

18
Parser/src/Parser.cpp
View File

@@ -7,6 +7,12 @@
namespace LX
{
template<Token::TokenType type>
static inline void ExpectToken(const Token& t)
{
ThrowIf<int>(type != t.type);
}
// Local struct so everything can be public //
struct Parser
{
@@ -118,14 +124,17 @@ namespace LX
{
case Token::FUNCTION:
{
// Skips over function token + name token
// TODO: Store function name in the type
p.index++; p.index++;
// Skips over function token + name token //
p.index++;
// Pushes a new function to the vector and gets a reference to it for adding the body //
output.functions.emplace_back();
FunctionDefinition& func = output.functions.back();
// Assigns the function name //
ExpectToken<Token::IDENTIFIER>(p.tokens[p.index]);
func.name = p.tokens[p.index++].contents;
// Loops over the body until it reaches the end //
// TODO: Detect the end instead of looping over the entire token vector
while (p.index < p.len)
@@ -144,7 +153,8 @@ namespace LX
// Lets the user know there is an error //
// TODO: Makes this error actually output useful information //
default:
std::cout << "UNKNOWN TOKEN FOUND" << std::endl;
std::cout << "UNKNOWN TOKEN FOUND: " << p.tokens[p.index].type << std::endl;
return output;
}
}

29
common/Lexer.h
View File

@@ -28,13 +28,17 @@ namespace LX
struct InvalidCharInSource
{
std::streamsize index;
std::streamsize line;
std::string lineContents;
char invalid;
};
// Data type to store a more computer readable version of files
struct __declspec(novtable) Token final
{
// Enum to hold the type of the token
// Enum to hold the type of the token //
enum TokenType : short
{
// General tokens //
@@ -60,18 +64,27 @@ namespace LX
UNDEFINED = -1
};
// Constructor of the tokens to set their info
Token(const TokenType _type, std::string _contents);
// Constructor of the tokens to set their info //
Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length);
// Contents of the token (may be empty if not needed)
// Const to avoid external changes
// Contents of the token (may be empty if not needed) //
// Const to avoid external changes //
const std::string contents;
// Type of the token
// Const to avoid external changes
// Type of the token //
// Const to avoid external changes //
const TokenType type;
// The line where the token is located in the source //
const std::streamsize line;
// Index on the line where the token starts //
const std::streamsize index;
// The length of the token on the line, may be different to the length of contents //
const std::streamsize length;
};
// Lexer function to take in a file and output a vector of tokens
// Lexer function to take in a file and output a vector of tokens //
const std::vector<Token> LexicalAnalyze(std::ifstream& src, std::ofstream* log);
}

5
common/Parser.h
View File

@@ -64,6 +64,9 @@ namespace LX
{
// Defualt constructor (none other given) //
FunctionDefinition();
// The name of the function //
std::string name;
// The instructions of the body of the function //
std::vector<std::unique_ptr<AST::Node>> body;
@@ -82,5 +85,5 @@ namespace LX
FileAST TurnTokensIntoAbstractSyntaxTree(std::vector<Token>& tokens, std::ofstream* log);
// Turns an abstract binary tree into LLVM intermediate representation //
void GenerateIR(FileAST& ast);
void GenerateIR(FileAST& ast, const std::string& name);
}