Pasha/MIRROR-LX-OriginalRepo
1
0
Fork 0
mirror of https://github.com/PashaBibko/LX.git synced 2026-04-04 01:49:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added lexer from previous project

Browse Source
This commit is contained in:
Pasha Bibko
2025-04-16 19:32:26 +01:00
parent 753c72c653
commit 4ac5061f03
14 changed files with 506 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
LX-Compiler.sln
View File

@@ -1,9 +1,14 @@
 
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17 # Visual Studio Version 17
VisualStudioVersion = 17.13.35931.197 d17.13 VisualStudioVersion = 17.13.35931.197
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LX-LLVM", "LX-LLVM.vcxproj", "{CC37E36F-B3B3-41B0-A887-01E8EFE84994}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "LX-Compiler", "LX-LLVM.vcxproj", "{CC37E36F-B3B3-41B0-A887-01E8EFE84994}"
ProjectSection(ProjectDependencies) = postProject
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA} = {4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Lexer", "Lexer\Lexer.vcxproj", "{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -21,6 +26,14 @@ Global
{CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x64.Build.0 = Release|x64 {CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x64.Build.0 = Release|x64
{CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x86.ActiveCfg = Release|Win32 {CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x86.ActiveCfg = Release|Win32
{CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x86.Build.0 = Release|Win32 {CC37E36F-B3B3-41B0-A887-01E8EFE84994}.Release|x86.Build.0 = Release|Win32
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Debug|x64.ActiveCfg = Debug|x64
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Debug|x64.Build.0 = Debug|x64
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Debug|x86.ActiveCfg = Debug|Win32
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Debug|x86.Build.0 = Debug|Win32
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Release|x64.ActiveCfg = Release|x64
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Release|x64.Build.0 = Release|x64
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Release|x86.ActiveCfg = Release|Win32
{4E4019F5-12E0-4EE2-9658-A0DD3038EEDA}.Release|x86.Build.0 = Release|Win32
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

9
LX-LLVM.vcxproj
View File

@@ -106,11 +106,13 @@
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode> <ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpp20</LanguageStandard> <LanguageStandard>stdcpp20</LanguageStandard>
<AdditionalIncludeDirectories>$(SolutionDir)/Common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation> <GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>Lexer.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)\;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Link> </Link>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
@@ -122,19 +124,22 @@
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode> <ConformanceMode>true</ConformanceMode>
<LanguageStandard>stdcpp20</LanguageStandard> <LanguageStandard>stdcpp20</LanguageStandard>
<AdditionalIncludeDirectories>$(SolutionDir)/Common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>$(SolutionDir)common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding> <EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences> <OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation> <GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalDependencies>Lexer.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)$(Platform)\$(Configuration)\;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Link> </Link>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="Main.cpp" /> <ClCompile Include="Main.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="common\Lexer.h" />
<ClInclude Include="common\Util.h" /> <ClInclude Include="common\Util.h" />
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

19
LX-LLVM.vcxproj.filters
View File

@@ -1,23 +1,20 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup> <ItemGroup>
<Filter Include="Source Files"> <Filter Include="Common">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Header Files">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier> <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions> <Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter> </Filter>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ClCompile Include="Main.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="common\Util.h"> <ClInclude Include="common\Util.h">
<Filter>Header Files</Filter> <Filter>Common</Filter>
</ClInclude>
<ClInclude Include="common\Lexer.h">
<Filter>Common</Filter>
</ClInclude> </ClInclude>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ClCompile Include="Main.cpp" />
</ItemGroup>
</Project> </Project>

4
LX-LLVM.vcxproj.user
View File

@@ -4,11 +4,11 @@
<ShowAllFiles>false</ShowAllFiles> <ShowAllFiles>false</ShowAllFiles>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LocalDebuggerCommandArguments>build-test/Main.lx build-test/Main.ll</LocalDebuggerCommandArguments> <LocalDebuggerCommandArguments>build-test/Main.lx build-test/Main.ll build-test/Log.txt</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LocalDebuggerCommandArguments>build-test/Main.lx build-test/Main.ll</LocalDebuggerCommandArguments> <LocalDebuggerCommandArguments>build-test/Main.lx build-test/Main.ll build-test/Log.txt</LocalDebuggerCommandArguments>
<DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor> <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
</PropertyGroup> </PropertyGroup>
</Project> </Project>

140
Lexer/Lexer.vcxproj Normal file
View File

@@ -0,0 +1,140 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>17.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
<ProjectGuid>{4e4019f5-12e0-4ee2-9658-a0dd3038eeda}</ProjectGuid>
<RootNamespace>Lexer</RootNamespace>
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>$(ProjectDir)inc;$(SolutionDir)common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<LanguageStandard>stdcpp20</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<ConformanceMode>true</ConformanceMode>
<AdditionalIncludeDirectories>$(ProjectDir)inc;$(SolutionDir)common;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<LanguageStandard>stdcpp20</LanguageStandard>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="src\Lexer.cpp" />
<ClCompile Include="src\Token.cpp" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

21
Lexer/Lexer.vcxproj.filters Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Header Files">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="src\Lexer.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="src\Token.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>

4
Lexer/Lexer.vcxproj.user Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup />
</Project>

181
Lexer/src/Lexer.cpp Normal file
View File

@@ -0,0 +1,181 @@
#include <Lexer.h>
#include <Util.h>
#include <unordered_map>
#include <string_view>
#include <fstream>
#include <vector>
#include <iostream>
namespace LX
{
// Local macros cause im lazy //
#define ITERATE index++; continue
#define TOKEN_CASE(type) case type: return #type;
static std::string ToString(Token::TokenType type)
{
switch (type)
{
TOKEN_CASE(Token::STRING_LITERAL);
TOKEN_CASE(Token::IDENTIFIER);
TOKEN_CASE(Token::FOR);
TOKEN_CASE(Token::WHILE);
TOKEN_CASE(Token::IF);
TOKEN_CASE(Token::ELSE);
TOKEN_CASE(Token::ELIF);
TOKEN_CASE(Token::FUNCTION);
default:
return std::string("Unknown: " + (short)type);
}
}
static const std::unordered_map<std::string, Token::TokenType> keywords =
{
{ "for", Token::FOR },
{ "while", Token::WHILE },
{ "if", Token::IF },
{ "else", Token::ELSE },
{ "elif", Token::ELIF },
{ "func", Token::FUNCTION },
};
const std::vector<Token> LX::LexicalAnalyze(std::ifstream& src, std::ofstream* log)
{
// Logs the start of the lexical analysis
SafeLog(log, LOG_BREAK, "Started lexing file", LOG_BREAK);
// Allocates a large ammount of memory to hold the output
// Will shrink the size later on to stop excess memory
std::vector<Token> tokens = {};
tokens.reserve(0xFFFF);
// Turns the contents of the file into a string //
// Gets length of the file because it is opened at the end
const std::streamsize len = src.tellg();
src.seekg(0, std::ios::beg); // Goes back to the beginning
std::string contents(len, '\0'); // Preallocates all space needed
src.read(&contents[0], len); // Transfers file to string
// Trackers for when the program is iterating over the file //
std::streamsize index = 0;
std::streamsize startOfWord = 0;
std::streamsize startOfStringLiteral = 0;
bool inComment = false;
bool inStringLiteral = false;
bool wasLastCharAlpha = false;
// Iterates over the file and turns it into tokens //
while (index < len)
{
// Stores the current character for easy access
const char current = contents[index];
// Updates string literal tracker and skips over rest if in a string literal
if (current == '"')
{
// Start of string literal
if (inStringLiteral == false)
{
// Updates the neccesarry trackers
startOfStringLiteral = index + 1;
inStringLiteral = true;
ITERATE;
}
// End of string literal
else
{
// Adds the string literal token to the token vector
std::string lit(contents.data() + startOfStringLiteral, index - startOfStringLiteral);
tokens.push_back({ Token::STRING_LITERAL, lit });
// Updates trackers
inStringLiteral = false;
ITERATE;
}
}
// Skips over rest if within a string literal
if (inStringLiteral) { ITERATE; }
// Updates comment state
if (current == '#')
{
inComment = !inComment;
ITERATE;
}
// Skips over if within a comment
if (inComment) { ITERATE; }
// Works out if the current character is alphabetic
bool isAlpha = (current >= 'a' && current <= 'z') || (current >= 'A' && current <= 'Z');
if (isAlpha == true)
{
// Start of a word
if (wasLastCharAlpha == false)
{
// Updates trackers
wasLastCharAlpha = true;
startOfWord = index;
}
ITERATE;
}
// End of a word
if (isAlpha == false && wasLastCharAlpha == true)
{
// Adds the word token to the token vector
std::string word(contents.data() + startOfWord, index - startOfWord);
if (auto keyword = keywords.find(word); keyword != keywords.end())
{
tokens.push_back({ keyword->second, word });
}
else
{
tokens.push_back({ Token::IDENTIFIER, word });
}
}
// Operators will eventually go here
// If it is here and not whitespace that means it's an invalid character
if (current == ' ' || current == '\t' || current == '\r' || current == '\n')
{
// Updates trackers
wasLastCharAlpha = isAlpha;
ITERATE;
}
// Throws an error to alert the user
throw InvalidCharInSource(index, current);
}
// Logs the tokens if logging is on //
if (log != nullptr)
{
for (auto& token : tokens)
{
SafeLog(log, ToString(token.type), ":\t", token.contents);
}
}
// Shrinks the vector down to minimum size before returning to avoid excess memory being allocated
tokens.shrink_to_fit();
return tokens;
}
}

11
Lexer/src/Token.cpp Normal file
View File

@@ -0,0 +1,11 @@
#include <Lexer.h>
#include <string>
#include <vector>
namespace LX
{
Token::Token(const TokenType _type, std::string _contents)
: type(_type), contents(_contents)
{}
}

47
Main.cpp
View File

@@ -2,7 +2,9 @@
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
#include <string> #include <string>
#include <memory>
#include <Lexer.h>
#include <Util.h> #include <Util.h>
namespace LX namespace LX
@@ -12,6 +14,7 @@ namespace LX
struct IncorrectCommandLineArgs {}; struct IncorrectCommandLineArgs {};
struct InvalidInputFilePath {}; struct InvalidInputFilePath {};
struct InvalidOutputFilePath {}; struct InvalidOutputFilePath {};
struct InvalidLogFilePath {};
} }
int main(int argc, char** argv) int main(int argc, char** argv)
@@ -19,7 +22,7 @@ int main(int argc, char** argv)
try try
{ {
// Checks there is the correct ammount of arguments // Checks there is the correct ammount of arguments
LX::ThrowIf<LX::IncorrectCommandLineArgs>(argc != 3); LX::ThrowIf<LX::IncorrectCommandLineArgs>((argc == 3 || argc == 4) == false);
// Turns the file paths into the C++ type for handling them // Turns the file paths into the C++ type for handling them
std::filesystem::path inpPath = argv[1]; std::filesystem::path inpPath = argv[1];
@@ -28,18 +31,32 @@ int main(int argc, char** argv)
// Prints the full paths to the console to let the user know compiling is being done // Prints the full paths to the console to let the user know compiling is being done
std::cout << std::filesystem::absolute(inpPath) << " -> " << std::filesystem::absolute(outPath) << std::endl; std::cout << std::filesystem::absolute(inpPath) << " -> " << std::filesystem::absolute(outPath) << std::endl;
// Checks the input file exists // Checks the input file exists and opens it
LX::ThrowIf<LX::InvalidInputFilePath>(std::filesystem::exists(inpPath) == false); LX::ThrowIf<LX::InvalidInputFilePath>(std::filesystem::exists(inpPath) == false);
std::ifstream inpFile(inpPath, std::ios::binary | std::ios::ate); // Opens in binary at the end for microptimisation
LX::ThrowIf<LX::InvalidInputFilePath>(inpFile.is_open() == false);
// Opens / Creates the output file and checks if it is open // Opens / Creates the output file
std::ofstream outFile(outPath); std::ofstream outFile(outPath);
LX::ThrowIf<LX::InvalidOutputFilePath>(outFile.is_open() == false); LX::ThrowIf<LX::InvalidOutputFilePath>(outFile.is_open() == false);
// Opens / Creates the log file
std::unique_ptr<std::ofstream> log = nullptr;
if (argc == 4)
{
log = std::make_unique<std::ofstream>(argv[3]);
LX::ThrowIf<LX::InvalidLogFilePath>(log->is_open() == false);
}
// Create tokens out of the input file
std::vector<LX::Token>tokens = LX::LexicalAnalyze(inpFile, log.get());
} }
catch (LX::IncorrectCommandLineArgs) catch (LX::IncorrectCommandLineArgs)
{ {
// Displays to the console of how to use the program // Displays to the console of how to use the program
std::cout << "\nUsage: [source file] [output file]\n"; std::cout << "\nUsage: [source file] [output file] (optional)[log file]\n";
return 1; return 1;
} }
@@ -47,7 +64,7 @@ int main(int argc, char** argv)
catch (LX::InvalidInputFilePath) catch (LX::InvalidInputFilePath)
{ {
// Tells user the input file could not be found // Tells user the input file could not be found
std::cout << "\nFile path: {" << argv[1] << "} could not be found\n"; std::cout << "\nFile path: {" << argv[1] << "} could not be opened\n";
return 2; return 2;
} }
@@ -55,7 +72,25 @@ int main(int argc, char** argv)
catch (LX::InvalidOutputFilePath) catch (LX::InvalidOutputFilePath)
{ {
// Tells the user the output file could not be opened // Tells the user the output file could not be opened
std::cout << "\nCould not open/create {" << argv[2] << "}"; std::cout << "\nCould not open/create {" << argv[2] << "}\n";
}
catch (LX::InvalidCharInSource& e)
{
//
std::cout << "\nInvalid character found in source file: {" << e.invalid << "} at index: " << e.index << "\n";
}
catch (std::exception& e)
{
// Prints the std exception to the console
std::cout << "\nAn error occured:\n" << e.what() << std::endl;
}
catch (...)
{
// Tells the user if an error has happened
std::cout << "\nAn Error occured\n";
} }
return 0; return 0;

9
build-test/Log.txt Normal file
View File

@@ -0,0 +1,9 @@
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Started lexing file
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Token::FUNCTION: func
Token::IDENTIFIER: main
Token::IDENTIFIER: print
Token::STRING_LITERAL: Hello World!

2
build-test/main.lx
View File

@@ -0,0 +1,2 @@
func main
print "Hello World!"

55
common/Lexer.h Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include <fstream>
#include <vector>
#include <string>
// This file contains everything that is exported from Lexer.lib
// The rest of the items within the Lexer project are internal only
namespace LX
{
struct InvalidCharInSource
{
std::streamsize index;
char invalid;
};
// Data type to store a more computer readable version of files
struct __declspec(novtable) Token final
{
// Enum to hold the type of the token
enum TokenType : short
{
// General tokens //
STRING_LITERAL,
IDENTIFIER,
// Keywords //
FOR, WHILE,
IF, ELSE, ELIF,
FUNCTION,
// You made a mistake somehow //
UNDEFINED = -1
};
// Constructor of the tokens to set their info
Token(const TokenType _type, std::string _contents);
// Contents of the token (may be empty if not needed)
// Const to avoid external changes
const std::string contents;
// Type of the token
// Const to avoid external changes
const TokenType type;
};
// Lexer function to take in a file and output a vector of tokens
const std::vector<Token> LexicalAnalyze(std::ifstream& src, std::ofstream* log);
}

10
common/Util.h
View File

@@ -1,8 +1,18 @@
#pragma once #pragma once
#include <fstream>
namespace LX namespace LX
{ {
template<typename T, typename... Args> template<typename T, typename... Args>
inline void ThrowIf(const bool condition, Args... args) inline void ThrowIf(const bool condition, Args... args)
{ if (condition) [[unlikely]] { throw T(args...); }} { if (condition) [[unlikely]] { throw T(args...); }}
template<typename... Args>
inline void SafeLog(std::ofstream* log, Args... args)
{
if (log != nullptr) { (*log << ... << args); *log << "\n"; }
}
constexpr const char* LOG_BREAK = "\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n";
} }