Pasha/LXC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added lexer tests

Browse Source
This commit is contained in:
Pasha Bibko
2025-07-22 20:46:42 +01:00
parent 829d4c224a
commit 4d6224529f
13 changed files with 288 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@@ -10,13 +10,15 @@ project(LXC_Project LANGUAGES CXX)
# Makes .exes be outputted within the root directory #
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
# Adds the G-Tests #
# Enables testing #
enable_testing()
add_subdirectory(external/googletest)
add_subdirectory(tests)
# Adds the sub-directories of all of the binaries #
add_subdirectory(Lexer)
# The app subdirectory #
add_subdirectory(LXC)
# Compiles the tests #
add_subdirectory(tests)

6
Common/LXC.h
View File

@@ -19,6 +19,6 @@
// LXC util files //
#include <Result.h>
#include <File.h>
#include <IO.h>
#include <modules/Result.h>
#include <modules/File.h>
#include <modules/IO.h>

0
Common/File.h → Common/modules/File.h
View File

2
Common/IO.h → Common/modules/IO.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include <OS.h>
#include <modules/OS.h>
#include <iostream>
#include <cstdlib>

0
Common/OS.h → Common/modules/OS.h
View File

2
Common/Result.h → Common/modules/Result.h
View File

@@ -1,6 +1,6 @@
#pragma once
#include <OS.h>
#include <modules/OS.h>
#include <type_traits>
#include <cstdlib>

2
LXC/LXC.cpp
View File

@@ -9,7 +9,7 @@ int main(int argc, char** argv)
// Creates the debug log //
Util::CreateLog("LXC.log");
std::filesystem::path src = "example/example.lx";
std::filesystem::path src = "examples/Fib.lx";
// Reads the given file to a string //
Util::ReturnVal fileContents = Util::ReadFile(src);

6
Lexer/inc/Token.h
View File

@@ -88,12 +88,6 @@ namespace LXC::Lexer
using T = std::underlying_type_t<TokenType>;
return static_cast<T>(type) & static_cast<T>(mask);
}
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token)
{
using T = std::underlying_type_t<TokenType>;
return static_cast<T>(token.type) & static_cast<T>(mask);
}
// Constructor to set the data of the token for more complex token types //
Token(const LexerContext& ctx, uint32_t start, unsigned short len, TokenType _type);

15
examples/Fib.lx Normal file
View File

@@ -0,0 +1,15 @@
int fib(int num)
{
# Base cases #
if (n == 0) { return 0 }
if (n == 1) { return 1 }
# RECURSION BABYYYY #
return fib(n - 1) + fib(n - 2)
}
int main(void)
{
int res = fib(8)
return res == 21
}

0
example/example.lx → examples/LawsOfMath.lx
View File

13
tests/CMakeLists.txt
View File

@@ -2,10 +2,23 @@
file (GLOB TestSources src/*.cpp inc/*.h)
add_executable(LXC_Tests ${TestSources})
# Creates the shared precompiled header #
target_include_directories(LXC_Tests PRIVATE ${CMAKE_SOURCE_DIR}/Common)
target_precompile_headers(LXC_Tests PRIVATE ${CMAKE_SOURCE_DIR}/Common/LXC.h)
# Includes headers for modules to test #
target_include_directories(LXC_Tests PRIVATE
${CMAKE_SOURCE_DIR}/Lexer/inc
)
# Links with GoogleTest #
target_link_libraries(LXC_Tests
# Testing libraries #
gtest
gtest_main
# Libraries to test #
Lexer
)
# Registers the test #

11
tests/src/LXC_Tests.cpp
View File

@@ -1,11 +0,0 @@
#include <gtest/gtest.h>
TEST(IWantSomeTestsToPass, BasicTest1)
{
EXPECT_EQ(1 + 1, 2);
}
TEST(IWantSomeTestsToPass, BasicTest2)
{
EXPECT_EQ(1 + 2, 2);
}

250
tests/src/LexerTests.cpp Normal file
View File

@@ -0,0 +1,250 @@
#include <gtest/gtest.h>
#include <Lexer.h>
#include <Token.h>
// Local util functions //
namespace LXC::Internal
{
static void ExpectTokens(const Lexer::LexerOutput& tokens, const std::vector<Lexer::Token::TokenType>& expected)
{
ASSERT_EQ(tokens.size(), expected.size());
for (size_t i = 0; i < tokens.size(); i++)
{
EXPECT_EQ(tokens[i].type, expected[i]);
}
}
}
// The tests for the lexer //
namespace LXC::Lexer
{
TEST(LexerTests, ReturnsTrueForMatching)
{
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Add));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Sub));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Mul));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Keyword>(Token::If));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Keyword>(Token::While));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::UserDefined>(Token::StringLiteral));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::UserDefined>(Token::NumLiteral));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Symbols>(Token::CloseBracket));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Symbols>(Token::Comma));
EXPECT_TRUE(Token::IsTypeClass<TokenClass::Misc>(Token::End_of_file));
}
TEST(LexerTests, ReturnsFalseForNonMatching)
{
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::StringLiteral));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::End_of_file));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::If));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Keyword>(Token::NumLiteral));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Keyword>(Token::Comma));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::UserDefined>(Token::Add));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::UserDefined>(Token::CloseBracket));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Symbols>(Token::While));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Symbols>(Token::Mul));
EXPECT_FALSE(Token::IsTypeClass<TokenClass::Misc>(Token::Sub));
}
TEST(LexerTests, EmptyInput)
{
Util::ReturnVal result = TokenizeFile("");
ASSERT_TRUE(result.Suceeded());
EXPECT_TRUE(result.Result().empty());
}
TEST(LexerTests, SingleIdentifier)
{
Util::ReturnVal result = TokenizeFile("hello");
ASSERT_TRUE(result.Suceeded());
Internal::ExpectTokens(result, { Token::Identifier });
}
TEST(LexerTests, SingleNumber)
{
Util::ReturnVal result = TokenizeFile("12345");
ASSERT_TRUE(result.Suceeded());
Internal::ExpectTokens(result, { Token::NumLiteral });
}
TEST(LexerTests, SingleStringLiteral)
{
Util::ReturnVal result = TokenizeFile("\"string literal\"");
ASSERT_TRUE(result.Suceeded());
Internal::ExpectTokens(result, { Token::StringLiteral });
}
TEST(LexerTests, MultipleSymbolsAndOperators)
{
Util::ReturnVal result = TokenizeFile("+ = (");
ASSERT_TRUE(result.Suceeded());
Internal::ExpectTokens(result, { Token::Add, Token::Assign, Token::OpenParen });
}
TEST(LexerTests, WhileTrueTokenTest)
{
Util::ReturnVal result = TokenizeFile("while (true)");
ASSERT_TRUE(result.Suceeded());
Internal::ExpectTokens(result, { Token::While, Token::OpenParen, Token::Identifier, Token::CloseParen });
}
TEST(LexerTests, UnterminatedString)
{
Util::ReturnVal result = TokenizeFile("\"This is supposed to be unterminated");
ASSERT_FALSE(result.Suceeded());
EXPECT_EQ(result.Error().reason, LexerError::UnterminatedStringLiteral);
}
TEST(LexerTests, InvalidCharacter)
{
Util::ReturnVal result = TokenizeFile("^^^");
ASSERT_FALSE(result.Suceeded());
EXPECT_EQ(result.Error().reason, LexerError::InvalidCharacter);
}
TEST(LexerTests, InvalidOperand)
{
Util::ReturnVal result = TokenizeFile("+/*");
ASSERT_FALSE(result.Suceeded());
EXPECT_EQ(result.Error().reason, LexerError::UnknownSymbolOrOperand);
}
TEST(LexerTests, ExampleFile_LawsOfMath)
{
Util::ReturnVal fileContents = Util::ReadFile("examples/LawsOfMath.lx");
ASSERT_TRUE(fileContents.Suceeded());
Util::ReturnVal tokens = TokenizeFile(fileContents);
Internal::ExpectTokens(tokens,
{
Token::Identifier, // int
Token::Identifier, // add
Token::OpenParen, // (
Token::Identifier, // int
Token::Identifier, // a
Token::Comma, // ,
Token::Identifier, // int
Token::Identifier, // b
Token::CloseParen, // )
Token::OpenBrace, // {
Token::Return, // return
Token::Identifier, // a
Token::Add, // +
Token::Identifier, // b
Token::CloseBrace, // }
Token::Identifier, // int
Token::Identifier, // main
Token::OpenParen, // (
Token::Identifier, // void
Token::CloseParen, // )
Token::OpenBrace, // {
Token::Identifier, // int
Token::Identifier, // c
Token::Assign, // =
Token::Identifier, // add
Token::OpenParen, // (
Token::NumLiteral, // 3
Token::Comma, // ,
Token::NumLiteral, // 4
Token::CloseParen, // )
Token::If, // if
Token::OpenParen, // (
Token::Identifier, // c
Token::Eql, // ==
Token::NumLiteral, // 7
Token::CloseParen, // )
Token::OpenBrace, // {
Token::Return, // return
Token::NumLiteral, // 0
Token::CloseBrace, // }
Token::Else, // else
Token::OpenBrace, // {
Token::Return, // return
Token::NumLiteral, // 1
Token::CloseBrace, // }
Token::CloseBrace // }
});
}
TEST(LexerTests, ExampleFile_Fib)
{
Util::ReturnVal fileContents = Util::ReadFile("examples/Fib.lx");
ASSERT_TRUE(fileContents.Suceeded());
Util::ReturnVal tokens = TokenizeFile(fileContents);
Internal::ExpectTokens(tokens,
{
Token::Identifier, // int
Token::Identifier, // fib
Token::OpenParen, // (
Token::Identifier, // int
Token::Identifier, // num
Token::CloseParen, // )
Token::If, // if
Token::OpenParen, // (
Token::Identifier, // n
Token::Eql, // ==
Token::NumLiteral, // 0
Token::OpenBrace, // {
Token::Return, // return
Token::NumLiteral, // 0
Token::CloseBrace, // }
Token::If, // if
Token::OpenParen, // (
Token::Identifier, // n
Token::Eql, // ==
Token::NumLiteral, // 1
Token::OpenBrace, // {
Token::Return, // return
Token::NumLiteral, // 1
Token::CloseBrace, // }
Token::Return, // return
Token::Identifier, // fib
Token::OpenParen, // (
Token::Identifier, // n
Token::Sub, // -
Token::NumLiteral, // 1
Token::CloseParen, // )
Token::Add, // +
Token::Identifier, // fib
Token::OpenParen, // (
Token::Identifier, // n
Token::Sub, // -
Token::NumLiteral, // 1
Token::CloseParen, // )
Token::Identifier, // int
Token::Identifier, // main
Token::OpenParen, // (
Token::Identifier, // void
Token::CloseParen, // )
Token::OpenBrace, // {
Token::Identifier, // int
Token::Identifier, // res
Token::Assign, // =
Token::Identifier, // fib
Token::OpenParen, // (
Token::NumLiteral, // 8
Token::CloseParen, // )
Token::Return, // return
Token::Identifier, // res
Token::Eql, // ==
Token::NumLiteral, // 21
Token::CloseBrace // }
});
}
}