Added lexer tests

CMakeLists.txt

@@ -10,13 +10,15 @@ project(LXC_Project LANGUAGES CXX)
 # Makes .exes be outputted within the root directory #
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
 
-# Adds the G-Tests #
+# Enables testing #
 enable_testing()
 add_subdirectory(external/googletest)
-add_subdirectory(tests)
 
 # Adds the sub-directories of all of the binaries #
 add_subdirectory(Lexer)
 
 # The app subdirectory #
 add_subdirectory(LXC)
+
+# Compiles the tests #
+add_subdirectory(tests)

Common/LXC.h

@@ -19,6 +19,6 @@
 
 // LXC util files //
 
-#include <Result.h>
+#include <modules/Result.h>
-#include <File.h>
+#include <modules/File.h>
-#include <IO.h>
+#include <modules/IO.h>

Common/modules/IO.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <OS.h>
+#include <modules/OS.h>
 
 #include <iostream>
 #include <cstdlib>

Common/modules/Result.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <OS.h>
+#include <modules/OS.h>
 
 #include <type_traits>
 #include <cstdlib>

LXC/LXC.cpp

@@ -9,7 +9,7 @@ int main(int argc, char** argv)
     // Creates the debug log //
     Util::CreateLog("LXC.log");
 
-    std::filesystem::path src = "example/example.lx";
+    std::filesystem::path src = "examples/Fib.lx";
 
     // Reads the given file to a string //
     Util::ReturnVal fileContents = Util::ReadFile(src);

Lexer/inc/Token.h

@@ -89,12 +89,6 @@ namespace LXC::Lexer
 	            return static_cast<T>(type) & static_cast<T>(mask);
 	        }
 
-	        template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token)
-	        {
-		        using T = std::underlying_type_t<TokenType>;
-	            return static_cast<T>(token.type) & static_cast<T>(mask);
-	        }
-
             // Constructor to set the data of the token for more complex token types //
             Token(const LexerContext& ctx, uint32_t start, unsigned short len, TokenType _type);
 

examples/Fib.lx

@@ -0,0 +1,15 @@
+int fib(int num)
+{
+	# Base cases #
+	if (n == 0) { return 0 }
+	if (n == 1) { return 1 }
+
+	# RECURSION BABYYYY #
+	return fib(n - 1) + fib(n - 2)
+}
+
+int main(void)
+{
+	int res = fib(8)
+	return res == 21
+}

tests/CMakeLists.txt

@@ -2,10 +2,23 @@
 file (GLOB TestSources src/*.cpp inc/*.h)
 add_executable(LXC_Tests ${TestSources})
 
+# Creates the shared precompiled header #
+target_include_directories(LXC_Tests PRIVATE ${CMAKE_SOURCE_DIR}/Common)
+target_precompile_headers(LXC_Tests PRIVATE ${CMAKE_SOURCE_DIR}/Common/LXC.h)
+
+# Includes headers for modules to test #
+target_include_directories(LXC_Tests PRIVATE
+	${CMAKE_SOURCE_DIR}/Lexer/inc
+)
+
 # Links with GoogleTest #
 target_link_libraries(LXC_Tests
+	# Testing libraries #
 	gtest
 	gtest_main
+
+	# Libraries to test #
+	Lexer
 )
 
 # Registers the test #

tests/src/LXC_Tests.cpp

@@ -1,11 +0,0 @@
-#include <gtest/gtest.h>
-
-TEST(IWantSomeTestsToPass, BasicTest1)
-{
-    EXPECT_EQ(1 + 1, 2);
-}
-
-TEST(IWantSomeTestsToPass, BasicTest2)
-{
-    EXPECT_EQ(1 + 2, 2);
-}

tests/src/LexerTests.cpp

@@ -0,0 +1,250 @@
+#include <gtest/gtest.h>
+
+#include <Lexer.h>
+#include <Token.h>
+
+// Local util functions //
+namespace LXC::Internal
+{
+    static void ExpectTokens(const Lexer::LexerOutput& tokens, const std::vector<Lexer::Token::TokenType>& expected)
+    {
+        ASSERT_EQ(tokens.size(), expected.size());
+        for (size_t i = 0; i < tokens.size(); i++)
+        {
+            EXPECT_EQ(tokens[i].type, expected[i]);
+        }
+    }
+}
+
+// The tests for the lexer //
+namespace LXC::Lexer
+{
+    TEST(LexerTests, ReturnsTrueForMatching)
+    {
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Add));
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Sub));
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Operator>(Token::Mul));
+
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Keyword>(Token::If));
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Keyword>(Token::While));
+
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::UserDefined>(Token::StringLiteral));
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::UserDefined>(Token::NumLiteral));
+
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Symbols>(Token::CloseBracket));
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Symbols>(Token::Comma));
+
+        EXPECT_TRUE(Token::IsTypeClass<TokenClass::Misc>(Token::End_of_file));
+    }
+
+    TEST(LexerTests, ReturnsFalseForNonMatching)
+    {
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::StringLiteral));
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::End_of_file));
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Operator>(Token::If));
+
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Keyword>(Token::NumLiteral));
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Keyword>(Token::Comma));
+
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::UserDefined>(Token::Add));
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::UserDefined>(Token::CloseBracket));
+
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Symbols>(Token::While));
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Symbols>(Token::Mul));
+
+        EXPECT_FALSE(Token::IsTypeClass<TokenClass::Misc>(Token::Sub));
+    }
+
+    TEST(LexerTests, EmptyInput)
+    {
+        Util::ReturnVal result = TokenizeFile("");
+        ASSERT_TRUE(result.Suceeded());
+        EXPECT_TRUE(result.Result().empty());
+    }
+
+    TEST(LexerTests, SingleIdentifier)
+    {
+        Util::ReturnVal result = TokenizeFile("hello");
+        ASSERT_TRUE(result.Suceeded());
+        Internal::ExpectTokens(result, { Token::Identifier });
+    }
+
+    TEST(LexerTests, SingleNumber)
+    {
+        Util::ReturnVal result = TokenizeFile("12345");
+        ASSERT_TRUE(result.Suceeded());
+        Internal::ExpectTokens(result, { Token::NumLiteral });
+    }
+
+    TEST(LexerTests, SingleStringLiteral)
+    {
+        Util::ReturnVal result = TokenizeFile("\"string literal\"");
+        ASSERT_TRUE(result.Suceeded());
+        Internal::ExpectTokens(result, { Token::StringLiteral });
+    }
+
+    TEST(LexerTests, MultipleSymbolsAndOperators)
+    {
+        Util::ReturnVal result = TokenizeFile("+ = (");
+        ASSERT_TRUE(result.Suceeded());
+        Internal::ExpectTokens(result, { Token::Add, Token::Assign, Token::OpenParen });
+    }
+
+    TEST(LexerTests, WhileTrueTokenTest)
+    {
+        Util::ReturnVal result = TokenizeFile("while (true)");
+        ASSERT_TRUE(result.Suceeded());
+        Internal::ExpectTokens(result, { Token::While, Token::OpenParen, Token::Identifier, Token::CloseParen });
+    }
+
+    TEST(LexerTests, UnterminatedString)
+    {
+        Util::ReturnVal result = TokenizeFile("\"This is supposed to be unterminated");
+        ASSERT_FALSE(result.Suceeded());
+        EXPECT_EQ(result.Error().reason, LexerError::UnterminatedStringLiteral);
+    }
+
+    TEST(LexerTests, InvalidCharacter)
+    {
+        Util::ReturnVal result = TokenizeFile("^^^");
+        ASSERT_FALSE(result.Suceeded());
+        EXPECT_EQ(result.Error().reason, LexerError::InvalidCharacter);
+    }
+
+    TEST(LexerTests, InvalidOperand)
+    {
+        Util::ReturnVal result = TokenizeFile("+/*");
+        ASSERT_FALSE(result.Suceeded());
+        EXPECT_EQ(result.Error().reason, LexerError::UnknownSymbolOrOperand);
+    }
+
+    TEST(LexerTests, ExampleFile_LawsOfMath)
+    {
+        Util::ReturnVal fileContents = Util::ReadFile("examples/LawsOfMath.lx");
+        ASSERT_TRUE(fileContents.Suceeded());
+        Util::ReturnVal tokens = TokenizeFile(fileContents);
+        Internal::ExpectTokens(tokens,
+        {
+            Token::Identifier, // int
+            Token::Identifier, // add
+            Token::OpenParen,  // (
+            Token::Identifier, // int
+            Token::Identifier, // a
+            Token::Comma,      // ,
+            Token::Identifier, // int
+            Token::Identifier, // b
+            Token::CloseParen, // )
+            Token::OpenBrace,  // {
+            Token::Return,     // return
+            Token::Identifier, // a
+            Token::Add,        // +
+            Token::Identifier, // b
+            Token::CloseBrace, // }
+
+            Token::Identifier, // int
+            Token::Identifier, // main
+            Token::OpenParen,  // (
+            Token::Identifier, // void
+            Token::CloseParen, // )
+            Token::OpenBrace,  // {
+            Token::Identifier, // int
+            Token::Identifier, // c
+            Token::Assign,     // =
+            Token::Identifier, // add
+            Token::OpenParen,  // (
+            Token::NumLiteral, // 3
+            Token::Comma,      // ,
+            Token::NumLiteral, // 4
+            Token::CloseParen, // )
+            Token::If,         // if
+            Token::OpenParen,  // (
+            Token::Identifier, // c
+            Token::Eql,        // ==
+            Token::NumLiteral, // 7
+            Token::CloseParen, // )
+            Token::OpenBrace,  // {
+            Token::Return,     // return
+            Token::NumLiteral, // 0
+            Token::CloseBrace, // }
+            Token::Else,       // else
+            Token::OpenBrace,  // {
+            Token::Return,     // return
+            Token::NumLiteral, // 1
+            Token::CloseBrace, // }
+            Token::CloseBrace  // }
+        });
+    }
+
+    TEST(LexerTests, ExampleFile_Fib)
+    {
+        Util::ReturnVal fileContents = Util::ReadFile("examples/Fib.lx");
+        ASSERT_TRUE(fileContents.Suceeded());
+        Util::ReturnVal tokens = TokenizeFile(fileContents);
+        Internal::ExpectTokens(tokens,
+        {
+            Token::Identifier, // int
+            Token::Identifier, // fib
+            Token::OpenParen,  // (
+            Token::Identifier, // int
+            Token::Identifier, // num
+            Token::CloseParen, // )
+
+            Token::If,         // if
+            Token::OpenParen,  // (
+            Token::Identifier, // n
+            Token::Eql,        // ==
+            Token::NumLiteral, // 0
+            Token::OpenBrace,  // {
+            Token::Return,     // return
+            Token::NumLiteral, // 0
+            Token::CloseBrace, // }
+
+            Token::If,         // if
+            Token::OpenParen,  // (
+            Token::Identifier, // n
+            Token::Eql,        // ==
+            Token::NumLiteral, // 1
+            Token::OpenBrace,  // {
+            Token::Return,     // return
+            Token::NumLiteral, // 1
+            Token::CloseBrace, // }
+
+            Token::Return,     // return
+            Token::Identifier, // fib
+            Token::OpenParen,  // (
+            Token::Identifier, // n
+            Token::Sub,        // -
+            Token::NumLiteral, // 1
+            Token::CloseParen, // )
+
+            Token::Add,        // +
+
+            Token::Identifier, // fib
+            Token::OpenParen,  // (
+            Token::Identifier, // n
+            Token::Sub,        // -
+            Token::NumLiteral, // 1
+            Token::CloseParen, // )
+
+            Token::Identifier, // int
+            Token::Identifier, // main
+            Token::OpenParen,  // (
+            Token::Identifier, // void
+            Token::CloseParen, // )
+
+            Token::OpenBrace,  // {
+            Token::Identifier, // int
+            Token::Identifier, // res
+            Token::Assign,     // =
+            Token::Identifier, // fib
+            Token::OpenParen,  // (
+            Token::NumLiteral, // 8
+            Token::CloseParen, // )
+            Token::Return,     // return
+            Token::Identifier, // res
+            Token::Eql,        // ==
+            Token::NumLiteral, // 21
+            Token::CloseBrace  // }
+        });
+    }
+}