From 8f75e52a0749f557f731f971e01151bb3d6db610 Mon Sep 17 00:00:00 2001
From: Pasha Bibko <156938226+PashaBibko@users.noreply.github.com>
Date: Sun, 20 Jul 2025 13:49:10 +0100
Subject: [PATCH] Added foundation for Lexer

---
 Common/LXC.h        |  2 +-
 Common/Result.h     |  4 +++-
 LXC/LXC.cpp         | 10 +++++++++-
 Lexer/inc/Lexer.h   | 14 +++++++++++++-
 Lexer/inc/Token.h   |  3 +++
 Lexer/src/Lexer.cpp | 18 ++++++++++++++++++
 6 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/Common/LXC.h b/Common/LXC.h
index d0b131f..019a0fe 100644
--- a/Common/LXC.h
+++ b/Common/LXC.h
@@ -2,7 +2,7 @@
 
 // Standard libraries //
 
-#include <iostream>
+#include <vector>
 
 // LXC util files //
 
diff --git a/Common/Result.h b/Common/Result.h
index a5a3f5c..f8e3076 100644
--- a/Common/Result.h
+++ b/Common/Result.h
@@ -9,7 +9,9 @@ namespace LXC::Util
 	template<typename ErrorType> struct FunctionFail
 	{
 		explicit FunctionFail(ErrorType _err)
-			: error(_err) {}
+			: error(_err)
+		{
+		}
 
 		ErrorType error;
 	};
diff --git a/LXC/LXC.cpp b/LXC/LXC.cpp
index cdd8f2a..47cec3c 100644
--- a/LXC/LXC.cpp
+++ b/LXC/LXC.cpp
@@ -7,7 +7,7 @@ int main(int argc, char** argv)
 	using namespace LXC;
 
 	// Reads the given file to a string //
-	Util::ReturnVal fileContents = Util::ReadFile("example/example.lx");
+	Util::ReturnVal fileContents = Util::ReadFile("example/example.jk.lx");
 	if (fileContents.Failed())
 	{
 		// Stores the error for easier access //
@@ -26,5 +26,13 @@ int main(int argc, char** argv)
 		return -1;
 	}
 
+	// Turns the file contents into a vector of tokens //
+	Util::ReturnVal tokens = Lexer::TokenizeFile(fileContents);
+	if (tokens.Failed())
+	{
+		// Returns with default error code //
+		return -1;
+	}
+
 	return 0;
 }
diff --git a/Lexer/inc/Lexer.h b/Lexer/inc/Lexer.h
index a9e844f..1b4e5d3 100644
--- a/Lexer/inc/Lexer.h
+++ b/Lexer/inc/Lexer.h
@@ -6,12 +6,24 @@ namespace LXC::Lexer
 {
 	struct LexerContext
 	{
+		// Constructor to set the information of the context //
+		LexerContext(const std::string& _source);
+
 		// Trackers for the Lexer itself //
-		std::string source;
+		const std::string& source;
 		size_t index;
 
+		LexerOutput out;
+		const size_t len;
+
 		// Trackers for where the Lexer is within the user version of source //
 		unsigned short column;
 		unsigned short line;
 	};
+
+	struct LexerError
+	{};
+
+	// Turns a file into a vector of tokens //
+	Util::ReturnVal<LexerOutput, LexerError> TokenizeFile(const std::string& fileContents);
 }
diff --git a/Lexer/inc/Token.h b/Lexer/inc/Token.h
index 8c89f81..829ba9b 100644
--- a/Lexer/inc/Token.h
+++ b/Lexer/inc/Token.h
@@ -112,4 +112,7 @@ namespace LXC::Lexer
 			// The data of the token //
 			const char* contents;
 	};
+
+	// Typedef for the output type of how the Lexer outputs //
+	typedef std::vector<Token> LexerOutput;
 }
diff --git a/Lexer/src/Lexer.cpp b/Lexer/src/Lexer.cpp
index 21a5932..4dda93e 100644
--- a/Lexer/src/Lexer.cpp
+++ b/Lexer/src/Lexer.cpp
@@ -5,4 +5,22 @@
 
 namespace LXC::Lexer
 {
+	LexerContext::LexerContext(const std::string& _source) :
+		source(_source), index(0), out{}, len(_source.length()), column(0), line(0)
+	{}
+
+	Util::ReturnVal<LexerOutput, LexerError> TokenizeFile(const std::string& fileContents)
+	{
+		// Creates the context for the lexer //
+		LexerContext context(fileContents);
+
+		while (context.index > context.len)
+		{
+			// Iterates to the next index //
+			context.column++;
+			context.index++;
+		}
+
+		return context.out;
+	}
 }