Added basic lexer

Lexer/inc/Token.h

@@ -4,9 +4,6 @@
 
 namespace LXC::Lexer
 {
-	// Foward declaration to allow it passing to the Token class //
-	struct LexerContext;
-
 	namespace TokenClass
 	{
 		// Bitmask for different token classes //
@@ -29,6 +26,8 @@ namespace LXC::Lexer
 		};
 	};
 
+	struct LexerContext;
+	
 	// Data type for storing the output of the lexer //
 	class Token final
 	{
@@ -56,8 +55,7 @@ namespace LXC::Lexer
 				// === User defined === //
 
 				String_Literal	= TokenClass::UserDefined,
-				Int_Literal,
-				Float_Literal,
+				Num_Literal,
 				Identifier,
 
 				// === Symbols === //
@@ -86,8 +84,8 @@ namespace LXC::Lexer
 			template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; }
 			template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; }
 
-			// Constructor to set the data of the token //
-			Token(const LexerContext& context, const unsigned short _length, TokenType _type);
+			// Constructor to set the data of the token for more complex token types //
+			Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type);
 
 			// Deconstructor to clean up the allocated memory //
 			~Token();
@@ -102,15 +100,12 @@ namespace LXC::Lexer
 			// The length of the token //
 			const unsigned short length;
 
-			// The line the token is on (starts on 1) //
-			const unsigned short line;
-
-			// The index on the line (starts on 1) //
-			const unsigned short column;
+			// Start index of the token //
+			const unsigned __int32 index;
 
 		private:
 			// The data of the token //
-			const char* contents;
+			char* contents;
 	};
 
 	// Typedef for the output type of how the Lexer outputs //