Added basic lexer

Lexer/src/Token.cpp

@@ -5,25 +5,19 @@
 
 namespace LXC::Lexer
 {
-	static const char* const CopySubstrToMem(const LexerContext& context, const size_t length, Token::TokenType type) 
-	{
-		// Only user defined class tokens need to store their type //
-		if (!Token::IsTypeClass<TokenClass::UserDefined>(type))
-			return nullptr;
-
-		// Copies the memory to a c-string //
-		char* cStr = new char[length + 1];
-		std::memcpy(cStr, context.source.data() + context.index, length);
-		cStr[length] = '\0';
-
-		return cStr;
-	}
-
 	// Constructor to assign the members of the token class //
-	Token::Token(const LexerContext& context, const unsigned short _length, TokenType _type) :
-		type(_type), length(_length), line(context.line), column(context.column),
-		contents(CopySubstrToMem(context, _length, _type))
-	{}
+	Token::Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type) :
+		type(_type), length(len), index(start), contents(nullptr)
+	{
+		// Only user defined class tokens need to store c-string //
+		if (Token::IsTypeClass<TokenClass::UserDefined>(type))
+		{
+			// Copies the memory to a c-string //
+			contents = new char[len + 1]; // +1 for null terminator
+			std::memcpy(contents, ctx.source.data() + start, len);
+			contents[len] = '\0';
+		}
+	}
 
 	// Destructor to clean up the memory of the token that can be allocated //
 	Token::~Token()