Added the token class

Lexer/src/Token.cpp

@@ -0,0 +1,37 @@
+#include <LXC.h>
+
+#include <Lexer.h>
+#include <Token.h>
+
+namespace LXC::Lexer
+{
+	static const char* const CopySubstrToMem(const LexerContext& context, const size_t length, Token::TokenType type) 
+	{
+		// Only user defined class tokens need to store their type //
+		if (!Token::IsTypeClass<TokenClass::UserDefined>(type))
+			return nullptr;
+
+		// Copies the memory to a c-string //
+		char* cStr = new char[length + 1];
+		std::memcpy(cStr, context.source.data() + context.index, length);
+		cStr[length] = '\0';
+
+		return cStr;
+	}
+
+	// Constructor to assign the members of the token class //
+	Token::Token(const LexerContext& context, const unsigned short _length, TokenType _type) :
+		type(_type), length(_length), line(context.line), column(context.column),
+		contents(CopySubstrToMem(context, _length, _type))
+	{}
+
+	// Destructor to clean up the memory of the token that can be allocated //
+	Token::~Token()
+	{
+		// Frees any allocated memory //
+		if (contents != nullptr)
+			delete[] contents;
+
+		contents = nullptr;
+	}
+}