Improved logging

Small errors with logging parser unexpected token errors. Tokens need to be re-written with how they store memory as it can be halved.

common/Lexer.h

@@ -9,13 +9,9 @@ namespace std
 	template<typename T1>
 	struct char_traits;
 
-	template<typename T1, typename T2>
-	class basic_ifstream;
-
 	template<typename T1, typename T2>
 	class basic_ofstream;
 
-	using ifstream = basic_ifstream<char, char_traits<char>>;
 	using ofstream = basic_ofstream<char, char_traits<char>>;
 }
 
@@ -27,14 +23,42 @@ namespace LX
 	// Error type with index and character to alert the user that LX does not understand that symbol //
 	struct InvalidCharInSource
 	{
-		std::streamsize index;
+		std::streamsize col;
 		std::streamsize line;
-
-		std::string lineContents;
+		std::streamsize index;
 
 		char invalid;
 	};
 
+	// Struct to store the current information of the lexer //
+	struct LexerInfo
+	{
+		// Current trackers of where in the source it is //
+
+		std::streamsize line = 1; // <- Lines start on 1 (probably because of non-programmer's)
+		std::streamsize index = 0;
+		std::streamsize column = 0; // <- Columns start on 1 (probably because of non-programmer's)
+
+		// Trackers for when a multi-char token started //
+
+		std::streamsize startOfWord = 0;
+		std::streamsize startOfNumberLiteral = 0;
+		std::streamsize startOfStringLiteral = 0;
+
+		// Different flags of the lexer //
+		// Stored as a bitset to minimse memory allocated (basically no difference, because only one exists at any given time) //
+
+		bool isAlpha : 1 = false;
+		bool isNumeric : 1 = false;
+		bool inComment : 1 = false;
+		bool inStringLiteral : 1 = false;
+		bool isNextCharAlpha : 1 = false;
+		bool isNextCharNumeric : 1 = false;
+		bool wasLastCharAlpha : 1 = false;
+		bool wasLastCharNumeric : 1 = false;
+		bool lexingNumber : 1 = false;
+	};
+
 	// Data type to store a more computer readable version of files
 	struct __declspec(novtable) Token final
 	{
@@ -65,7 +89,7 @@ namespace LX
 		};
 
 		// Constructor of the tokens to set their info //
-		Token(const TokenType _type, std::string _contents, std::streamsize _line, std::streamsize _index, std::streamsize _length);
+		Token(const TokenType _type, const LexerInfo& info, std::string _contents, std::streamsize _length);
 
 		// Contents of the token (may be empty if not needed) //
 		// Const to avoid external changes //
@@ -74,17 +98,23 @@ namespace LX
 		// Type of the token //
 		// Const to avoid external changes //
 		const TokenType type;
-
-		// The line where the token is located in the source //
-		const std::streamsize line;
 		
-		// Index on the line where the token starts //
+		// Index in the source of the token //
 		const std::streamsize index;
 
 		// The length of the token on the line, may be different to the length of contents //
 		const std::streamsize length;
+
+		// The line the token is located on //
+		const std::streamsize line;
+
+		// The column on the line where it is located //
+		const std::streamsize column;
 	};
 
+	// Logging function to turn a tokentype enum val into it's string //
+	std::string ToString(Token::TokenType t);
+	
 	// Lexer function to take in a file and output a vector of tokens //
-	const std::vector<Token> LexicalAnalyze(std::ifstream& src, std::ofstream* log);
+	const std::vector<Token> LexicalAnalyze(const std::string& contents, const std::streamsize len, std::ofstream* log);
 }

common/Parser.h

@@ -61,6 +61,20 @@ namespace LX
 	// Thrown if there was an error during IR Generation //
 	struct IRGenerationError {};
 
+	// Thrown if there was an unexpected (incorrect) token //
+	struct UnexpectedToken
+	{
+		// The token type that should be there //
+		Token::TokenType expected;
+
+		// If there are multiple expected types there is an option for a custom message //
+		std::string override;
+
+		// What token was actually at that position //
+		// Stored as Token not TokenType to store the location of it within the source //
+		Token got;
+	};
+
 	// Holds all needed info about a function //
 	// Currently only holds the body but in the future will hold: name, params, namespace/class-member
 	struct FunctionDefinition

common/Util.h

@@ -2,6 +2,9 @@
 
 #include <fstream>
 
+// Defining this is only if you are at the point where you should be using a debugger //
+#define LOG_EVERYTHING
+
 namespace LX
 {
 	template<typename T, typename... Args>