Tabs -> Spaces
This commit is contained in:
Pasha Bibko
@@ -4,112 +4,112 @@
|
||||
|
||||
namespace LXC::Lexer
|
||||
{
|
||||
namespace TokenClass
|
||||
{
|
||||
// Bitmask for different token classes //
|
||||
enum ClassMask : unsigned short
|
||||
{
|
||||
// Mathematical and logic operators //
|
||||
Operator = 1 << (1 + 8),
|
||||
namespace TokenClass
|
||||
{
|
||||
// Bitmask for different token classes //
|
||||
enum ClassMask : unsigned short
|
||||
{
|
||||
// Mathematical and logic operators //
|
||||
Operator = 1 << (1 + 8),
|
||||
|
||||
// Special words defined by the compiler //
|
||||
Keyword = 1 << (2 + 8),
|
||||
// Special words defined by the compiler //
|
||||
Keyword = 1 << (2 + 8),
|
||||
|
||||
// Words such as literals and identifiers //
|
||||
UserDefined = 1 << (3 + 8),
|
||||
// Words such as literals and identifiers //
|
||||
UserDefined = 1 << (3 + 8),
|
||||
|
||||
// Symbols in the source like (? , . ! <) //
|
||||
Symbols = 1 << (4 + 8),
|
||||
// Symbols in the source like (? , . ! <) //
|
||||
Symbols = 1 << (4 + 8),
|
||||
|
||||
// Tokens not defined by previous classes //
|
||||
Misc = 1 << (5 + 8)
|
||||
};
|
||||
};
|
||||
// Tokens not defined by previous classes //
|
||||
Misc = 1 << (5 + 8)
|
||||
};
|
||||
};
|
||||
|
||||
struct LexerContext;
|
||||
|
||||
// Data type for storing the output of the lexer //
|
||||
class Token final
|
||||
{
|
||||
public:
|
||||
// Enum of token type organised by their token class //
|
||||
enum TokenType : unsigned short
|
||||
{
|
||||
// === Operators === //
|
||||
struct LexerContext;
|
||||
|
||||
// Data type for storing the output of the lexer //
|
||||
class Token final
|
||||
{
|
||||
public:
|
||||
// Enum of token type organised by their token class //
|
||||
enum TokenType : unsigned short
|
||||
{
|
||||
// === Operators === //
|
||||
|
||||
Add = TokenClass::Operator,
|
||||
Sub,
|
||||
Mul,
|
||||
Div,
|
||||
Mod,
|
||||
Add = TokenClass::Operator,
|
||||
Sub,
|
||||
Mul,
|
||||
Div,
|
||||
Mod,
|
||||
|
||||
// === Keywords === //
|
||||
// === Keywords === //
|
||||
|
||||
For = TokenClass::Keyword,
|
||||
While,
|
||||
If,
|
||||
ElseIf,
|
||||
Else,
|
||||
Return,
|
||||
For = TokenClass::Keyword,
|
||||
While,
|
||||
If,
|
||||
ElseIf,
|
||||
Else,
|
||||
Return,
|
||||
|
||||
// === User defined === //
|
||||
// === User defined === //
|
||||
|
||||
StringLiteral = TokenClass::UserDefined,
|
||||
NumLiteral,
|
||||
Identifier,
|
||||
StringLiteral = TokenClass::UserDefined,
|
||||
NumLiteral,
|
||||
Identifier,
|
||||
|
||||
// === Symbols === //
|
||||
// === Symbols === //
|
||||
|
||||
Assign = TokenClass::Symbols,
|
||||
Assign = TokenClass::Symbols,
|
||||
|
||||
CloseBracket,
|
||||
OpenBracket,
|
||||
CloseBracket,
|
||||
OpenBracket,
|
||||
|
||||
CloseBrace,
|
||||
OpenBrace,
|
||||
CloseBrace,
|
||||
OpenBrace,
|
||||
|
||||
CloseParen,
|
||||
OpenParen,
|
||||
CloseParen,
|
||||
OpenParen,
|
||||
|
||||
Comma,
|
||||
Comma,
|
||||
|
||||
// === Misc === //
|
||||
// === Misc === //
|
||||
|
||||
End_of_file = TokenClass::Misc,
|
||||
End_of_file = TokenClass::Misc,
|
||||
|
||||
UNDEFINED = 65535 // Invalid token type (max number)
|
||||
};
|
||||
UNDEFINED = 65535 // Invalid token type (max number)
|
||||
};
|
||||
|
||||
// Util function calculating wether a token is of a given class //
|
||||
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; }
|
||||
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; }
|
||||
// Util function calculating wether a token is of a given class //
|
||||
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(TokenType type) { return type & mask; }
|
||||
template<TokenClass::ClassMask mask> static constexpr bool IsTypeClass(Token token) { return token.type & mask; }
|
||||
|
||||
// Constructor to set the data of the token for more complex token types //
|
||||
Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type);
|
||||
// Constructor to set the data of the token for more complex token types //
|
||||
Token(const LexerContext& ctx, unsigned __int32 start, unsigned short len, TokenType _type);
|
||||
|
||||
// Deconstructor to clean up the allocated memory //
|
||||
~Token();
|
||||
// Deconstructor to clean up the allocated memory //
|
||||
~Token();
|
||||
|
||||
// Getters for the c-string to stop it being reassigned (or deleted) //
|
||||
inline const char* const Str() const { return contents; }
|
||||
// Getters for the c-string to stop it being reassigned (or deleted) //
|
||||
inline const char* const Str() const { return contents; }
|
||||
|
||||
// Outputs all the relevant infomration in a string for logging purposes //
|
||||
std::string LogStr() const;
|
||||
// Outputs all the relevant infomration in a string for logging purposes //
|
||||
std::string LogStr() const;
|
||||
|
||||
// The type of the token //
|
||||
const TokenType type;
|
||||
// The type of the token //
|
||||
const TokenType type;
|
||||
|
||||
// The length of the token //
|
||||
const unsigned short length;
|
||||
// The length of the token //
|
||||
const unsigned short length;
|
||||
|
||||
// Start index of the token //
|
||||
const unsigned __int32 index;
|
||||
// Start index of the token //
|
||||
const unsigned __int32 index;
|
||||
|
||||
private:
|
||||
// The data of the token //
|
||||
char* contents;
|
||||
};
|
||||
private:
|
||||
// The data of the token //
|
||||
char* contents;
|
||||
};
|
||||
|
||||
// Typedef for the output type of how the Lexer outputs //
|
||||
typedef std::vector<Token> LexerOutput;
|
||||
// Typedef for the output type of how the Lexer outputs //
|
||||
typedef std::vector<Token> LexerOutput;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user