#include #include #include #include namespace PashaBibko::LXC::Lexer { // Constructor to assign the members of the token class // Token::Token(const LexerContext& ctx, const uint32_t start, unsigned short len, TokenType _type) : type(_type), length(len), index(start), contents(nullptr) { // Only user defined class tokens need to store c-string // if (Token::IsTypeClass(type)) { // Copies the memory to a c-string // contents = new char[len + 1]; // +1 for null terminator std::memcpy(contents, ctx.source.data() + start, len); contents[len] = '\0'; } } // Copy constructor // Token::Token(const Token& other) : type(other.type), length(other.length), index(other.index), contents(nullptr) { if (other.contents != nullptr) { size_t len = std::strlen(other.contents) + 1; // Adds one for null-terminator contents = new char[len]; std::memcpy(contents, other.contents, len); } } // Move constructor (transfers memory allocated) // Token::Token(Token&& other) noexcept : type(other.type), length(other.length), index(other.index), contents(other.contents) { // Stops the other from thinking it owns the memory // other.contents = nullptr; } // Destructor to clean up the memory of the token that can be allocated // Token::~Token() { // Frees any allocated memory // if (contents != nullptr) _UNLIKELY { delete[] contents; contents = nullptr; } } // Helper macro for converting type to string // #define TOKEN_TYPE_CASE(type) case type: return #type; static constexpr const char* TokenTypeToCStr(Token::TokenType type) { switch (type) { // All the different types of tokens // TOKEN_TYPE_CASE(Token::Add); TOKEN_TYPE_CASE(Token::Sub); TOKEN_TYPE_CASE(Token::Mul); TOKEN_TYPE_CASE(Token::Div); TOKEN_TYPE_CASE(Token::Mod); TOKEN_TYPE_CASE(Token::Eql); TOKEN_TYPE_CASE(Token::For); TOKEN_TYPE_CASE(Token::While); TOKEN_TYPE_CASE(Token::If); TOKEN_TYPE_CASE(Token::ElseIf); TOKEN_TYPE_CASE(Token::Else); TOKEN_TYPE_CASE(Token::Return); TOKEN_TYPE_CASE(Token::FunctionDef); TOKEN_TYPE_CASE(Token::StringLiteral); TOKEN_TYPE_CASE(Token::NumLiteral); TOKEN_TYPE_CASE(Token::Identifier); TOKEN_TYPE_CASE(Token::Assign); TOKEN_TYPE_CASE(Token::Colon); TOKEN_TYPE_CASE(Token::CloseBracket); TOKEN_TYPE_CASE(Token::OpenBracket); TOKEN_TYPE_CASE(Token::CloseBrace); TOKEN_TYPE_CASE(Token::OpenBrace); TOKEN_TYPE_CASE(Token::CloseParen); TOKEN_TYPE_CASE(Token::OpenParen); TOKEN_TYPE_CASE(Token::CloseCrocodile); TOKEN_TYPE_CASE(Token::OpenCrocodile); TOKEN_TYPE_CASE(Token::Comma); TOKEN_TYPE_CASE(Token::End_of_file); TOKEN_TYPE_CASE(Token::UNDEFINED); // When the case has not been defined yet // default: return "UNKNOWN"; } } std::string LXC::Lexer::Token::LogStr() const { // Output stream to log to // std::ostringstream os; os << std::setw(25) << std::left << TokenTypeToCStr(type) << " | "; // Prints the contents if they are not null // if (contents != nullptr) os << std::setw(25) << std::left << std::string('"' + std::string(contents) + '"'); else os << std::setw(25) << std::left << "EMPTY"; return os.str(); } }