diff options
Diffstat (limited to 'SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp')
| -rw-r--r-- | SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp | 222 |
1 files changed, 72 insertions, 150 deletions
diff --git a/SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp b/SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp index f6a23a0..bf5185b 100644 --- a/SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp +++ b/SQLiteStudio3/coreSQLiteStudio/parser/lexer.cpp @@ -2,21 +2,18 @@ #include "keywords.h" #include "log.h" #include "lexer_low_lev.h" -#include "sqlite2_parse.h" #include "sqlite3_parse.h" #include "common/utils_sql.h" #include <QString> #include <QMultiHash> #include <QDebug> -QHash<Token::Type,QSet<TokenPtr> > Lexer::everyTokenType2; QHash<Token::Type,QSet<TokenPtr> > Lexer::everyTokenType3; QHash<Token*,TokenPtr> Lexer::everyTokenTypePtrMap; -TokenPtr Lexer::semicolonTokenSqlite2; TokenPtr Lexer::semicolonTokenSqlite3; -Lexer::Lexer(Dialect dialect) - : dialect(dialect), sqlToTokenize(QString::null) +Lexer::Lexer() + : sqlToTokenize(QString()) { } @@ -25,7 +22,7 @@ Lexer::~Lexer() cleanUp(); } -TokenList Lexer::tokenize(const QString &sql) +TokenList Lexer::process(const QString &sql) { TokenList resultList; int lgt; @@ -40,7 +37,7 @@ TokenList Lexer::tokenize(const QString &sql) else token = TokenPtr::create(); - lgt = lexerGetToken(str, token, dialect == Dialect::Sqlite2 ? 2 : 3, tolerant); + lgt = lexerGetToken(str, token, 3, tolerant); if (lgt == 0) break; @@ -73,7 +70,7 @@ TokenPtr Lexer::getToken() else token = TokenPtr::create(); - int lgt = lexerGetToken(sqlToTokenize, token, dialect == Dialect::Sqlite2 ? 2 : 3, tolerant); + int lgt = lexerGetToken(sqlToTokenize, token, 3, tolerant); if (lgt == 0) return TokenPtr(); @@ -110,9 +107,7 @@ QSet<TokenPtr> Lexer::getEveryTokenType(QSet<Token::Type> types) { // Process set of types QSet<TokenPtr> results; - QHashIterator<Token::Type,QSet<TokenPtr> > i( - dialect == Dialect::Sqlite2 ? everyTokenType2 : everyTokenType3 - ); + QHashIterator<Token::Type,QSet<TokenPtr> > i(everyTokenType3); while (i.hasNext()) { i.next(); @@ -130,149 +125,79 @@ bool Lexer::isEnd() const return sqlToTokenize.isEmpty(); } -TokenPtr Lexer::getSemicolonToken(Dialect dialect) +TokenPtr Lexer::getSemicolonToken() { - return (dialect == Dialect::Sqlite3) ? semicolonTokenSqlite3 : semicolonTokenSqlite2; + return semicolonTokenSqlite3; } void Lexer::staticInit() { - createTokenType(Dialect::Sqlite3, TK3_SPACE, Token::SPACE, " "); - createTokenType(Dialect::Sqlite3, TK3_COMMENT, Token::COMMENT, "--"); - createTokenType(Dialect::Sqlite3, TK3_MINUS, Token::OPERATOR, "-"); - createTokenType(Dialect::Sqlite3, TK3_SPACE, Token::SPACE, " "); - createTokenType(Dialect::Sqlite3, TK3_LP, Token::PAR_LEFT, "("); - createTokenType(Dialect::Sqlite3, TK3_RP, Token::PAR_RIGHT, ")"); + createTokenType(TK3_SPACE, Token::SPACE, " "); + createTokenType(TK3_COMMENT, Token::COMMENT, "--"); + createTokenType(TK3_MINUS, Token::OPERATOR, "-"); + createTokenType(TK3_SPACE, Token::SPACE, " "); + createTokenType(TK3_LP, Token::PAR_LEFT, "("); + createTokenType(TK3_RP, Token::PAR_RIGHT, ")"); semicolonTokenSqlite3 = - createTokenType(Dialect::Sqlite3, TK3_SEMI, Token::OPERATOR, ";"); - createTokenType(Dialect::Sqlite3, TK3_PLUS, Token::OPERATOR, "+"); - createTokenType(Dialect::Sqlite3, TK3_STAR, Token::OPERATOR, "*"); - createTokenType(Dialect::Sqlite3, TK3_SLASH, Token::OPERATOR, "/"); - createTokenType(Dialect::Sqlite3, TK3_COMMENT, Token::COMMENT, "/* */"); - createTokenType(Dialect::Sqlite3, TK3_EQ, Token::OPERATOR, "="); - createTokenType(Dialect::Sqlite3, TK3_EQ, Token::OPERATOR, "=="); - createTokenType(Dialect::Sqlite3, TK3_LE, Token::OPERATOR, "<="); - createTokenType(Dialect::Sqlite3, TK3_NE, Token::OPERATOR, "<>"); - createTokenType(Dialect::Sqlite3, TK3_NE, Token::OPERATOR, "!="); - createTokenType(Dialect::Sqlite3, TK3_LSHIFT, Token::OPERATOR, "<<"); - createTokenType(Dialect::Sqlite3, TK3_LT, Token::OPERATOR, "<"); - createTokenType(Dialect::Sqlite3, TK3_GE, Token::OPERATOR, ">="); - createTokenType(Dialect::Sqlite3, TK3_RSHIFT, Token::OPERATOR, ">>"); - createTokenType(Dialect::Sqlite3, TK3_GT, Token::OPERATOR, ">"); - createTokenType(Dialect::Sqlite3, TK3_BITOR, Token::OPERATOR, "|"); - createTokenType(Dialect::Sqlite3, TK3_CONCAT, Token::OPERATOR, "||"); - createTokenType(Dialect::Sqlite3, TK3_COMMA, Token::OPERATOR, ","); - createTokenType(Dialect::Sqlite3, TK3_BITAND, Token::OPERATOR, "&"); - createTokenType(Dialect::Sqlite3, TK3_BITNOT, Token::OPERATOR, "~"); - createTokenType(Dialect::Sqlite3, TK3_STRING, Token::STRING, "' '"); - createTokenType(Dialect::Sqlite3, TK3_ID, Token::OTHER, "id"); - createTokenType(Dialect::Sqlite3, TK3_DOT, Token::OPERATOR, "."); - createTokenType(Dialect::Sqlite3, TK3_INTEGER, Token::INTEGER, "1"); - createTokenType(Dialect::Sqlite3, TK3_FLOAT, Token::FLOAT, "1.0"); - createTokenType(Dialect::Sqlite3, TK3_VARIABLE, Token::BIND_PARAM, "?"); - createTokenType(Dialect::Sqlite3, TK3_BLOB, Token::BLOB, "X'53'"); + createTokenType(TK3_SEMI, Token::OPERATOR, ";"); + createTokenType(TK3_PLUS, Token::OPERATOR, "+"); + createTokenType(TK3_STAR, Token::OPERATOR, "*"); + createTokenType(TK3_SLASH, Token::OPERATOR, "/"); + createTokenType(TK3_COMMENT, Token::COMMENT, "/* */"); + createTokenType(TK3_EQ, Token::OPERATOR, "="); + createTokenType(TK3_EQ, Token::OPERATOR, "=="); + createTokenType(TK3_LE, Token::OPERATOR, "<="); + createTokenType(TK3_NE, Token::OPERATOR, "<>"); + createTokenType(TK3_NE, Token::OPERATOR, "!="); + createTokenType(TK3_LSHIFT, Token::OPERATOR, "<<"); + createTokenType(TK3_LT, Token::OPERATOR, "<"); + createTokenType(TK3_GE, Token::OPERATOR, ">="); + createTokenType(TK3_RSHIFT, Token::OPERATOR, ">>"); + createTokenType(TK3_GT, Token::OPERATOR, ">"); + createTokenType(TK3_BITOR, Token::OPERATOR, "|"); + createTokenType(TK3_CONCAT, Token::OPERATOR, "||"); + createTokenType(TK3_COMMA, Token::OPERATOR, ","); + createTokenType(TK3_BITAND, Token::OPERATOR, "&"); + createTokenType(TK3_BITNOT, Token::OPERATOR, "~"); + createTokenType(TK3_STRING, Token::STRING, "' '"); + createTokenType(TK3_ID, Token::OTHER, "id"); + createTokenType(TK3_DOT, Token::OPERATOR, "."); + createTokenType(TK3_INTEGER, Token::INTEGER, "1"); + createTokenType(TK3_FLOAT, Token::FLOAT, "1.0"); + createTokenType(TK3_VARIABLE, Token::BIND_PARAM, "?"); + createTokenType(TK3_BLOB, Token::BLOB, "X'53'"); // Contextual ID tokens - createTokenType(Dialect::Sqlite3, TK3_ID_DB, Token::CTX_DATABASE, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_TAB, Token::CTX_TABLE, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_TAB_NEW, Token::CTX_TABLE_NEW, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_COL, Token::CTX_COLUMN, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_COL_NEW, Token::CTX_COLUMN_NEW, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_COL_TYPE, Token::CTX_COLUMN_TYPE, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_COLLATE, Token::CTX_COLLATION, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_FN, Token::CTX_FUNCTION, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_ERR_MSG, Token::CTX_ERROR_MESSAGE, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_IDX, Token::CTX_INDEX, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_IDX_NEW, Token::CTX_INDEX_NEW, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_VIEW, Token::CTX_VIEW, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_VIEW_NEW, Token::CTX_VIEW_NEW, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_JOIN_OPTS, Token::CTX_JOIN_OPTS, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_CONSTR, Token::CTX_CONSTRAINT, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_FK_MATCH, Token::CTX_FK_MATCH, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_TRANS, Token::CTX_TRANSACTION, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_ALIAS, Token::CTX_ALIAS, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_PRAGMA, Token::CTX_PRAGMA, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_TRIG, Token::CTX_TRIGGER, ""); - createTokenType(Dialect::Sqlite3, TK3_ID_TRIG_NEW, Token::CTX_TRIGGER_NEW, ""); - createTokenType(Dialect::Sqlite3, TK3_CTX_ROWID_KW, Token::CTX_ROWID_KW, "ROWID"); - createTokenType(Dialect::Sqlite3, TK3_ID, Token::CTX_OLD_KW, "OLD"); - createTokenType(Dialect::Sqlite3, TK3_ID, Token::CTX_NEW_KW, "NEW"); + createTokenType(TK3_ID_DB, Token::CTX_DATABASE, ""); + createTokenType(TK3_ID_TAB, Token::CTX_TABLE, ""); + createTokenType(TK3_ID_TAB_NEW, Token::CTX_TABLE_NEW, ""); + createTokenType(TK3_ID_COL, Token::CTX_COLUMN, ""); + createTokenType(TK3_ID_COL_NEW, Token::CTX_COLUMN_NEW, ""); + createTokenType(TK3_ID_COL_TYPE, Token::CTX_COLUMN_TYPE, ""); + createTokenType(TK3_ID_COLLATE, Token::CTX_COLLATION, ""); + createTokenType(TK3_ID_FN, Token::CTX_FUNCTION, ""); + createTokenType(TK3_ID_ERR_MSG, Token::CTX_ERROR_MESSAGE, ""); + createTokenType(TK3_ID_IDX, Token::CTX_INDEX, ""); + createTokenType(TK3_ID_IDX_NEW, Token::CTX_INDEX_NEW, ""); + createTokenType(TK3_ID_VIEW, Token::CTX_VIEW, ""); + createTokenType(TK3_ID_VIEW_NEW, Token::CTX_VIEW_NEW, ""); + createTokenType(TK3_ID_JOIN_OPTS, Token::CTX_JOIN_OPTS, ""); + createTokenType(TK3_ID_CONSTR, Token::CTX_CONSTRAINT, ""); + createTokenType(TK3_ID_FK_MATCH, Token::CTX_FK_MATCH, ""); + createTokenType(TK3_ID_TRANS, Token::CTX_TRANSACTION, ""); + createTokenType(TK3_ID_ALIAS, Token::CTX_ALIAS, ""); + createTokenType(TK3_ID_PRAGMA, Token::CTX_PRAGMA, ""); + createTokenType(TK3_ID_TRIG, Token::CTX_TRIGGER, ""); + createTokenType(TK3_ID_TRIG_NEW, Token::CTX_TRIGGER_NEW, ""); + createTokenType(TK3_CTX_ROWID_KW, Token::CTX_ROWID_KW, "ROWID"); + createTokenType(TK3_ID, Token::CTX_OLD_KW, "OLD"); + createTokenType(TK3_ID, Token::CTX_NEW_KW, "NEW"); QHashIterator<QString,int> i3(getKeywords3()); while (i3.hasNext()) { i3.next(); - createTokenType(Dialect::Sqlite3, i3.value(), Token::KEYWORD, i3.key()); - } - - // - // SQLite 2 - // - - createTokenType(Dialect::Sqlite2, TK2_SPACE, Token::SPACE, " "); - createTokenType(Dialect::Sqlite2, TK2_COMMENT, Token::COMMENT, "--"); - createTokenType(Dialect::Sqlite2, TK2_MINUS, Token::OPERATOR, "-"); - createTokenType(Dialect::Sqlite2, TK2_SPACE, Token::SPACE, " "); - createTokenType(Dialect::Sqlite2, TK2_LP, Token::PAR_LEFT, "("); - createTokenType(Dialect::Sqlite2, TK2_RP, Token::PAR_RIGHT, ")"); - semicolonTokenSqlite2 = - createTokenType(Dialect::Sqlite2, TK2_SEMI, Token::OPERATOR, ";"); - createTokenType(Dialect::Sqlite2, TK2_PLUS, Token::OPERATOR, "+"); - createTokenType(Dialect::Sqlite2, TK2_STAR, Token::OPERATOR, "*"); - createTokenType(Dialect::Sqlite2, TK2_SLASH, Token::OPERATOR, "/"); - createTokenType(Dialect::Sqlite2, TK2_COMMENT, Token::COMMENT, "/* */"); - createTokenType(Dialect::Sqlite2, TK2_EQ, Token::OPERATOR, "="); - createTokenType(Dialect::Sqlite2, TK2_EQ, Token::OPERATOR, "=="); - createTokenType(Dialect::Sqlite2, TK2_LE, Token::OPERATOR, "<="); - createTokenType(Dialect::Sqlite2, TK2_NE, Token::OPERATOR, "<>"); - createTokenType(Dialect::Sqlite2, TK2_NE, Token::OPERATOR, "!="); - createTokenType(Dialect::Sqlite2, TK2_LSHIFT, Token::OPERATOR, "<<"); - createTokenType(Dialect::Sqlite2, TK2_LT, Token::OPERATOR, "<"); - createTokenType(Dialect::Sqlite2, TK2_GE, Token::OPERATOR, ">="); - createTokenType(Dialect::Sqlite2, TK2_RSHIFT, Token::OPERATOR, ">>"); - createTokenType(Dialect::Sqlite2, TK2_GT, Token::OPERATOR, ">"); - createTokenType(Dialect::Sqlite2, TK2_BITOR, Token::OPERATOR, "|"); - createTokenType(Dialect::Sqlite2, TK2_CONCAT, Token::OPERATOR, "||"); - createTokenType(Dialect::Sqlite2, TK2_COMMA, Token::OPERATOR, ","); - createTokenType(Dialect::Sqlite2, TK2_BITAND, Token::OPERATOR, "&"); - createTokenType(Dialect::Sqlite2, TK2_BITNOT, Token::OPERATOR, "~"); - createTokenType(Dialect::Sqlite2, TK2_STRING, Token::STRING, "' '"); - createTokenType(Dialect::Sqlite2, TK2_ID, Token::OTHER, "id"); - createTokenType(Dialect::Sqlite2, TK2_DOT, Token::OPERATOR, "."); - createTokenType(Dialect::Sqlite2, TK2_INTEGER, Token::INTEGER, "1"); - createTokenType(Dialect::Sqlite2, TK2_FLOAT, Token::FLOAT, "1.0"); - createTokenType(Dialect::Sqlite2, TK2_VARIABLE, Token::BIND_PARAM, "?"); - - // Contextual ID tokens - createTokenType(Dialect::Sqlite2, TK2_ID_DB, Token::CTX_DATABASE, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_TAB, Token::CTX_TABLE, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_TAB_NEW, Token::CTX_TABLE_NEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_COL, Token::CTX_COLUMN, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_COL_NEW, Token::CTX_COLUMN_NEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_COL_TYPE, Token::CTX_COLUMN_TYPE, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_FN, Token::CTX_FUNCTION, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_ERR_MSG, Token::CTX_ERROR_MESSAGE, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_IDX, Token::CTX_INDEX, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_IDX_NEW, Token::CTX_INDEX_NEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_VIEW, Token::CTX_VIEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_VIEW_NEW, Token::CTX_VIEW_NEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_JOIN_OPTS, Token::CTX_JOIN_OPTS, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_CONSTR, Token::CTX_CONSTRAINT, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_FK_MATCH, Token::CTX_FK_MATCH, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_TRANS, Token::CTX_TRANSACTION, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_ALIAS, Token::CTX_ALIAS, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_PRAGMA, Token::CTX_PRAGMA, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_TRIG, Token::CTX_TRIGGER, ""); - createTokenType(Dialect::Sqlite2, TK2_ID_TRIG_NEW, Token::CTX_TRIGGER_NEW, ""); - createTokenType(Dialect::Sqlite2, TK2_ID, Token::CTX_ROWID_KW, "ROWID"); - createTokenType(Dialect::Sqlite2, TK2_ID, Token::CTX_OLD_KW, "OLD"); - createTokenType(Dialect::Sqlite2, TK2_ID, Token::CTX_NEW_KW, "NEW"); - - QHashIterator<QString,int> i2(getKeywords2()); - while (i2.hasNext()) - { - i2.next(); - createTokenType(Dialect::Sqlite2, i2.value(), Token::KEYWORD, i2.key()); + createTokenType(i3.value(), Token::KEYWORD, i3.key()); } } @@ -308,7 +233,7 @@ QString Lexer::detokenize(const TokenPtr& token) case Token::CTX_TRIGGER_NEW: case Token::CTX_VIEW: case Token::CTX_VIEW_NEW: - return token->value.isEmpty() ? wrapObjName(token->value, Dialect::Sqlite3, NameWrapper::DOUBLE_QUOTE) : token->value; + return token->value.isEmpty() ? wrapObjName(token->value, NameWrapper::DOUBLE_QUOTE) : token->value; case Token::CTX_ERROR_MESSAGE: case Token::STRING: return token->value.isEmpty() ? wrapString(token->value) : token->value; @@ -318,10 +243,10 @@ QString Lexer::detokenize(const TokenPtr& token) return token->value; } -TokenList Lexer::tokenize(const QString& sql, Dialect dialect) +TokenList Lexer::tokenize(const QString& sql) { - Lexer lexer(dialect); - return lexer.tokenize(sql); + Lexer lexer; + return lexer.process(sql); } TokenPtr Lexer::getEveryTokenTypePtr(Token *token) @@ -333,13 +258,10 @@ TokenPtr Lexer::getEveryTokenTypePtr(Token *token) return TokenPtr(); } -TokenPtr Lexer::createTokenType(Dialect dialect, int lemonType, Token::Type type, const QString &value) +TokenPtr Lexer::createTokenType(int lemonType, Token::Type type, const QString &value) { TokenPtr tokenPtr = TokenPtr::create(lemonType, type, value, -100, -100); - if (dialect == Dialect::Sqlite2) - everyTokenType2[type] << tokenPtr; - else - everyTokenType3[type] << tokenPtr; + everyTokenType3[type] << tokenPtr; everyTokenTypePtrMap[tokenPtr.data()] = tokenPtr; return tokenPtr; |