1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
#ifndef LEXER_LOW_LEV_H
#define LEXER_LOW_LEV_H
#include "parser/token.h"
#include <QString>
#include <QTextStream>
/** @file */
/**
* @brief Low level tokenizer function used by the Lexer.
* @param z Query to tokenize.
* @param[out] token Token container to fill with values. Can be also a TolerantToken.
* @param prevToken Previous token returned from this function (if this is a subsequent call), or empty pointer othwewise. It's required for a contextual logic.
* @param sqliteVersion SQLite version, for which the tokenizer should work (currently only 3).
* Version affects the list of recognized keywords, a BLOB expression and an object name wrapper with the grave accent character (`).
* @param tolerant If true, then all multi-line and unfinished tokens (strings, comments)
* will be reported with invalid=true in TolerantToken, but the token itself will have type like it was finished.
* If this is true, then \p token must be of type TolerantToken, otherwise the the method will return 0 and log a critical error.
* @return Lemon token ID (see sqlite3_parse.h for possible token IDs).
*
* You shouldn't normally need to use this method. Instead of that, use Lexer class, as it provides higher level API.
*
* Most of the method code was taken from SQLite tokenizer code. It is modified to support both SQLite 3 gramma
* and other SQLiteStudio specific features.
*/
int lexerGetToken(const QString& z, TokenPtr& token, const TokenPtr& prevToken, int sqliteVersion, bool tolerant = false);
#endif // LEXER_LOW_LEV_H
|
