Imported Upstream version 2.99.6upstream/2.99.6

1 files changed, 733 insertions, 0 deletions

diff --git a/SQLiteStudio3/coreSQLiteStudio/parser/token.h b/SQLiteStudio3/coreSQLiteStudio/parser/token.h
new file mode 100644
index 0000000..1090bd4
--- /dev/null
+++ b/SQLiteStudio3/coreSQLiteStudio/parser/token.h
@@ -0,0 +1,733 @@
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include "common/utils.h"
+#include <QString>
+#include <QList>
+#include <QSharedPointer>
+
+/** @file */
+
+/**
+ * @def TOKEN_TYPE_MASK_DB_OBJECT
+ *
+ * Bit mask used to test Token::Type for representing any database object name, in any form.
+ * It's used in Token::isDbObjectType().
+ */
+#define TOKEN_TYPE_MASK_DB_OBJECT 0x1000
+
+struct Token;
+
+/**
+ * @brief Shared pointer to the Token.
+ */
+typedef QSharedPointer<Token> TokenPtr;
+
+/**
+ * @brief SQL query entity representing isolated part of the query.
+ *
+ * Tokens are generated by Lexer. Each token represents isolated part of the query,
+ * like a name, a number, an operator, a string, a keyword, or a comment, etc.
+ *
+ * In other words, tokenizing SQL query is splitting it into logical parts.
+ *
+ * Each token has a type, a value and position indexes of where it starts and where it ends (in the query string).
+ *
+ * Tokens are used mainly by the Parser to perform syntax analysis, but they can be also very helpful
+ * in other areas. They provide easy way for safe manipulation on the query string, without worrying
+ * about counting open or close characters of the string, etc. If the string has a single-quote used twice inside,
+ * this is still a regular SQL string and in that case there will be only a single string token representing it.
+ *
+ * If you're constructing Token outside of the Lemon parser, you should be interested only in 4 constructors:
+ * Token(), Token(QString value), Token(Type type, QString value)
+ * and Token(Type type, QString value, qint64 start, qint64 end).
+ * Rest of the constructors are to be used from the Lemon parser, as they require Lemon token ID to be provided.
+ *
+ * You will usually have the most to do with tokens when dealing with SqliteStatement and its 2 members:
+ * SqliteStatement::tokens and SqliteStatement::tokenMap.
+ */
+struct API_EXPORT Token
+{
+    /**
+     * @brief Token type.
+     *
+     * There are 2 kind of types - regular and context-oriented.
+     *
+     * Regular types are those defined by the SQL grammar and they represent real tokens. A special token type
+     * from group of regular types is the Type::INVALID, which means, that the character(s) encountered
+     * by the Lexer are invalid in SQL syntax understanding, or when there was no more query characters to read
+     * (which in this case means that tokenizing ended before this token).
+     *
+     * The context-oriented types are special meta types used by the Parser to probe potential candidates
+     * for next valid token when Parser::getNextTokenCandidates() is called. They are then processed by
+     * CompletionHelper. You won't deal with this kind of token types on regular basis. Context-oriented
+     * types are those starting with <tt>CTX_</tt>.
+     */
+    enum Type
+    {
+        INVALID = 0x0001,               /**< Invalid token, or no more tokens available from Lexer. */
+        OTHER = 0x1002,                 /**< A name, a word. */
+        STRING = 0x0003,                /**< A string (value will be stripped of the surrounding quotes). */
+        COMMENT = 0x0004,               /**< A comment, including starting/ending markers. */
+        FLOAT = 0x0005,                 /**< A decimal number. */
+        INTEGER = 0x0006,               /**< An integer number. */
+        BIND_PARAM = 0x0007,            /**< A bind parameter (<tt>:param</tt>, <tt>\@param</tt>, or <tt>?</tt>). */
+        OPERATOR = 0x0008,              /**< An operator (like <tt>";"</tt>, <tt>"+"</tt>, <tt>","</tt>, etc). */
+        PAR_LEFT = 0x0009,              /**< A left parenthesis (<tt>"("</tt>). */
+        PAR_RIGHT = 0x0010,             /**< A right parenthesis (<tt>")"</tt>). */
+        SPACE = 0x0011,                 /**< White space(s), including new line characters and tabs. */
+        BLOB = 0x0012,                  /**< Literal BLOB value (<tt>X'...'</tt> or <tt>x'...'</tt>). */
+        KEYWORD = 0x0013,               /**< A keyword (see getKeywords3() and getKeywords2()). */
+        CTX_COLUMN = 0x1014,            /**< Existing column name is valid at this token position. */
+        CTX_TABLE = 0x1015,             /**< Existing table name is valid at this token potision. */
+        CTX_DATABASE = 0x1016,          /**< Database name is valid at this token position. */
+        CTX_FUNCTION = 0x0017,          /**< Function name is valid at this token position. */
+        CTX_COLLATION = 0x0018,         /**< Collation name is valid at this token position. */
+        CTX_INDEX = 0x1019,             /**< Existing index name is valid at this token position. */
+        CTX_TRIGGER = 0x1020,           /**< Existing trigger name is valid at this token position. */
+        CTX_VIEW = 0x1021,              /**< View name is valid at this token position. */
+        CTX_JOIN_OPTS = 0x0022,         /**< JOIN keywords are valid at this token position (see getJoinKeywords()). */
+        CTX_TABLE_NEW = 0x0023,         /**< Name for new table is valid at this token position. */
+        CTX_INDEX_NEW = 0x0024,         /**< Name for new index is valid at this token position. */
+        CTX_VIEW_NEW = 0x0025,          /**< Name for new view is valid at this token position. */
+        CTX_TRIGGER_NEW = 0x0026,       /**< Name for new trigger is valid at this token position. */
+        CTX_ALIAS = 0x0027,             /**< Alias name is valid at this token position. */
+        CTX_TRANSACTION = 0x0028,       /**< Transaction name is valid at this token position. */
+        CTX_COLUMN_NEW = 0x0029,        /**< Name for the new column is valid at this token position. */
+        CTX_COLUMN_TYPE = 0x0030,       /**< Data type for the new column is valid at this token position. */
+        CTX_CONSTRAINT = 0x0031,        /**< Name for the new constraint is valid at this token position. */
+        CTX_FK_MATCH = 0x0032,          /**< MATCH keywords are valid at this token position (see getFkMatchKeywords()). */
+        CTX_PRAGMA = 0x0033,            /**< Pragma name is valid at this token position. */
+        CTX_ROWID_KW = 0x0034,          /**< ROWID keywords is valid at this token position (see isRowIdKeyword()). */
+        CTX_NEW_KW = 0x0035,            /**< The <tt>NEW</tt> keyword is valid at this token position. */
+        CTX_OLD_KW = 0x0036,            /**< The <tt>OLD</tt> keyword is valid at this token position. */
+        CTX_ERROR_MESSAGE = 0x0037      /**< Error message string is valid at this token position. */
+    };
+
+    /**
+     * @brief Creates empty token with type Type::INVALID.
+     *
+     * Lemon token ID is set to 0 and start/end positions are set to -1.
+     */
+    Token();
+
+    /**
+     * @brief Creates fully defined token.
+     * @param lemonType Lemon token ID to use (see sqlite2_parser.h and sqlite3_parser.h).
+     * @param type Token type.
+     * @param value Value of the token.
+     * @param start Start position of the token (index of the first character in the query).
+     * @param end End position of the token (index of last character in the query).
+     *
+     * This constructor is intended to be used from Lemon parser only. For other use cases
+     * use constructors without the \p lemonType parameter, unless you need it and you know what you're doing.
+     */
+    Token(int lemonType, Type type, QString value, qint64 start, qint64 end);
+
+    /**
+     * @overload
+     */
+    Token(int lemonType, Type type, QChar value, qint64 start, qint64 end);
+
+    /**
+     * @overload
+     */
+    Token(int lemonType, Type type, QString value);
+
+    /**
+     * @brief Creates token with type Type::INVALID and given value.
+     * @param value Value for the token.
+     *
+     * Start/end positions are set to -1.
+     */
+    explicit Token(QString value);
+
+    /**
+     * @brief Creates token of given type and with given value.
+     * @param type Type for the token.
+     * @param value Value for the token.
+     *
+     * Start/end positions are set to -1.
+     */
+    Token(Type type, QString value);
+
+    /**
+     * @brief Creates fully defined token.
+     * @param type Type of the token.
+     * @param value Value for the token.
+     * @param start Start position of the token (index of the first character in the query).
+     * @param end End position of the token (index of last character in the query).
+     */
+    Token(Type type, QString value, qint64 start, qint64 end);
+
+    /**
+     * @brief Destructor declared as virtual. Does nothing in this implementation.
+     */
+    virtual ~Token();
+
+    /**
+     * @brief Serializes token to human readable form.
+     * @return Token values in format: <tt>{type value start end}</tt>
+     */
+    QString toString();
+
+    /**
+     * @brief Converts given token type into its string representation.
+     * @param type Type to convert.
+     * @return Type as a string (same as textual representation of the enum in the code).
+     */
+    static const QString typeToString(Type type);
+
+    /**
+     * @brief Provides range of the token in the query.
+     * @return Start and end character index in relation to the query it comes from.
+     */
+    Range getRange();
+
+    /**
+     * @brief Tests whether this token represents any kind of whitespace.
+     * @return true if it's a whitespace, or false otherwise.
+     *
+     * Note, that from SQL perspective also comments are whitespaces.
+     */
+    bool isWhitespace() const;
+
+    /**
+     * @brief Tests whether this token represents separating value (like an operator, or parenthesis) in SQL understanding.
+     * @return true if it's a separating token, or false otherwise.
+     */
+    bool isSeparating() const;
+
+    /**
+     * @brief Tests whether this token is representing any kind of database object name.
+     * @return true if the token is the name an object, or false otherwise.
+     *
+     * From regular token types only the Type::OTHER represents.
+     * For context-oriented types there are several types representing object name.
+     * Use this method to find out which is and which is not.
+     *
+     * You won't need to use this method in most cases. It's useful only to CompletionHelper
+     * for now.
+     */
+    bool isDbObjectType() const;
+
+    /**
+     * @brief Converts token's type into a string representation.
+     * @return Token type as a string.
+     */
+    QString typeString() const;
+
+    /**
+     * @brief Compares other token to this token.
+     * @param other Other token to compare.
+     * @return 1 if tokens are equal, 0 if they're different.
+     *
+     * Tokens are equal then 4 members are equal: type, value, start and end.
+     * The lemonType member is ignored by this operator.
+     */
+    int operator==(const Token& other);
+
+    /**
+     * @brief Compares other token to this token and tells which one is greater.
+     * @param other Other token to compare.
+     * @return true if the other token is greater than this one, or false if it's smaller or equal.
+     *
+     * This operator compares only 2 members: the start and the end indexes. This operator
+     * is used to sort tokens by the character position they occurred at.
+     *
+     * The start value has higher precedence than the end value, but if start values are equal,
+     * then the end value is conclusive.
+     */
+    bool operator<(const Token& other) const;
+
+    /**
+     * @brief Lemon token ID. Used by the Parser class only.
+     */
+    int lemonType;
+
+    /**
+     * @brief Token type, describing general class of the token.
+     */
+    Type type;
+
+    /**
+     * @brief Literal value of the token, captured directly from the query.
+     */
+    QString value = QString::null;
+
+    /**
+     * @brief Start position (first character index) of the token in the query.
+     */
+    qint64 start;
+
+    /**
+     * @brief End position (last character index) of the token in the query.
+     */
+    qint64 end;
+};
+
+/**
+ * @brief qHash implementation for TokenPtr, so it can be used as a key in QHash and QSet.
+ * @param token Token that the hash code is calculated for.
+ * @return Unique integer value for the token.
+ */
+uint qHash(const TokenPtr& token);
+
+struct TolerantToken;
+/**
+ * @brief Shared pointer to TolerantToken.
+ */
+typedef QSharedPointer<TolerantToken> TolerantTokenPtr;
+
+/**
+ * @brief Variation of token that has additional "invalid" flag.
+ *
+ * TolerantToken is used by Lexer to tolerate unfinished comments, like when you start the
+ * comment at the end of the query, but you never close the comment. This is tolerable case,
+ * while not entire correct by the syntax.
+ *
+ * In such cases the syntax highlighter must be aware of the token being invalid, so the proper
+ * state is marked for the paragraph.
+ */
+struct TolerantToken : public Token
+{
+    /**
+     * @brief Invalid state flag for the token.
+     */
+    bool invalid = false;
+};
+
+/**
+ * @brief Ordered list of tokens.
+ *
+ * This is pretty much a QList of Token pointers, but it also provides some
+ * utility methods regarding this collection, which is very common in SQLiteStudio.
+ */
+class API_EXPORT TokenList : public QList<TokenPtr>
+{
+    public:
+        /**
+         * @brief Creates empty list.
+         */
+        TokenList();
+
+        /**
+         * @brief Creates list filled with the same entries as the other list.
+         * @param other List to copy pointers from.
+         */
+        TokenList(const QList<TokenPtr>& other);
+
+        /**
+         * @brief Serializes contents of the list into readable form.
+         * @return Contents in format: <tt>{type1 value1 start1 end1} {type2 value2 start2 end2} ...</tt>.
+         *
+         * Tokens are serialized with Token::toString(), then all serialized values are joined with single whitespace
+         * into the QString.
+         */
+        QString toString() const;
+
+        /**
+         * @brief Serializes tokens from the list into strings.
+         * @return List of tokens serialized into strings.
+         *
+         * Tokens are serialized with Token::toString().
+         */
+        QStringList toStringList() const;
+
+        /**
+         * @brief Provides index of first occurrence of the token in the list.
+         * @param token Token to look for.
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int indexOf(TokenPtr token) const;
+
+        /**
+         * @brief Provides index of first occurrence of the token with given type.
+         * @param type Toke type to look for.
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int indexOf(Token::Type type) const;
+
+        /**
+         * @brief Provides index of first occurrence of the token with given type and value.
+         * @param type Token type to look for.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int indexOf(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Provides index of first occurrence of the token with given value.
+         * @param value Value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int indexOf(const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Provides index of last occurrence of the token in the list.
+         * @param token Token to look for.
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int lastIndexOf(TokenPtr token) const;
+
+        /**
+         * @brief Provides index of last occurrence of the token with given type.
+         * @param type Token type to look for.
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int lastIndexOf(Token::Type type) const;
+
+        /**
+         * @brief Provides index of last occurrence of the token with given type and value.
+         * @param type Token type to look for.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int lastIndexOf(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Provides index of last occurrence of the token with given value.
+         * @param value Value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Index of the token, or -1 if token was not found.
+         */
+        int lastIndexOf(const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Finds first token with given type in the list.
+         * @param type Type to look for.
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr find(Token::Type type) const;
+
+        /**
+         * @brief Finds first token with given type and value.
+         * @param type Type to look for.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr find(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Finds first token with given type and value.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr find(const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Finds last token with given type in the list.
+         * @param type Type to look for.
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr findLast(Token::Type type) const;
+
+        /**
+         * @brief Finds last token with given type and value.
+         * @param type Type to look for.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr findLast(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Finds last token with given value.
+         * @param value Token value to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr findLast(const QString& value, Qt::CaseSensitivity caseSensitivity = Qt::CaseSensitive) const;
+
+        /**
+         * @brief Finds token that according to its start and end values covers given character position.
+         * @param cursorPosition Position to get token for.
+         * @return Token found, or null pointer if it was not found.
+         */
+        TokenPtr atCursorPosition(quint64 cursorPosition) const;
+
+        /**
+         * @brief Inserts tokens at given position in this list.
+         * @param i Position to insert at.
+         * @param list List of tokens to insert.
+         */
+        void insert(int i, const TokenList& list);
+
+        /**
+         * @brief Inserts a token at given position in this list.
+         * @param i Position to insert at.
+         * @param token Token to insert.
+         */
+        void insert(int i, TokenPtr token);
+
+        /**
+         * @brief Puts all tokens from the other list to this list.
+         * @param other List to get tokens from.
+         * @return Reference to this list.
+         *
+         * It erases any previous entries in this list, just like you would normally expect for the <tt>=</tt> operator.
+         */
+        TokenList& operator=(const QList<TokenPtr>& other);
+
+        /**
+         * @brief Detokenizes all tokens from this list.
+         * @return String from all tokens.
+         *
+         * This is just a convenient method to call Lexer::detokenize() on this list.
+         */
+        QString detokenize() const;
+
+        /**
+         * @brief Replaces tokens on this list with other tokens.
+         * @param startIdx Position of the first token in this list to replace.
+         * @param length Number of tokens to replace.
+         * @param newTokens New tokens to put in place of removed ones.
+         */
+        void replace(int startIdx, int length, const TokenList& newTokens);
+
+        /**
+         * @brief Replaces tokens on this list with another token.
+         * @param startIdx Position of the first token in this list to replace.
+         * @param length Number of tokens to replace.
+         * @param newToken New token to put in place of removed ones.
+         */
+        void replace(int startIdx, int length, TokenPtr newToken);
+
+        /**
+         * @brief Replaces token with another token.
+         * @param startIdx Position of the token in this list to replace.
+         * @param newToken New token to put in place of removed one.
+         */
+        void replace(int startIdx, TokenPtr newToken);
+
+        /**
+         * @brief Replaces token on this list with other tokens.
+         * @param startIdx Position of the token in this list to replace.
+         * @param newTokens New tokens to put in place of removed ones.
+         */
+        void replace(int startIdx, const TokenList& newTokens);
+
+        /**
+         * @brief Replaces tokens on this list with other tokens.
+         * @param startToken First token to replace.
+         * @param endToken Last token to replace.
+         * @param newTokens New tokens to put in place of removed ones.
+         * @return Number of tokens replaced.
+         *
+         * If either \p startToken or \p endToken were not found on the list, this method does nothing
+         * and returns 0.
+         */
+        int replace(TokenPtr startToken, TokenPtr endToken, const TokenList& newTokens);
+
+        /**
+         * @brief Replaces tokens on this list with other tokens.
+         * @param startToken First token to replace.
+         * @param endToken Last token to replace.
+         * @param newToken New token to put in place of removed ones.
+         * @return Number of tokens replaced.
+         *
+         * If either \p startToken or \p endToken were not found on the list, this method does nothing
+         * and returns 0.
+         */
+        int replace(TokenPtr startToken, TokenPtr endToken, TokenPtr newToken);
+
+        /**
+         * @brief Replaces token on this list with another token.
+         * @param oldToken Token to replace.
+         * @param newToken Token to replace with.
+         * @return true if \p oldToken was found and replaced, or false otherwise.
+         */
+        bool replace(TokenPtr oldToken, TokenPtr newToken);
+
+        /**
+         * @brief Replaces token on this list with other tokens.
+         * @param oldToken Token to replace.
+         * @param newTokens Tokens to replace with.
+         * @return true if \p oldToken was found and replaced, or false otherwise.
+         */
+        bool replace(TokenPtr oldToken, const TokenList& newTokens);
+
+        /**
+         * @brief Removes tokens from the list.
+         * @param startToken First token to remove.
+         * @param endToken Last token to remove.
+         * @return true if \p startToken and \p endToken were found in the list and removed, or false otherwise.
+         *
+         * In case \p startToken is placed after \p endToken, this method does nothing and returns false.
+         */
+        bool remove(TokenPtr startToken, TokenPtr endToken);
+
+        /**
+         * @brief Removes first token of given type from the list.
+         * @param type Token type to remove.
+         * @return true if token was located and removed, or false otherwise.
+         */
+        bool remove(Token::Type type);
+
+        /**
+         * @brief Removes all white-space tokens from the beginning of the list.
+         *
+         * White-space tokens are tested with Token::isWhitespace().
+         */
+        void trimLeft();
+
+        /**
+         * @brief Removes all white-space tokens from the end of the list.
+         *
+         * White-space tokens are tested with Token::isWhitespace().
+         */
+        void trimRight();
+
+        /**
+         * @brief Removes all white-space tokens from both the beginning and the end of the list.
+         *
+         * White-space tokens are tested with Token::isWhitespace().
+         */
+        void trim();
+
+        /**
+         * @brief Removes all tokens that match given criteria from the beginning of the list.
+         * @param type Token type to remove.
+         * @param alsoTrim Token value to remove.
+         *
+         * This method is an extension to the regular trimLeft(). It removes white-space tokens,
+         * as well as tokens that are of given \p type and have given \p value (both conditions must be met).
+         */
+        void trimLeft(Token::Type type, const QString& alsoTrim);
+
+        /**
+         * @brief Removes all tokens that match given criteria from the end of the list.
+         * @param type Token type to remove.
+         * @param alsoTrim Token value to remove.
+         *
+         * This method is an extension to the regular trimRight(). It removes white-space tokens,
+         * as well as tokens that are of given \p type and have given \p value (both conditions must be met).
+         */
+        void trimRight(Token::Type type, const QString& alsoTrim);
+
+        /**
+         * @brief Removes all tokens that match given criteria from the beginning and the end of the list.
+         * @param type Token type to remove.
+         * @param alsoTrim Token value to remove.
+         *
+         * This method is an extension to the regular trim(). It removes white-space tokens,
+         * as well as tokens that are of given \p type and have given \p value (both conditions must be met).
+         */
+        void trim(Token::Type type, const QString& alsoTrim);
+
+        /**
+         * @brief Creates list of tokens from this list, letting through only tokens of given type.
+         * @param type Type of tokens to provide in the new list.
+         * @return List of tokens from this list matching given \p type.
+         */
+        TokenList filter(Token::Type type) const;
+
+        /**
+         * @brief Creates list of tokens from this list, letting through only tokens that are not a whitespace.
+         * @return List of tokens from this list that are not a whitespace.
+         *
+         * The condition to test if tokens is a whitespace is a call to Token::isWhitespace().
+         */
+        TokenList filterWhiteSpaces() const;
+
+        /**
+         * @brief Returns sub-list of tokens from this list.
+         * @param pos Position to start sublist from.
+         * @param length Number of tokens to get from this list. If -1 (default), then all from the \p pos to the end.
+         * @return Sub-list of tokens from this list.
+         */
+        TokenList mid(int pos, int length = -1) const;
+
+    private:
+        /**
+         * @brief Finds first occurrence of token with given type.
+         * @param type Type of token to look for.
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findFirst() and indexOf() methods, as they share common logic.
+         */
+        TokenPtr findFirst(Token::Type type, int* idx) const;
+
+        /**
+         * @brief Finds first occurrence of token with given type and value.
+         * @param type Type of token to look for.
+         * @param value Value of the token to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findFirst() and indexOf() methods, as they share common logic.
+         */
+        TokenPtr findFirst(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const;
+
+        /**
+         * @brief Finds first occurrence of token with given value.
+         * @param value Value of the token to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findFirst() and indexOf() methods, as they share common logic.
+         */
+        TokenPtr findFirst(const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const;
+
+        /**
+         * @brief Finds last occurrence of token with given type.
+         * @param type Type of token to look for.
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findLast() and lastIndexOf() methods, as they share common logic.
+         */
+        TokenPtr findLast(Token::Type type, int* idx) const;
+
+        /**
+         * @brief Finds last occurrence of token with given type and value.
+         * @param type Type of token to look for.
+         * @param value Value of the token to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findLast() and lastIndexOf() methods, as they share common logic.
+         */
+        TokenPtr findLast(Token::Type type, const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const;
+
+        /**
+         * @brief Finds last occurrence of token with given value.
+         * @param value Value of the token to look for.
+         * @param caseSensitivity Should value lookup be case sensitive?
+         * @param idx Pointer to integer variable to store position in.
+         * @return Token found, or null pointer if token was not found.
+         *
+         * If \p idx is not null, then the position of token found is stored in it. If token was not found,
+         * then -1 is stored in the \p idx.
+         *
+         * This method is used by the public findLast() and lastIndexOf() methods, as they share common logic.
+         */
+        TokenPtr findLast(const QString& value, Qt::CaseSensitivity caseSensitivity, int* idx) const;
+};
+
+
+#endif // TOKEN_H