tokenizer.php

<?php
namespace Bitrix\Perfmon\Sql;
 
class Token
{
    const T_WHITESPACE = 0;
    const T_STRING = 1;
    const T_CHAR = 2;
 
    const T_SINGLE_QUOTE = 3;
    const T_DOUBLE_QUOTE = 4;
    const T_BACK_QUOTE = 5;
    const T_SQUARE_QUOTE = 6;
 
    const T_COMMENT = 7;
 
    public $type;
    public $text;
    public $upper;
    public $line;
    public $level;
 
    public function __construct($type, $text)
    {
        $this->type = $type;
        $this->text = $text;
        $this->upper = mb_strtoupper($this->text);
    }
 
    public function setText($text)
    {
        $this->text = $text;
        $this->upper = mb_strtoupper($this->text);
    }
 
    public function appendText($text)
    {
        $this->text .= $text;
        $this->upper = mb_strtoupper($this->text);
    }
}
 
class Tokenizer
{
    protected $index = 0;
    protected $bookmark = 0;
    protected $tokens = [];
 
    public static function createFromString($sql)
    {
        $tokenizer = new self;
        $tokenizer->_tokenize($sql);
        $tokenizer->makeLines();
        $tokenizer->makeParenthesis();
        return $tokenizer;
    }
 
    public static function createFromTokens(array $tokens)
    {
        $tokenizer = new self;
        $tokenizer->tokens = $tokens;
        return $tokenizer;
    }
 
    public function getTokens()
    {
        return $this->tokens;
    }
 
    public function resetState()
    {
        $this->index = 0;
    }
 
    public function setBookmark()
    {
        $this->bookmark = $this->index;
    }
 
    public function restoreBookmark()
    {
        $this->index = $this->bookmark;
    }
 
    public function putBack()
    {
        $this->index--;
    }
 
    public function endOfInput()
    {
        return !isset($this->tokens[$this->index]);
    }
 
    public function getPrevToken()
    {
        $token = $this->tokens[$this->index - 1] ?? null;
        return $token;
    }
 
    public function getCurrentToken()
    {
        $token = $this->tokens[$this->index] ?? null;
        return $token;
    }
 
    public function nextToken()
    {
        $this->index++;
        $token = $this->tokens[$this->index] ?? null;
        return $token;
    }
 
    public function nextNotWhiteSpaceToken()
    {
        $i = $this->index + 1;
        while (isset($this->tokens[$i]))
        {
            $token = $this->tokens[$i];
            if ($token->type == Token::T_WHITESPACE || $token->type == Token::T_COMMENT)
            {
                $i++;
            }
            else
            {
                break;
            }
        }
        return $this->tokens[$i] ?? null;
    }
 
    public function skipWhiteSpace()
    {
        while (isset($this->tokens[$this->index]))
        {
            $token = $this->tokens[$this->index];
            if ($token->type == Token::T_WHITESPACE || $token->type == Token::T_COMMENT)
            {
                $this->index++;
            }
            else
            {
                break;
            }
        }
    }
 
    public function testUpperText($text)
    {
        if (isset($this->tokens[$this->index]))
        {
            $token = $this->tokens[$this->index];
            if ($token->upper === $text)
            {
                $this->index++;
                return true;
            }
        }
        return false;
    }
 
    public function testText($text)
    {
        if (isset($this->tokens[$this->index]))
        {
            $token = $this->tokens[$this->index];
            if ($token->upper === $text)
            {
                $this->index++;
                return true;
            }
        }
        return false;
    }
 
    private function _tokenize($sql)
    {
        $this->tokens = [];
        $tokenCount = 0;
        $chars = '(),.:=;/';
        $rawTokens = preg_split("/(
            [ \\t\\n\\r]+                   # WHITESPACE
            |\\\\+                          # BACKSLASHES
            |\"                             # DOUBLE QUOTE
            |'                              # SINGLE QUOTE
            |`[^`]+`                        # BACK QUOTE
            |\\[[^\\]]+\\]                  # SQUARE QUOTE
            |\\/\\*.*?\\*\\/                # COMMENTARY
            |--.*?\\n                       # COMMENTARY
            |\\#[^']*?\\n                   # COMMENTARY
            |[" . preg_quote($chars, '/') . ']  # CHARACTER
        )/xs', $sql, -1, PREG_SPLIT_DELIM_CAPTURE);
        $isInSingleQuote = false;
        $isInDoubleQuote = false;
        foreach ($rawTokens as $rawToken)
        {
            if ($rawToken === '')
            {
                continue;
            }
 
            $prevToken = $this->tokens[$tokenCount - 1] ?? null;
 
            if ($isInSingleQuote)
            {
                $prevToken->appendText($rawToken);
                if (
                    $rawToken === "'"
                    && preg_match("/(\\\\)*'\$/", $prevToken->text, $match)
                    && (
                        mb_strlen($match[0]) === 0
                        || (mb_strlen($match[0]) % 2) === 1
                    )
                )
                {
                    $isInSingleQuote = false;
                }
            }
            elseif ($isInDoubleQuote)
            {
                $prevToken->appendText($rawToken);
                if (
                    $rawToken === '"'
                    && preg_match('/(\\\\)*"$/', $prevToken->text, $match)
                    && (mb_strlen($match[0]) % 2) === 1
                )
                {
                    $isInDoubleQuote = false;
                }
            }
            elseif ($rawToken[0] === '`')
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_BACK_QUOTE, $rawToken);
            }
            elseif ($rawToken[0] === '[')
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_SQUARE_QUOTE, $rawToken);
            }
            elseif (
                ($rawToken[0] === '/' && $rawToken[1] === '*')
                || ($rawToken[0] === '-' && $rawToken[1] === '-')
                || ($rawToken[0] === '#')
            )
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_COMMENT, $rawToken);
            }
            elseif (mb_strlen($rawToken) == 1 && mb_strpos($chars, $rawToken) !== false)
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_CHAR, $rawToken);
            }
            elseif ($rawToken === '"')
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_DOUBLE_QUOTE, $rawToken);
                $isInDoubleQuote = true;
            }
            elseif ($rawToken === "'")
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_SINGLE_QUOTE, $rawToken);
                $isInSingleQuote = true;
            }
            elseif (preg_match("/^[ \\t\\n\\r]+\$/", $rawToken))
            {
                $this->tokens[$tokenCount++] = new Token(Token::T_WHITESPACE, $rawToken);
            }
            else
            {
                if ($tokenCount > 0 && $prevToken->type === Token::T_STRING)
                {
                    $prevToken->appendText($rawToken);
                }
                else
                {
                    $this->tokens[$tokenCount++] = new Token(Token::T_STRING, $rawToken);
                }
            }
        }
    }
 
    private function makeLines()
    {
        $line = 1;
        foreach ($this->tokens as $token)
        {
            $token->line = $line;
            if (preg_match_all("/\\n/", $token->text, $m))
            {
                $line += count($m[0]);
            }
        }
    }
 
    private function makeParenthesis()
    {
        $level = 0;
        foreach ($this->tokens as $token)
        {
            if ($token->text === ')')
            {
                $level--;
            }
            $token->level = $level;
            if ($token->text === '(')
            {
                $level++;
            }
        }
    }
}