Add utility to tokenize query

It's mostly useful for debugging changes in parser.

See #147.

Signed-off-by: Michal Čihař <michal@cihar.com>

Author: nijel

README.md

@@ -39,6 +39,12 @@ Command line utility to lint SQL query:
 ./vendor/bin/lint-query --query "SELECT 1"
 ```
 
+Command line utility to tokenize SQL query:
+
+```sh
+./vendor/bin/tokenize-query --query "SELECT 1"
+```
+
 ### Formatting SQL query
 
 ```php

bin/tokenize-query

@@ -0,0 +1,29 @@
+#!/usr/bin/env php
+<?php
+
+$files = array(
+    __DIR__ . "/../vendor/autoload.php",
+    __DIR__ . "/../../vendor/autoload.php",
+    __DIR__ . "/../../../autoload.php",
+    "vendor/autoload.php"
+);
+
+$found = false;
+foreach ($files as $file) {
+    if (file_exists($file)) {
+        require_once $file;
+        $found = true;
+        break;
+    }
+}
+
+if (!$found) {
+    die(
+        "You need to set up the project dependencies using the following commands:" . PHP_EOL .
+        "curl -sS https://getcomposer.org/installer | php" . PHP_EOL .
+        "php composer.phar install" . PHP_EOL
+    );
+}
+
+$cli = new PhpMyAdmin\SqlParser\Utils\CLI();
+exit($cli->runTokenize());

src/Utils/CLI.php

@@ -130,4 +130,54 @@ public function runLint()
 
         return 1;
     }
+
+    public function usageTokenize()
+    {
+        echo "Usage: tokenize-query --query SQL\n";
+    }
+
+    public function parseTokenize()
+    {
+        $longopts = array('help', 'query:');
+        $params = $this->getopt(
+            'hq:', $longopts
+        );
+        $this->mergeLongOpts($params, $longopts);
+
+        return $params;
+    }
+
+    public function runTokenize()
+    {
+        $params = $this->parseTokenize();
+        if ($params === false) {
+            return 1;
+        }
+        if (isset($params['h'])) {
+            $this->usageTokenize();
+
+            return 0;
+        }
+        if (isset($params['q'])) {
+            $lexer = new Lexer($params['q'], false);
+            foreach ($lexer->list->tokens as $idx => $token) {
+                echo '[TOKEN ', $idx, "]\n";
+                echo 'Type = ', $token->type, "\n";
+                echo 'Flags = ', $token->flags, "\n";
+                echo 'Value = ';
+                var_export($token->value);
+                echo "\n";
+                echo 'Token = ';
+                var_export($token->token);
+                echo "\n";
+                echo "\n";
+            }
+
+            return 0;
+        }
+        echo "ERROR: Missing parameters!\n";
+        $this->usageTokenize();
+
+        return 1;
+    }
 }

tests/Utils/CLITest.php

@@ -142,4 +142,57 @@ public function lintParams()
             ),
         );
     }
+
+    /**
+     * @dataProvider tokenizeParams
+     *
+     * @param mixed $getopt
+     * @param mixed $output
+     * @param mixed $result
+     */
+    public function testRunTokenize($getopt, $output, $result)
+    {
+        $cli = $this->getCLI($getopt);
+        $this->expectOutputString($output);
+        $this->assertEquals($result, $cli->runTokenize());
+    }
+
+    public function tokenizeParams()
+    {
+        $result = (
+            "[TOKEN 0]\nType = 1\nFlags = 3\nValue = 'SELECT'\nToken = 'SELECT'\n\n"
+            . "[TOKEN 1]\nType = 3\nFlags = 0\nValue = ' '\nToken = ' '\n\n"
+            . "[TOKEN 2]\nType = 6\nFlags = 0\nValue = 1\nToken = '1'\n\n"
+            . "[TOKEN 3]\nType = 9\nFlags = 0\nValue = NULL\nToken = NULL\n\n"
+        );
+
+        return array(
+            array(
+                array('q' => 'SELECT 1'),
+                $result,
+                0,
+            ),
+            array(
+                array('query' => 'SELECT 1'),
+                $result,
+                0,
+            ),
+            array(
+                array('h' => true),
+                'Usage: tokenize-query --query SQL' . "\n",
+                0,
+            ),
+            array(
+                array(),
+                'ERROR: Missing parameters!' . "\n" .
+                'Usage: tokenize-query --query SQL' . "\n",
+                1,
+            ),
+            array(
+                false,
+                '',
+                1,
+            ),
+        );
+    }
 }