Add CommentStrategy to optionally preserve leading comments

readme.md

@@ -60,6 +60,27 @@ foreach ($parser->parseFileStream($stream) as $query) {
 
 Available parsers: `MySqlMultiQueryParser`, `PostgreSqlMultiQueryParser`, `SqlServerMultiQueryParser`, `SqliteMultiQueryParser`.
 
+**Keep leading comments:**
+
+By default, comments are stripped and only query strings are yielded. To control what happens to
+comments, pass a `CommentStrategy` to the parser constructor. The bundled `PrependLeadingComments`
+strategy keeps the comments preceding a query as a prefix of that query -- useful when comments
+carry meaningful annotations, e.g. so they remain visible in observability tools:
+
+```php
+use Nextras\MultiQueryParser\Strategy\PrependLeadingComments;
+
+$parser = new MySqlMultiQueryParser(new PrependLeadingComments());
+
+$sql = "-- create the users table\nCREATE TABLE users (id INT);";
+
+foreach ($parser->parseString($sql) as $query) {
+    echo $query; // "-- create the users table\nCREATE TABLE users (id INT)"
+}
+```
+
+All comment styles supported by the given dialect (`--`, `/* */`, and `#` for MySQL) that directly precede a query are preserved with their original formatting; only pure leading whitespace is stripped. A comment that sits between two queries is treated as preceding the following one. Comments not followed by any query (e.g. a trailing comment at the end of input) are dropped.
+
 ### License
 
 MIT. See full [license](license.md).

src/BaseMultiQueryParser.php

@@ -5,13 +5,26 @@
 use ArrayIterator;
 use Iterator;
 use Nextras\MultiQueryParser\Exception\RuntimeException;
+use Nextras\MultiQueryParser\Fragment\Comment;
+use Nextras\MultiQueryParser\Fragment\Fragment;
+use Nextras\MultiQueryParser\Fragment\Query;
+use Nextras\MultiQueryParser\Strategy\DropComments;
 use function feof;
 use function fopen;
 use function fread;
 
 
 abstract class BaseMultiQueryParser implements IMultiQueryParser
 {
+	private CommentStrategy $commentStrategy;
+
+
+	public function __construct(?CommentStrategy $commentStrategy = null)
+	{
+		$this->commentStrategy = $commentStrategy ?? new DropComments();
+	}
+
+
 	/**
 	 * @param  positive-int $chunkSize
 	 * @return Iterator<string>
@@ -52,7 +65,32 @@ public function parseString(string $s): Iterator
 	 * @param  Iterator<string> $stream
 	 * @return Iterator<string>
 	 */
-	abstract public function parseStringStream(Iterator $stream): Iterator;
+	public function parseStringStream(Iterator $stream): Iterator
+	{
+		return $this->commentStrategy->apply($this->parseStringStreamToFragments($stream));
+	}
+
+
+	/**
+	 * @param  Iterator<string> $stream
+	 * @return Iterator<Fragment>
+	 */
+	abstract protected function parseStringStreamToFragments(Iterator $stream): Iterator;
+
+
+	/**
+	 * @return Iterator<Fragment>
+	 */
+	protected function buildFragments(?string $leadingComments, ?string $query): Iterator
+	{
+		if ($leadingComments !== null && $leadingComments !== '') {
+			yield new Comment($leadingComments);
+		}
+
+		if ($query !== null && $query !== '') {
+			yield new Query($query);
+		}
+	}
 
 
 	/**

src/CommentStrategy.php

@@ -0,0 +1,22 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser;
+
+use Iterator;
+use Nextras\MultiQueryParser\Fragment\Fragment;
+
+
+/**
+ * Decides what happens to the comments found in the parsed SQL stream.
+ *
+ * The parsers themselves only tokenize the input into a neutral {@see Fragment} stream of queries
+ * and comments; the strategy collapses that stream into the final stream of query strings.
+ */
+interface CommentStrategy
+{
+	/**
+	 * @param  Iterator<Fragment> $fragments
+	 * @return Iterator<string>
+	 */
+	public function apply(Iterator $fragments): Iterator;
+}

src/Fragment/Comment.php

@@ -0,0 +1,15 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser\Fragment;
+
+
+/**
+ * A run of one or more comments (and the whitespace interleaved with them).
+ */
+final class Comment implements Fragment
+{
+	public function __construct(
+		public string $text,
+	) {
+	}
+}

src/Fragment/Fragment.php

@@ -0,0 +1,11 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser\Fragment;
+
+
+/**
+ * @phpstan-sealed Query|Comment
+ */
+interface Fragment
+{
+}

src/Fragment/Query.php

@@ -0,0 +1,15 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser\Fragment;
+
+
+/**
+ * A single SQL query (without its terminating delimiter).
+ */
+final class Query implements Fragment
+{
+	public function __construct(
+		public string $sql,
+	) {
+	}
+}

src/MySqlMultiQueryParser.php

@@ -8,16 +8,15 @@
 
 class MySqlMultiQueryParser extends BaseMultiQueryParser
 {
-	public function parseStringStream(Iterator $stream): Iterator
+	protected function parseStringStreamToFragments(Iterator $stream): Iterator
 	{
 		$patternIterator = new PatternIterator($stream, $this->getQueryPattern(';'));
 
 		foreach ($patternIterator as $match) {
+			yield from $this->buildFragments($match['leadingComments'] ?? null, $match['query'] ?? null);
+
 			if (isset($match['delimiter']) && $match['delimiter'] !== '') {
 				$patternIterator->setPattern($this->getQueryPattern($match['delimiter']));
-
-			} elseif (isset($match['query']) && $match['query'] !== '') {
-				yield $match['query'];
 			}
 		}
 	}
@@ -30,12 +29,15 @@ private function getQueryPattern(string $delimiter): string
 
 		return /** @lang PhpRegExp */ "
 		~
-			(?:
-					\\s
-				|   /\\* (*PRUNE) (?: [^*]++ | \\*(?!/) )*+  \\*/
-				|   --[^\\n]*+(?:\\n|\\z)
-				|   \\#[^\\n]*+(?:\\n|\\z)
-			)*+
+			\\s*+
+			(?<leadingComments>
+				(?:
+						\\s
+					|   /\\* (*PRUNE) (?: [^*]++ | \\*(?!/) )*+  \\*/
+					|   --[^\\n]*+(?:\\n|\\z)
+					|   \\#[^\\n]*+(?:\\n|\\z)
+				)*+
+			)
 
 			(?:
 				(?i:

src/PatternIterator.php

@@ -30,7 +30,7 @@
  * the regex engine commits to the construct — if the closing delimiter is missing (because
  * it is in a later chunk), the overall match fails, causing the iterator to load more data.
  *
- * @implements IteratorAggregate<int, array<mixed>>
+ * @implements IteratorAggregate<int, array<array-key, string>>
  */
 class PatternIterator implements IteratorAggregate
 {

src/PostgreSqlMultiQueryParser.php

@@ -7,14 +7,12 @@
 
 class PostgreSqlMultiQueryParser extends BaseMultiQueryParser
 {
-	public function parseStringStream(Iterator $stream): Iterator
+	protected function parseStringStreamToFragments(Iterator $stream): Iterator
 	{
 		$patternIterator = new PatternIterator($stream, $this->getQueryPattern());
 
 		foreach ($patternIterator as $match) {
-			if (isset($match['query']) && $match['query'] !== '') {
-				yield $match['query'];
-			}
+			yield from $this->buildFragments($match['leadingComments'] ?? null, $match['query'] ?? null);
 		}
 	}
 
@@ -29,11 +27,14 @@ private function getQueryPattern(): string
 				(?<nestedBc> /\\* (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/ )
 			)
 
-			(?:
-					\\s
-				|   /\\* (*PRUNE) (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/
-				|   -- [^\\n]*+
-			)*+
+			\\s*+
+			(?<leadingComments>
+				(?:
+						\\s
+					|   /\\* (*PRUNE) (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/
+					|   -- [^\\n]*+
+				)*+
+			)
 
 			(?:
 				(?:

src/SqlServerMultiQueryParser.php

@@ -7,14 +7,12 @@
 
 class SqlServerMultiQueryParser extends BaseMultiQueryParser
 {
-	public function parseStringStream(Iterator $stream): Iterator
+	protected function parseStringStreamToFragments(Iterator $stream): Iterator
 	{
 		$patternIterator = new PatternIterator($stream, $this->getQueryPattern());
 
 		foreach ($patternIterator as $match) {
-			if (isset($match['query']) && $match['query'] !== '') {
-				yield $match['query'];
-			}
+			yield from $this->buildFragments($match['leadingComments'] ?? null, $match['query'] ?? null);
 		}
 	}
 
@@ -45,11 +43,14 @@ private function getQueryPattern(): string
 				(?<nestedBc> /\\* (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/ )
 			)
 
-			(?:
-					\\s
-				|   /\\* (*PRUNE) (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/
-				|   -- [^\\n]*+
-			)*+
+			\\s*+
+			(?<leadingComments>
+				(?:
+						\\s
+					|   /\\* (*PRUNE) (?: [^/*]++ | /(?!\\*) | \\*(?!/) | (?&nestedBc) )*+ \\*/
+					|   -- [^\\n]*+
+				)*+
+			)
 
 			(?:
 				(?:

src/SqliteMultiQueryParser.php

@@ -7,14 +7,12 @@
 
 class SqliteMultiQueryParser extends BaseMultiQueryParser
 {
-	public function parseStringStream(Iterator $stream): Iterator
+	protected function parseStringStreamToFragments(Iterator $stream): Iterator
 	{
 		$patternIterator = new PatternIterator($stream, $this->getQueryPattern());
 
 		foreach ($patternIterator as $match) {
-			if (isset($match['query']) && $match['query'] !== '') {
-				yield $match['query'];
-			}
+			yield from $this->buildFragments($match['leadingComments'] ?? null, $match['query'] ?? null);
 		}
 	}
 
@@ -55,7 +53,8 @@ private function getQueryPattern(): string
 				)
 			)
 
-			(?&skip)
+			\s*+
+			(?<leadingComments> (?&skip) )
 
 			(?:
 				(?:

src/Strategy/DropComments.php

@@ -0,0 +1,20 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser\Strategy;
+
+use Iterator;
+use Nextras\MultiQueryParser\CommentStrategy;
+use Nextras\MultiQueryParser\Fragment\Query;
+
+
+final class DropComments implements CommentStrategy
+{
+	public function apply(Iterator $fragments): Iterator
+	{
+		foreach ($fragments as $fragment) {
+			if ($fragment instanceof Query) {
+				yield $fragment->sql;
+			}
+		}
+	}
+}

src/Strategy/PrependLeadingComments.php

@@ -0,0 +1,33 @@
+<?php declare(strict_types = 1);
+
+namespace Nextras\MultiQueryParser\Strategy;
+
+use Iterator;
+use Nextras\MultiQueryParser\CommentStrategy;
+use Nextras\MultiQueryParser\Fragment\Comment;
+use Nextras\MultiQueryParser\Fragment\Query;
+
+
+/**
+ * Prepends the comments preceding a query to that query, keeping their original formatting.
+ *
+ * Comments not followed by any query (e.g. a trailing comment at the end of input) are dropped,
+ * since there is no query to attach them to.
+ */
+final class PrependLeadingComments implements CommentStrategy
+{
+	public function apply(Iterator $fragments): Iterator
+	{
+		$leadingComments = '';
+
+		foreach ($fragments as $fragment) {
+			if ($fragment instanceof Comment) {
+				$leadingComments .= $fragment->text;
+
+			} elseif ($fragment instanceof Query) {
+				yield $leadingComments . $fragment->sql;
+				$leadingComments = '';
+			}
+		}
+	}
+}