Fix using escape chars

The lexer did have too many different patterns which caused various issues in the interpretation of escape chars.
With this patch we are better compatible with sphinx. The results are validated with sphinx 7.

The lexer now has a separate BACKSLASH char, which allows parsers and rules to handle them as they need.
by skipping the `\` in most rules we do never have conflicts and rules are not interpreting the char after the `\`.

Author: jaapio

packages/guides-restructured-text/src/RestructuredText/Parser/InlineLexer.php

@@ -53,7 +53,7 @@ final class InlineLexer extends AbstractLexer
     public const STRONG_DELIMITER = 22;
     public const NBSP = 23;
     public const VARIABLE_DELIMITER = 24;
-    public const ESCAPED_SIGN = 25;
+    public const BACKSLASH = 25;
 
     /**
      * Map between string position and position in token list.
@@ -68,8 +68,7 @@ final class InlineLexer extends AbstractLexer
     protected function getCatchablePatterns(): array
     {
         return [
-            '\\\\``', // must be a separate case, as the next pattern would split in "\`" + "`", causing it to become a intepreted text
-            '\\\\[\s\S]', // Escaping hell... needs escaped slash in regex, but also in php.
+            '\\\\', // Escaping hell... needs escaped slash in regex, but also in php.
             '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}',
             '(?<=^|\s)[a-z0-9-]+_{2}', //Inline href.
             '(?<=^|\s)[a-z0-9-]+_{1}(?=[\s\.+]|$)', //Inline href.
@@ -133,7 +132,7 @@ protected function getType(string &$value)
             '[' => self::ANNOTATION_START,
             ']' => self::ANNOTATION_END,
             '~' => self::NBSP,
-            '\\``' => self::ESCAPED_SIGN,
+            '\\' => self::BACKSLASH,
             default => null,
         };
 
@@ -150,10 +149,6 @@ protected function getType(string &$value)
             return self::NAMED_REFERENCE;
         }
 
-        if (strlen($value) === 2 && $value[0] === '\\') {
-            return self::ESCAPED_SIGN;
-        }
-
         if (strlen($value) === 1 && ctype_space($value)) {
             return self::WHITESPACE;
         }

packages/guides-restructured-text/src/RestructuredText/Parser/Productions/InlineRules/DefaultTextRoleRule.php

@@ -49,6 +49,12 @@ public function apply(BlockContext $blockContext, InlineLexer $lexer): InlineNod
                         ->getDefaultTextRole()
                         ->processNode($blockContext->getDocumentParserContext(), '', $text, $text);
 
+                case InlineLexer::BACKSLASH:
+                    $lexer->moveNext();
+
+                    $text .= $lexer->token->value;
+
+                    break;
                 default:
                     $text .= $token->value;
             }

packages/guides-restructured-text/src/RestructuredText/Parser/Productions/InlineRules/EmphasisRule.php

@@ -48,6 +48,11 @@ public function apply(BlockContext $blockContext, InlineLexer $lexer): InlineNod
 
                     return new EmphasisInlineNode([new PlainTextInlineNode($text)]);
 
+                case InlineLexer::BACKSLASH:
+                    $lexer->moveNext();
+                    $text .= $lexer->token->value;
+
+                    break;
                 default:
                     $text .= $token->value;
             }

packages/guides-restructured-text/src/RestructuredText/Parser/Productions/InlineRules/EscapeRule.php

@@ -18,7 +18,6 @@
 use phpDocumentor\Guides\RestructuredText\Parser\InlineLexer;
 
 use function preg_match;
-use function substr;
 
 /**
  * Rule to escape characters with a backslash
@@ -27,13 +26,14 @@ final class EscapeRule extends ReferenceRule
 {
     public function applies(InlineLexer $lexer): bool
     {
-        return $lexer->token?->type === InlineLexer::ESCAPED_SIGN;
+        return $lexer->token?->type === InlineLexer::BACKSLASH;
     }
 
     public function apply(BlockContext $blockContext, InlineLexer $lexer): PlainTextInlineNode|null
     {
+        $lexer->moveNext();
+
         $char = $lexer->token?->value ?? '';
-        $char = substr($char, 1);
         $lexer->moveNext();
 
         if (preg_match('/^\s$/', $char)) {

packages/guides-restructured-text/src/RestructuredText/Parser/Productions/InlineRules/TextRoleRule.php

@@ -17,8 +17,6 @@
 use phpDocumentor\Guides\RestructuredText\Parser\BlockContext;
 use phpDocumentor\Guides\RestructuredText\Parser\InlineLexer;
 
-use function substr;
-
 /**
  * Rule to parse for text roles such as ``:ref:`something` `
  */
@@ -77,9 +75,11 @@ public function apply(BlockContext $blockContext, InlineLexer $lexer): InlineNod
                     $rawPart .= $token->value;
 
                     break;
-                case InlineLexer::ESCAPED_SIGN:
-                    $part .= substr($token->value, 1);
+                case InlineLexer::BACKSLASH:
                     $rawPart .= $token->value;
+                    $lexer->moveNext();
+                    $part .= $lexer->token->value;
+                    $rawPart .= $lexer->token->value;
 
                     break;
                 default:

packages/guides-restructured-text/src/RestructuredText/TextRoles/LiteralTextRole.php

@@ -43,6 +43,6 @@ public function processNode(
         string $content,
         string $rawContent,
     ): GenericTextRoleInlineNode {
-        return new GenericTextRoleInlineNode('literal', $rawContent, $this->getClass());
+        return new GenericTextRoleInlineNode('literal', $content, $this->getClass());
     }
 }

packages/guides-restructured-text/tests/unit/Parser/InlineLexerTest.php

@@ -80,7 +80,7 @@ public static function inlineLexerProvider(): array
             ],
             'Escaped double backtick' => [
                 '\\``[email protected]`',
-                [InlineLexer::ESCAPED_SIGN],
+                [InlineLexer::BACKSLASH],
             ],
         ];
     }

packages/guides-restructured-text/tests/unit/Parser/Productions/InlineRules/TextRoleRuleTest.php

@@ -51,6 +51,27 @@ public static function roleFormatProvider(): Generator
             'con`tent',
             'con\`tent',
         ];
+
+        yield 'role with escaped backslash' => [
+            ':role:`a\\\\b`',
+            'role',
+            'a\\b',
+            'a\\\\b',
+        ];
+
+        yield 'role with escaped backtick at end' => [
+            ':role:`text\``',
+            'role',
+            'text`',
+            'text\`',
+        ];
+
+        yield 'role with only escaped backtick' => [
+            ':role:`\``',
+            'role',
+            '`',
+            '\`',
+        ];
     }
 
     #[DataProvider('roleFormatProvider')]

tests/Functional/tests/code-textrole-no-escape/code-textrole-no-escape.html

@@ -1,3 +1,4 @@
 <p><code>Backslashes are not escaped \\ \` \&lt; in literals</code></p>
-<p><code>Backslashes are not escaped \\ \` \&lt; in default text roles</code></p>
-<p><code>Backslashes are not escaped \\ \` \&lt; in code</code></p>
+<p><code>Backslashes are not escaped \ ` &lt; in default text roles</code></p>
+<p><code>Backslashes are not escaped \ ` &lt; in code</code></p>
+<p><code>Backslashes are not escaped \ ` &lt; in code</code></p>

tests/Functional/tests/code-textrole-no-escape/code-textrole-no-escape.rst

@@ -3,3 +3,5 @@
 `Backslashes are not escaped \\ \` \< in default text roles`
 
 :code:`Backslashes are not escaped \\ \` \< in code`
+
+:literal:`Backslashes are not escaped \\ \` \< in code`