feat: can use escaped tokens inside quotes including quotes.

Change tokenizer to support:

  cmd with arg  "string with embedded \" double quote"

works for single quotes too. Mixed quotes can skip the \" or \'
escaping.

Also:

  quoted cmds args "can include \n newline, \t tab and \r return"

Added a doc example, also tests for new feature.

Author: rouilj

roundup/admin.py

@@ -297,6 +297,7 @@ def help_all(self):
            address="1 2 3"  (1 token: address=1 2 3)
            \\\\               (1 token: \\)
            \\n\\r\\t           (1 token: a newline, carriage-return and tab)
+           f "test\\"q"      (2 tokens: f test"q)
 
 When multiple nodes are specified to the roundup get or roundup set
 commands, the specified properties are retrieved or set on all the listed

roundup/token_r.py

@@ -96,6 +96,12 @@ def token_split(s, whitespace=' \r\n\t', quotes='\'"',
                 oldstate = state
                 state = ESCAPE
                 continue
+        elif state == QUOTE and c == '\\':
+            # in a quoted token and found an escape sequence
+            pos = pos + 1
+            oldstate = state
+            state = ESCAPE
+            continue
         elif state == QUOTE and c == quotechar:
             # in a quoted token and found a matching quote char
             pos = pos + 1

test/test_token.py

@@ -33,6 +33,28 @@ def testEmbedQuote(self):
         l = token_split('address="1 2 3"')
         self.assertEqual(l, ['address=1 2 3'])
 
+    def testEmbedEscapeQuote(self):
+        l = token_split(r'"Roch\'e Compaan"')
+        self.assertEqual(l, ["Roch'e Compaan"])
+
+        l = token_split(r'"Roch\"e Compaan"')
+        self.assertEqual(l, ['Roch"e Compaan'])
+
+        l = token_split(r'sql "COLLATE = \"utf8mb4_unicode_ci\";"')
+        self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])
+
+        l = token_split(r'''sql 'COLLATE = "utf8mb4_unicode_ci";' ''')
+        self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])
+
+        l = token_split(r'''sql 'COLLATE = \"utf8mb4_unicode_ci\";' ''')
+        self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])
+
+        l = token_split(r'''sql 'COLLATE = \'utf8mb4_unicode_ci\';' ''')
+        self.assertEqual(l, ["sql", "COLLATE = 'utf8mb4_unicode_ci';"])
+
+        l = token_split(r'''sql 'new\nline\rneed \ttab' ''')
+        self.assertEqual(l, ["sql", "new\nline\rneed \ttab"])
+
     def testEscaping(self):
         l = token_split('"Roch\'e" Compaan')
         self.assertEqual(l, ["Roch'e", "Compaan"])
@@ -42,6 +64,10 @@ def testEscaping(self):
         self.assertEqual(l, ['\\'])
         l = token_split(r'\n')
         self.assertEqual(l, ['\n'])
+        l = token_split(r'\r')
+        self.assertEqual(l, ['\r'])
+        l = token_split(r'\t')
+        self.assertEqual(l, ['\t'])
 
     def testBadQuote(self):
         self.assertRaises(ValueError, token_split, '"hello world')