diff --git a/roundup/admin.py b/roundup/admin.py index 8d9a823f..ef3192e0 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -297,6 +297,7 @@ def help_all(self): address="1 2 3" (1 token: address=1 2 3) \\\\ (1 token: \\) \\n\\r\\t (1 token: a newline, carriage-return and tab) + f "test\\"q" (2 tokens: f test"q) When multiple nodes are specified to the roundup get or roundup set commands, the specified properties are retrieved or set on all the listed diff --git a/roundup/token_r.py b/roundup/token_r.py index 391cf6e6..e3f0713b 100644 --- a/roundup/token_r.py +++ b/roundup/token_r.py @@ -96,6 +96,12 @@ def token_split(s, whitespace=' \r\n\t', quotes='\'"', oldstate = state state = ESCAPE continue + elif state == QUOTE and c == '\\': + # in a quoted token and found an escape sequence + pos = pos + 1 + oldstate = state + state = ESCAPE + continue elif state == QUOTE and c == quotechar: # in a quoted token and found a matching quote char pos = pos + 1 diff --git a/test/test_token.py b/test/test_token.py index a8cc90bd..e9f3174d 100644 --- a/test/test_token.py +++ b/test/test_token.py @@ -33,6 +33,28 @@ def testEmbedQuote(self): l = token_split('address="1 2 3"') self.assertEqual(l, ['address=1 2 3']) + def testEmbedEscapeQuote(self): + l = token_split(r'"Roch\'e Compaan"') + self.assertEqual(l, ["Roch'e Compaan"]) + + l = token_split(r'"Roch\"e Compaan"') + self.assertEqual(l, ['Roch"e Compaan']) + + l = token_split(r'sql "COLLATE = \"utf8mb4_unicode_ci\";"') + self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";']) + + l = token_split(r'''sql 'COLLATE = "utf8mb4_unicode_ci";' ''') + self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";']) + + l = token_split(r'''sql 'COLLATE = \"utf8mb4_unicode_ci\";' ''') + self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";']) + + l = token_split(r'''sql 'COLLATE = \'utf8mb4_unicode_ci\';' ''') + self.assertEqual(l, ["sql", "COLLATE = 'utf8mb4_unicode_ci';"]) + + l = token_split(r'''sql 'new\nline\rneed \ttab' ''') + self.assertEqual(l, ["sql", "new\nline\rneed \ttab"]) + def testEscaping(self): l = token_split('"Roch\'e" Compaan') self.assertEqual(l, ["Roch'e", "Compaan"]) @@ -42,6 +64,10 @@ def testEscaping(self): self.assertEqual(l, ['\\']) l = token_split(r'\n') self.assertEqual(l, ['\n']) + l = token_split(r'\r') + self.assertEqual(l, ['\r']) + l = token_split(r'\t') + self.assertEqual(l, ['\t']) def testBadQuote(self): self.assertRaises(ValueError, token_split, '"hello world')