Skip to content

Commit

Permalink
feat: can use escaped tokens inside quotes including quotes.
Browse files Browse the repository at this point in the history
Change tokenizer to support:

  cmd with arg  "string with embedded \" double quote"

works for single quotes too. Mixed quotes can skip the \" or \'
escaping.

Also:

  quoted cmds args "can include \n newline, \t tab and \r return"

Added a doc example, also tests for new feature.
  • Loading branch information
rouilj committed Apr 7, 2024
1 parent aafa001 commit 128a14e
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 0 deletions.
1 change: 1 addition & 0 deletions roundup/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ def help_all(self):
address="1 2 3" (1 token: address=1 2 3)
\\\\ (1 token: \\)
\\n\\r\\t (1 token: a newline, carriage-return and tab)
f "test\\"q" (2 tokens: f test"q)
When multiple nodes are specified to the roundup get or roundup set
commands, the specified properties are retrieved or set on all the listed
Expand Down
6 changes: 6 additions & 0 deletions roundup/token_r.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,12 @@ def token_split(s, whitespace=' \r\n\t', quotes='\'"',
oldstate = state
state = ESCAPE
continue
elif state == QUOTE and c == '\\':
# in a quoted token and found an escape sequence
pos = pos + 1
oldstate = state
state = ESCAPE
continue
elif state == QUOTE and c == quotechar:
# in a quoted token and found a matching quote char
pos = pos + 1
Expand Down
26 changes: 26 additions & 0 deletions test/test_token.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,28 @@ def testEmbedQuote(self):
l = token_split('address="1 2 3"')
self.assertEqual(l, ['address=1 2 3'])

def testEmbedEscapeQuote(self):
l = token_split(r'"Roch\'e Compaan"')
self.assertEqual(l, ["Roch'e Compaan"])

l = token_split(r'"Roch\"e Compaan"')
self.assertEqual(l, ['Roch"e Compaan'])

l = token_split(r'sql "COLLATE = \"utf8mb4_unicode_ci\";"')
self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])

l = token_split(r'''sql 'COLLATE = "utf8mb4_unicode_ci";' ''')
self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])

l = token_split(r'''sql 'COLLATE = \"utf8mb4_unicode_ci\";' ''')
self.assertEqual(l, ["sql", 'COLLATE = "utf8mb4_unicode_ci";'])

l = token_split(r'''sql 'COLLATE = \'utf8mb4_unicode_ci\';' ''')
self.assertEqual(l, ["sql", "COLLATE = 'utf8mb4_unicode_ci';"])

l = token_split(r'''sql 'new\nline\rneed \ttab' ''')
self.assertEqual(l, ["sql", "new\nline\rneed \ttab"])

def testEscaping(self):
l = token_split('"Roch\'e" Compaan')
self.assertEqual(l, ["Roch'e", "Compaan"])
Expand All @@ -42,6 +64,10 @@ def testEscaping(self):
self.assertEqual(l, ['\\'])
l = token_split(r'\n')
self.assertEqual(l, ['\n'])
l = token_split(r'\r')
self.assertEqual(l, ['\r'])
l = token_split(r'\t')
self.assertEqual(l, ['\t'])

def testBadQuote(self):
self.assertRaises(ValueError, token_split, '"hello world')
Expand Down

0 comments on commit 128a14e

Please sign in to comment.