Merge pull request #258 from mhcomm/dvl/quentin/httprequest/recursive…

…_parsing

HTTPRequest Node: new url recursive parser

pypeman/contrib/http.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import ssl
+import re
 
 import aiohttp
 
@@ -12,6 +13,20 @@
 
 logger = logging.getLogger(__name__)
 
+# Regex to extract dynamic params from a string to permits complex search
+# in nested dicts:
+#
+# Example:
+# url = "toto/tutu/%(gigi.gogo)s/bla/%(rigo)r/toto"
+# Yields 2 results: "gigi.gogo" and "rigo"
+str_named_param_regex = re.compile(r"%\((?P<keyval>[^\)]*)\)[r|s|d|]")
+
+# Regex used to split a string by not escaped "."
+# example:
+# "titi.toto.tutu" yields 3 results: "titi", "toto", "tutu"
+# "titi.toto\.tutu" yields 2 results: "titi", "toto\.tutu" (the \ will be removed in code)
+not_escaped_dot_regex = re.compile(r"(?<!\\)\.")
+
 
 class HTTPEndpoint(endpoints.SocketEndpoint):
     """
@@ -155,7 +170,7 @@ class HttpRequest(nodes.BaseNode):
 
     def __init__(self, url, *args, method=None, headers=None, auth=None,
                  verify=True, params=None, client_cert=None, cookies=None,
-                 binary=False, json=False, send_as_json=False, **kwargs):
+                 binary=False, json=False, send_as_json=False, old_url_parsing=True, **kwargs):
         super().__init__(*args, **kwargs)
         self.url = url
         self.method = method
@@ -165,31 +180,58 @@ def __init__(self, url, *args, method=None, headers=None, auth=None,
         self.verify = verify
         self.params = params
         self.client_cert = client_cert
-        self.url = self.url.replace('%(meta.', '%(')
-        self.payload_in_url_dict = 'payload.' in self.url
+        self.url = self.url.replace('%(meta.', '%(')  # TODO: why ???
+        self.payload_in_url_dict = 'payload.' in self.url  # TODO: should I remove ?
+        self.params_in_url = str_named_param_regex.findall(self.url)
         self.binary = binary
         self.json = json
         self.send_as_json = send_as_json
+        self.old_url_parsing = old_url_parsing
         # TODO: create used payload keys for better perf of generate_request_url()
 
     def generate_request_url(self, msg):
-        url_dict = msg.meta
-        if self.payload_in_url_dict:
-            url_dict = dict(url_dict)
+        request_url = self.url
+        if self.old_url_parsing:
+            url_dict = msg.meta
+            if self.payload_in_url_dict:
+                url_dict = dict(url_dict)
+                try:
+                    for key, val in msg.payload.items():
+                        url_dict['payload.' + key] = val
+                except AttributeError:
+                    self.channel.logger.error(
+                        "Payload must be a python dict if used to generate url. "
+                        "This can be fixed using JsonToPython node before your "
+                        "RequestNode")
+                    raise
             try:
-                for key, val in msg.payload.items():
-                    url_dict['payload.' + key] = val
-            except AttributeError:
-                self.channel.logger.error(
-                    "Payload must be a python dict if used to generate url. "
-                    "This can be fixed using JsonToPython node before your "
-                    "RequestNode")
-                raise
-        try:
-            request_url = self.url % url_dict
-        except Exception as exc:
-            logger.error("cannot create url %r with args %r", self.url, repr(url_dict))
-            raise exc
+                request_url = self.url % url_dict
+            except Exception as exc:
+                logger.error("cannot create url %r with args %r", self.url, repr(url_dict))
+                raise exc
+        else:
+            # New recursive params parsing
+            if self.params_in_url:
+                params = {}
+                for param in self.params_in_url:
+                    subparams = not_escaped_dot_regex.split(param)
+                    if subparams[0] == "payload":
+                        data = msg.payload
+                        subparams.pop(0)
+                    elif subparams[0] == "meta":
+                        data = msg.meta
+                        subparams.pop(0)
+                    else:
+                        data = msg.meta
+                    for subparam in subparams:
+                        subparam = subparam.replace(r"\.", ".")
+                        data = data[subparam]
+                    params[param] = data
+                try:
+                    request_url = self.url % params
+                except Exception as exc:
+                    logger.error("cannot create url %r with args %r", self.url, repr(params))
+                    raise exc
         return request_url
 
     async def handle_request(self, msg):

pypeman/tests/test_nodes.py

@@ -407,7 +407,7 @@ def test_httprequest_node(self):
 
         auth = ("login", "mdp")
         url = 'http://url/%(meta.beta)s/%(payload.alpha)s'
-        b_auth = aiohttp.BasicAuth(auth[0], auth[1])
+        b_auth = aiohttp.BasicAuth(*auth)
         client_cert = ('/cert.key', '/cert.crt')
         http_node1 = nodes.HttpRequest(url=url, verify=False, auth=auth)
         http_node1.channel = channel
@@ -576,6 +576,101 @@ def test_httprequest_node(self):
 
             mock_session.reset_mock()
 
+    @unittest.skipIf((sys.version_info[:2] == (3, 7)),
+                     "difficulty to mock async with statement in py3.7")  # TODO: rm in py3.8+
+    def test_httprequest_node2_new_parsing(self):
+        """ Whether HttpRequest node recursive url parser is functional """
+
+        channel = FakeChannel(self.loop)
+
+        auth = ("login", "mdp")
+        url = 'http://url/%(meta.beta.beta2)s/%(payload.alpha.toto)s'
+        b_auth = aiohttp.BasicAuth(*auth)
+        client_cert = ('/cert.key', '/cert.crt')
+        http_node1 = nodes.HttpRequest(
+            url=url, verify=False, auth=auth,
+            old_url_parsing=False,)
+        http_node1.channel = channel
+
+        content1 = {"alpha": {"toto": "payload_url"}}
+        msg1 = generate_msg(message_content=content1)
+        meta_params = {'omega': 'meta_params'}
+        headers1 = {'test': 'test'}
+        msg1.meta = {
+            "beta": {"beta2": "meta_url"}, 'params': meta_params, 'headers': headers1}
+        req_url1 = 'http://url/meta_url/payload_url'
+        req_kwargs1 = {
+            'data': None,
+            'params': [('omega', 'meta_params')],
+            'url': req_url1,
+            'headers': headers1,
+            'method': 'get',
+            'auth': b_auth
+            }
+
+        msg2 = generate_msg(message_content=content1)
+        msg2.meta = dict(msg1.meta)
+        msg2.meta['method'] = 'post'
+        msg2.meta['params'] = {'zeta': ['un', 'deux', 'trois']}
+        req_kwargs2 = dict(req_kwargs1)
+        req_kwargs2['method'] = 'post'
+        req_kwargs2['params'] = [
+                ('zeta', 'un'),
+                ('zeta', 'deux'),
+                ('zeta', 'trois'),
+                ]
+        req_kwargs2['data'] = content1
+
+        args_headers = {'args_headers': 'args_headers'}
+        args_params = {'theta': ['uno', 'dos'], 'omega': tstfct2}
+        http_node2 = nodes.HttpRequest(
+            url=url,
+            method='post',
+            client_cert=client_cert,
+            auth=b_auth,
+            headers=args_headers,
+            params=args_params,
+            old_url_parsing=False,
+        )
+        http_node2.channel = channel
+
+        with mock.patch(
+            'pypeman.contrib.http.aiohttp.ClientSession',
+            autospec=True) as mock_client_session, mock.patch(
+                'ssl.SSLContext',
+                autospec=True) as mock_ssl_context:
+            mock_ctx_mgr = mock_client_session.return_value
+            mock_session = mock_ctx_mgr.__aenter__.return_value
+            mg = mock.MagicMock()
+            mg.text = get_mock_coro(mock.MagicMock())
+            mock_session.request = get_mock_coro(mg)
+            mock_load_cert_chain = mock_ssl_context.return_value.load_cert_chain
+
+            """
+                Test 1:
+                - default get,
+                - auth tuple in object BasicAuth,
+                - imbricated dict params from meta,
+                - headers from meta
+                - url construction
+            """
+            self.loop.run_until_complete(http_node1.handle(msg1))
+            mock_session.request.assert_called_once_with(**req_kwargs1)
+            mock_load_cert_chain.assert_not_called()
+
+            mock_session.reset_mock()
+
+            """
+                Test 2:
+                - post in meta with data from content,
+                - imbricated dict params from meta,
+            """
+            self.loop.run_until_complete(http_node1.handle(msg2))
+            mock_session.request.assert_called_once_with(**req_kwargs2)
+            mock_load_cert_chain.assert_not_called()
+
+            mock_session.reset_mock()
+
     def test_file_reader_node(self):
         """if FileReader are functionnal"""