From 509d26610a0de7130d63f1540e03924d5743a92f Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Mon, 26 Feb 2024 16:48:16 -0500 Subject: [PATCH 01/22] Upgraded python and dependencies --- .github/workflows/pythonpackage.yml | 2 +- .github/workflows/unittest.yml | 2 +- .scrutinizer.yml | 2 +- .travis.yml | 1 + requirements/dev.txt | 124 ++++++++++++---------------- requirements/run.in | 24 +++--- requirements/run.txt | 91 ++++++++++---------- setup.py | 1 + tox.ini | 2 +- 9 files changed, 115 insertions(+), 134 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1f1e78784..504664767 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: '3.9' + python-version: '3.11' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 3258d9126..29af7bec8 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.9] + python-version: [3.11] steps: - uses: actions/checkout@v1 diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 29c2ae0ca..b04108095 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -4,7 +4,7 @@ checks: duplicate_code: true build: environment: - python: 3.9.12 + python: 3.11.2 postgresql: false redis: false dependencies: diff --git a/.travis.yml b/.travis.yml index fe8752fd0..1c91ae797 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.7" - "3.8" - "3.9" + - "3.11" install: - pip install --upgrade pip - pip install -r requirements/dev.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index 0a4fbd166..de275b2c0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/dev.txt --resolver=backtracking requirements/dev.in requirements/run.txt +# pip-compile --output-file=requirements/dev.txt requirements/dev.in requirements/run.txt # -e . # via -r requirements/dev.in @@ -15,20 +15,20 @@ # kytos alabaster==0.7.12 # via sphinx -anyio==3.6.2 +anyio==4.3.0 # via # -r requirements/run.txt - # httpcore + # httpx # kytos # starlette # watchfiles -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/run.txt # kytos astroid==2.13.5 # via pylint -asttokens==2.0.8 +asttokens==2.4.1 # via # -r requirements/run.txt # kytos @@ -41,16 +41,11 @@ attrs==21.4.0 # pytest babel==2.9.0 # via sphinx -backcall==0.1.0 - # via - # -r requirements/run.txt - # ipython - # kytos black==23.3.0 # via kytos build==0.10.0 # via pip-tools -certifi==2021.10.8 +certifi==2024.2.2 # via # -r requirements/run.txt # elastic-apm @@ -60,7 +55,7 @@ certifi==2021.10.8 # requests charset-normalizer==2.0.10 # via requests -click==8.1.3 +click==8.1.7 # via # -r requirements/run.txt # black @@ -71,7 +66,7 @@ colorama==0.4.6 # via sphinx-autobuild coverage[toml]==7.2.2 # via pytest-cov -decorator==4.4.2 +decorator==5.1.1 # via # -r requirements/run.txt # ipython @@ -80,11 +75,12 @@ dill==0.3.4 # via pylint distlib==0.3.6 # via virtualenv -dnspython==2.2.1 +dnspython==2.6.1 # via # -r requirements/run.txt # email-validator # kytos + # pymongo docopt==0.6.2 # via yala docutils==0.19 @@ -93,7 +89,12 @@ docutils==0.19 # kytos # python-daemon # sphinx -elastic-apm==6.9.1 +ecs-logging==2.1.0 + # via + # -r requirements/run.txt + # elastic-apm + # kytos +elastic-apm==6.20.0 # via # -r requirements/run.txt # kytos @@ -101,9 +102,7 @@ email-validator==1.3.0 # via # -r requirements/run.txt # kytos -exceptiongroup==1.1.1 - # via pytest -executing==1.0.0 +executing==2.0.1 # via # -r requirements/run.txt # kytos @@ -118,22 +117,22 @@ h11==0.14.0 # httpcore # kytos # uvicorn -httpcore==0.16.3 +httpcore==1.0.4 # via # -r requirements/run.txt # httpx # kytos -httptools==0.5.0 +httptools==0.6.1 # via # -r requirements/run.txt # kytos # uvicorn -httpx==0.24.0 +httpx==0.27.0 # via # -r requirements/run.txt # kytos # starlette -idna==3.3 +idna==3.6 # via # -r requirements/run.txt # anyio @@ -143,11 +142,9 @@ idna==3.3 # requests imagesize==1.4.1 # via sphinx -importlib-metadata==4.12.0 - # via sphinx iniconfig==2.0.0 # via pytest -ipython==8.1.1 +ipython==8.22.1 # via # -r requirements/run.txt # kytos @@ -170,12 +167,12 @@ janus==1.0.0 # via # -r requirements/run.txt # kytos -jedi==0.16.0 +jedi==0.19.1 # via # -r requirements/run.txt # ipython # kytos -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/run.txt # kytos @@ -207,7 +204,7 @@ lockfile==0.12.2 # -r requirements/run.txt # kytos # python-daemon -markupsafe==2.1.1 +markupsafe==2.1.5 # via # -r requirements/run.txt # jinja2 @@ -253,7 +250,7 @@ parse==1.19.0 # -r requirements/run.txt # kytos # openapi-core -parso==0.6.2 +parso==0.8.3 # via # -r requirements/run.txt # jedi @@ -266,12 +263,7 @@ pathable==0.4.3 # openapi-core pathspec==0.11.1 # via black -pexpect==4.8.0 - # via - # -r requirements/run.txt - # ipython - # kytos -pickleshare==0.7.5 +pexpect==4.9.0 # via # -r requirements/run.txt # ipython @@ -287,12 +279,12 @@ pluggy==1.0.0 # via # pytest # tox -prompt-toolkit==3.0.5 +prompt-toolkit==3.0.43 # via # -r requirements/run.txt # ipython # kytos -ptyprocess==0.6.0 +ptyprocess==0.7.0 # via # -r requirements/run.txt # kytos @@ -312,13 +304,13 @@ pydantic==1.9.0 # via # -r requirements/run.txt # kytos -pygments==2.13.0 +pygments==2.17.2 # via # -r requirements/run.txt # ipython # kytos # sphinx -pyjwt==2.4.0 +pyjwt==2.8.0 # via # -r requirements/run.txt # kytos @@ -326,7 +318,7 @@ pylint==2.15.0 # via # kytos # yala -pymongo==4.1.0 +pymongo==4.6.2 # via # -r requirements/run.txt # kytos @@ -350,12 +342,12 @@ python-daemon==2.3.1 # via # -r requirements/run.txt # kytos -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via # -r requirements/run.txt # kytos # uvicorn -python-multipart==0.0.6 +python-multipart==0.0.9 # via # -r requirements/run.txt # kytos @@ -389,7 +381,6 @@ sniffio==1.3.0 # via # -r requirements/run.txt # anyio - # httpcore # httpx # kytos snowballstemmer==2.2.0 @@ -412,35 +403,26 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -stack-data==0.5.0 +stack-data==0.6.3 # via # -r requirements/run.txt # ipython # kytos -starlette[full]==0.26.0 +starlette[full]==0.37.1 # via # -r requirements/run.txt # kytos -tenacity==8.0.1 +tenacity==8.2.3 # via # -r requirements/run.txt # kytos -tomli==2.0.1 - # via - # black - # build - # coverage - # pylint - # pyproject-hooks - # pytest - # tox tomlkit==0.11.7 # via pylint tornado==6.2 # via livereload tox==3.28.0 # via kytos -traitlets==5.3.0 +traitlets==5.14.1 # via # -r requirements/run.txt # ipython @@ -449,26 +431,22 @@ traitlets==5.3.0 typing-extensions==4.5.0 # via # -r requirements/run.txt - # astroid - # black # janus # jsonschema-spec # kytos # openapi-core # pydantic - # pylint - # starlette -urllib3==1.26.7 +urllib3==1.26.18 # via # -r requirements/run.txt # elastic-apm # kytos # requests -uvicorn[standard]==0.21.1 +uvicorn[standard]==0.27.1 # via # -r requirements/run.txt # kytos -uvloop==0.17.0 +uvloop==0.19.0 # via # -r requirements/run.txt # kytos @@ -477,21 +455,21 @@ virtualenv==20.21.0 # via # kytos # tox -watchdog==2.1.9 +watchdog==4.0.0 # via # -r requirements/run.txt # kytos -watchfiles==0.19.0 +watchfiles==0.21.0 # via # -r requirements/run.txt # kytos # uvicorn -wcwidth==0.1.9 +wcwidth==0.2.13 # via # -r requirements/run.txt # kytos # prompt-toolkit -websockets==11.0 +websockets==12.0 # via # -r requirements/run.txt # kytos @@ -503,12 +481,14 @@ werkzeug==2.0.3 # openapi-core wheel==0.40.0 # via pip-tools -wrapt==1.15.0 - # via astroid +wrapt==1.14.1 + # via + # -r requirements/run.txt + # astroid + # elastic-apm + # kytos yala==3.2.0 # via kytos -zipp==3.8.1 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/run.in b/requirements/run.in index 6f912d329..322f6dc2b 100644 --- a/requirements/run.in +++ b/requirements/run.in @@ -1,21 +1,21 @@ # Since the install procedure install the packages below from pypi, we cannot # pin versions outside pypi, i.e. latest git master branch version. -ipython==8.1.1 +ipython==8.22.1 lockfile==0.12.2 python-openflow python-daemon==2.3.1 janus==1.0.0 -jinja2==3.1.2 -watchdog==2.1.9 -pyjwt==2.4.0 -pymongo==4.1.0 +jinja2==3.1.3 +watchdog==4.0.0 +pyjwt==2.8.0 +pymongo==4.6.2 pydantic==1.9.0 -dnspython==2.2.1 +dnspython==2.6.1 email-validator==1.3.0 -tenacity==8.0.1 -elastic-apm==6.9.1 +tenacity==8.2.3 +elastic-apm==6.20.0 openapi-core==0.16.6 -httpx==0.24.0 -starlette[full]==0.26.0 -uvicorn[standard]==0.21.1 -asgiref==3.6.0 +httpx==0.27.0 +starlette[full]==0.37.1 +uvicorn[standard]==0.27.1 +asgiref==3.7.2 diff --git a/requirements/run.txt b/requirements/run.txt index 643126a66..6840802b6 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -1,61 +1,62 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/run.txt --resolver=backtracking requirements/run.in +# pip-compile --output-file=requirements/run.txt requirements/run.in # -anyio==3.6.2 +anyio==4.3.0 # via - # httpcore + # httpx # starlette # watchfiles -asgiref==3.6.0 +asgiref==3.7.2 # via -r requirements/run.in -asttokens==2.0.8 +asttokens==2.4.1 # via stack-data attrs==21.4.0 # via jsonschema -backcall==0.1.0 - # via ipython -certifi==2021.10.8 +certifi==2024.2.2 # via # elastic-apm # httpcore # httpx -click==8.1.3 +click==8.1.7 # via uvicorn -decorator==4.4.2 +decorator==5.1.1 # via ipython -dnspython==2.2.1 +dnspython==2.6.1 # via # -r requirements/run.in # email-validator + # pymongo docutils==0.19 # via python-daemon -elastic-apm==6.9.1 +ecs-logging==2.1.0 + # via elastic-apm +elastic-apm==6.20.0 # via -r requirements/run.in email-validator==1.3.0 # via -r requirements/run.in -executing==1.0.0 +executing==2.0.1 # via stack-data h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.16.3 +httpcore==1.0.4 # via httpx -httptools==0.5.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.0 +httpx==0.27.0 # via # -r requirements/run.in # starlette -idna==3.3 +idna==3.6 # via # anyio # email-validator # httpx -ipython==8.1.1 +ipython==8.22.1 # via -r requirements/run.in isodate==0.6.1 # via openapi-core @@ -63,9 +64,9 @@ itsdangerous==2.1.2 # via starlette janus==1.0.0 # via -r requirements/run.in -jedi==0.16.0 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/run.in # starlette @@ -84,7 +85,7 @@ lockfile==0.12.2 # via # -r requirements/run.in # python-daemon -markupsafe==2.1.1 +markupsafe==2.1.5 # via jinja2 matplotlib-inline==0.1.6 # via ipython @@ -100,37 +101,35 @@ openapi-spec-validator==0.5.6 # via openapi-core parse==1.19.0 # via openapi-core -parso==0.6.2 +parso==0.8.3 # via jedi pathable==0.4.3 # via # jsonschema-spec # openapi-core -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -pickleshare==0.7.5 +prompt-toolkit==3.0.43 # via ipython -prompt-toolkit==3.0.5 - # via ipython -ptyprocess==0.6.0 +ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data pydantic==1.9.0 # via -r requirements/run.in -pygments==2.13.0 +pygments==2.17.2 # via ipython -pyjwt==2.4.0 +pyjwt==2.8.0 # via -r requirements/run.in -pymongo==4.1.0 +pymongo==4.6.2 # via -r requirements/run.in pyrsistent==0.18.0 # via jsonschema python-daemon==2.3.1 # via -r requirements/run.in -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn -python-multipart==0.0.6 +python-multipart==0.0.9 # via starlette python-openflow # via -r requirements/run.in @@ -149,15 +148,14 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx -stack-data==0.5.0 +stack-data==0.6.3 # via ipython -starlette[full]==0.26.0 +starlette[full]==0.37.1 # via -r requirements/run.in -tenacity==8.0.1 +tenacity==8.2.3 # via -r requirements/run.in -traitlets==5.3.0 +traitlets==5.14.1 # via # ipython # matplotlib-inline @@ -167,23 +165,24 @@ typing-extensions==4.5.0 # jsonschema-spec # openapi-core # pydantic - # starlette -urllib3==1.26.7 +urllib3==1.26.18 # via elastic-apm -uvicorn[standard]==0.21.1 +uvicorn[standard]==0.27.1 # via -r requirements/run.in -uvloop==0.17.0 +uvloop==0.19.0 # via uvicorn -watchdog==2.1.9 +watchdog==4.0.0 # via -r requirements/run.in -watchfiles==0.19.0 +watchfiles==0.21.0 # via uvicorn -wcwidth==0.1.9 +wcwidth==0.2.13 # via prompt-toolkit -websockets==11.0 +websockets==12.0 # via uvicorn werkzeug==2.0.3 # via openapi-core +wrapt==1.14.1 + # via elastic-apm # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index 7d50f0de8..e3c512bbd 100644 --- a/setup.py +++ b/setup.py @@ -251,6 +251,7 @@ def run(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: System :: Networking', 'Development Status :: 4 - Beta', 'Environment :: Console', diff --git a/tox.ini b/tox.ini index 7659908dd..1d415c068 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = coverage,lint [gh-actions] python = - 3.9: py39 + 3.11: py311 [testenv] whitelist_externals= From 7f2979547de4093794dc7da23b2044425978590b Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Mon, 26 Feb 2024 17:55:16 -0500 Subject: [PATCH 02/22] Unspecified python 3.11 version --- .scrutinizer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index b04108095..62c09a9a4 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -4,7 +4,7 @@ checks: duplicate_code: true build: environment: - python: 3.11.2 + python: 3.11 postgresql: false redis: false dependencies: From 450434d79f872492c60e70b683f5739387666f15 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Mon, 26 Feb 2024 19:45:32 -0500 Subject: [PATCH 03/22] Updated python version to 3.11.8 --- .scrutinizer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 62c09a9a4..f4ff0d0d0 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -4,7 +4,7 @@ checks: duplicate_code: true build: environment: - python: 3.11 + python: 3.11.8 postgresql: false redis: false dependencies: From 6d6dd4157c20a7a93fb0ee7fdfc6149fc5c37152 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Fri, 1 Mar 2024 02:34:54 -0500 Subject: [PATCH 04/22] Fixed deprecated socket usage --- kytos/core/atcp_server.py | 4 ++-- kytos/core/connection.py | 21 +++++++++++++++------ kytos/core/controller.py | 4 ++-- kytos/core/kytosd.py | 2 +- tests/unit/test_core/test_connection.py | 9 +++------ 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/kytos/core/atcp_server.py b/kytos/core/atcp_server.py index deb273950..9aea60373 100644 --- a/kytos/core/atcp_server.py +++ b/kytos/core/atcp_server.py @@ -116,7 +116,7 @@ def __init__(self): if not self.server: raise ValueError("server instance must be assigned before init") - def connection_made(self, transport): + def connection_made(self, transport: asyncio.Transport): """Handle new client connection, passing it to the controller. Build a new Kytos `Connection` and send a ``kytos/core.connection.new`` @@ -130,7 +130,7 @@ def connection_made(self, transport): LOG.info("New connection from %s:%s", addr, port) - self.connection = Connection(addr, port, socket) + self.connection = Connection(addr, port, socket, transport) # This allows someone to inherit from KytosServer and start a server # on another port to handle a different protocol. diff --git a/kytos/core/connection.py b/kytos/core/connection.py index a973d66b1..04a230d31 100644 --- a/kytos/core/connection.py +++ b/kytos/core/connection.py @@ -1,9 +1,9 @@ """Module with main classes related to Connections.""" import logging +from asyncio import Transport, trsock from enum import Enum from errno import EBADF, ENOTCONN from socket import SHUT_RDWR -from socket import error as SocketError __all__ = ('Connection', 'ConnectionProtocol', 'ConnectionState') @@ -33,19 +33,28 @@ def __init__(self, name=None, version=None, state=None): class Connection: """Connection class to abstract a network connections.""" - def __init__(self, address, port, socket, switch=None): + def __init__( + self, + address: str, + port: int, + socket: trsock.TransportSocket, + transport: Transport, + switch=None + ): """Assign parameters to instance variables. Args: address (|hw_address|): Source address. port (int): Port number. - socket (socket): socket. + socket (TransportSocket): socket. + transport (Transport): transport. switch (:class:`~.Switch`): switch with this connection. """ self.address = address self.port = port self.socket = socket self.switch = switch + self.transport = transport self.state = ConnectionState.NEW self.protocol = ConnectionProtocol() self.remaining_data = b'' @@ -90,8 +99,8 @@ def send(self, buffer): """ try: if self.is_alive(): - self.socket.sendall(buffer) - except (OSError, SocketError) as exception: + self.transport.write(buffer) + except OSError as exception: LOG.debug('Could not send packet. Exception: %s', exception) self.close() raise @@ -103,7 +112,7 @@ def close(self): try: self.socket.shutdown(SHUT_RDWR) - self.socket.close() + self.transport.close() self.socket = None LOG.debug('Connection Closed: %s', self.id) except OSError as exception: diff --git a/kytos/core/controller.py b/kytos/core/controller.py index fa4fb404c..983cf54b0 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -38,7 +38,7 @@ from kytos.core.auth import Auth from kytos.core.buffers import KytosBuffers from kytos.core.config import KytosConfig -from kytos.core.connection import ConnectionState +from kytos.core.connection import Connection, ConnectionState from kytos.core.db import db_conn_wait from kytos.core.dead_letter import DeadLetter from kytos.core.events import KytosEvent @@ -107,7 +107,7 @@ def __init__(self, options=None, loop: AbstractEventLoop = None): #: This dict stores all connections between the controller and the #: switches. The key for this dict is a tuple (ip, port). The content #: is a Connection - self.connections = {} + self.connections: dict[tuple, Connection] = {} #: dict: mapping of events and event listeners. #: #: The key of the dict is a KytosEvent (or a string that represent a diff --git a/kytos/core/kytosd.py b/kytos/core/kytosd.py index 72773c743..b40ba3616 100644 --- a/kytos/core/kytosd.py +++ b/kytos/core/kytosd.py @@ -77,7 +77,7 @@ def start_shell(controller=None): # Avoiding sqlite3.ProgrammingError when trying to save command history # on Kytos shutdown - cfg.HistoryAccessor.enabled = False + cfg.HistoryAccessor.enabled = True ipshell = InteractiveShellEmbed(config=cfg, banner1=banner1, diff --git a/tests/unit/test_core/test_connection.py b/tests/unit/test_core/test_connection.py index 01bc93fc6..aeaa94867 100644 --- a/tests/unit/test_core/test_connection.py +++ b/tests/unit/test_core/test_connection.py @@ -1,5 +1,4 @@ """Test kytos.core.connection module.""" -from socket import error as SocketError from unittest.mock import MagicMock import pytest @@ -51,17 +50,15 @@ def test_send(self): """Test send method.""" self.connection.send(b'data') - self.connection.socket.sendall.assert_called_with(b'data') + self.connection.transport.write.assert_called_with(b'data') def test_send_error(self): """Test send method to error case.""" - self.connection.socket.sendall.side_effect = SocketError + self.connection.transport.write.side_effect = OSError - with pytest.raises(SocketError): + with pytest.raises(OSError): self.connection.send(b'data') - assert self.connection.socket is None - def test_close(self): """Test close method.""" self.connection.close() From b9c1b24f95842a25c81d500901ce613634ef7496 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Sun, 3 Mar 2024 03:22:01 -0500 Subject: [PATCH 05/22] Updated openapi-core --- kytos/core/connection.py | 2 +- kytos/core/controller.py | 3 +- kytos/core/helpers.py | 41 ++++++++----------- kytos/core/rest_api.py | 58 -------------------------- requirements/dev.txt | 61 ++++++++++++++++++---------- requirements/run.in | 2 +- requirements/run.txt | 51 +++++++++++++++-------- tests/unit/test_core/test_helpers.py | 10 ++--- 8 files changed, 97 insertions(+), 131 deletions(-) diff --git a/kytos/core/connection.py b/kytos/core/connection.py index 04a230d31..6123b4d81 100644 --- a/kytos/core/connection.py +++ b/kytos/core/connection.py @@ -100,7 +100,7 @@ def send(self, buffer): try: if self.is_alive(): self.transport.write(buffer) - except OSError as exception: + except (OSError, TypeError) as exception: LOG.debug('Could not send packet. Exception: %s', exception) self.close() raise diff --git a/kytos/core/controller.py b/kytos/core/controller.py index 983cf54b0..733c24502 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -28,7 +28,6 @@ from importlib import reload as reload_module from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path -from socket import error as SocketError from pyof.foundation.exceptions import PackException @@ -626,7 +625,7 @@ async def msg_out_event_handler(self): message.header.xid, packet.hex()) self.notify_listeners(triggered_event) - except (OSError, SocketError): + except (OSError, TypeError): await self.publish_connection_error(triggered_event) self.log.info("connection closed. Cannot send message") except PackException as err: diff --git a/kytos/core/helpers.py b/kytos/core/helpers.py index 5a081c56f..ed8eab3df 100644 --- a/kytos/core/helpers.py +++ b/kytos/core/helpers.py @@ -9,17 +9,13 @@ from pathlib import Path from threading import Thread -from openapi_core.spec import Spec -from openapi_core.spec.shortcuts import create_spec -from openapi_core.validation.request import openapi_request_validator -from openapi_core.validation.request.datatypes import RequestValidationResult -from openapi_spec_validator import validate_spec -from openapi_spec_validator.readers import read_from_filename +from openapi_core import OpenAPI +from openapi_core.contrib.starlette import StarletteOpenAPIRequest +from openapi_core.unmarshalling.request.datatypes import RequestUnmarshalResult from kytos.core.apm import ElasticAPM from kytos.core.config import KytosConfig -from kytos.core.rest_api import (AStarletteOpenAPIRequest, HTTPException, - Request, StarletteOpenAPIRequest, +from kytos.core.rest_api import (HTTPException, Request, content_type_json_or_415, get_body) __all__ = ['listen_to', 'now', 'run_on_thread', 'get_time'] @@ -351,20 +347,12 @@ def get_time(data=None): return date.replace(tzinfo=timezone.utc) -def _read_from_filename(yml_file_path: Path) -> dict: - """Read from yml filename.""" - spec_dict, _ = read_from_filename(yml_file_path) - return spec_dict - - def load_spec(yml_file_path: Path): """Load and validate spec object given a yml file path.""" - spec_dict = _read_from_filename(yml_file_path) - validate_spec(spec_dict) - return create_spec(spec_dict) + return OpenAPI.from_file_path(yml_file_path) -def _request_validation_result_or_400(result: RequestValidationResult) -> None: +def _request_validation_result_or_400(result: RequestUnmarshalResult) -> None: """Request validation result or raise HTTP 400.""" if not result.errors: return @@ -372,7 +360,10 @@ def _request_validation_result_or_400(result: RequestValidationResult) -> None: "The request body contains invalid API data." ) errors = result.errors[0] - if hasattr(errors, "schema_errors"): + if not errors.__cause__: + error_response = str(errors) + elif hasattr(errors.__cause__, "schema_errors"): + errors = errors.__cause__ schema_errors = errors.schema_errors[0] error_log = { "error_message": schema_errors.message, @@ -388,12 +379,12 @@ def _request_validation_result_or_400(result: RequestValidationResult) -> None: f" {'/'.join(map(str,schema_errors.path))}." ) else: - error_response = str(errors) + error_response = str(errors.__cause__) raise HTTPException(400, detail=error_response) def validate_openapi_request( - spec: Spec, request: Request, loop: AbstractEventLoop + spec: OpenAPI, request: Request, loop: AbstractEventLoop ) -> bytes: """Validate a Request given an OpenAPI spec. @@ -405,13 +396,13 @@ def validate_openapi_request( if body: content_type_json_or_415(request) openapi_request = StarletteOpenAPIRequest(request, body) - result = openapi_request_validator.validate(spec, openapi_request) + result = spec.unmarshal_request(openapi_request) _request_validation_result_or_400(result) return body async def avalidate_openapi_request( - spec: Spec, + spec: OpenAPI, request: Request, ) -> bytes: """Async validate_openapi_request. @@ -429,8 +420,8 @@ async def avalidate_openapi_request( body = await request.body() if body: content_type_json_or_415(request) - openapi_request = AStarletteOpenAPIRequest(request, body) - result = openapi_request_validator.validate(spec, openapi_request) + openapi_request = StarletteOpenAPIRequest(request, body) + result = spec.unmarshal_request(openapi_request) _request_validation_result_or_400(result) return body diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index b638ccdcd..01a6a3fd0 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -6,17 +6,9 @@ from datetime import datetime from typing import Any, Optional -from openapi_core.contrib.starlette import \ - StarletteOpenAPIRequest as _StarletteOpenAPIRequest -from openapi_core.validation.request.datatypes import RequestParameters from starlette.exceptions import HTTPException from starlette.requests import Request from starlette.responses import JSONResponse as StarletteJSONResponse -from starlette.responses import Response - -Request = Request -Response = Response -HTTPException = HTTPException def _json_serializer(obj): @@ -99,53 +91,3 @@ def render(self, content) -> bytes: separators=(",", ":"), default=_json_serializer, ).encode("utf-8") - - -# pylint: disable=super-init-not-called -class AStarletteOpenAPIRequest(_StarletteOpenAPIRequest): - """Async StarletteOpenAPIRequest.""" - - def __init__(self, request: Request, body: bytes) -> None: - """Constructor of AsycnStarletteOpenAPIRequest. - - This constructor doesn't call super().__init__() to keep it async - """ - self.request = request - self.parameters = RequestParameters( - query=self.request.query_params, - header=self.request.headers, - cookie=self.request.cookies, - ) - self._body = body - - @property - def body(self) -> Optional[str]: - body = self._body - if body is None: - return None - return body.decode("utf-8") - - -# pylint: disable=super-init-not-called -class StarletteOpenAPIRequest(_StarletteOpenAPIRequest): - """Sync StarletteOpenAPIRequest.""" - - def __init__(self, request: Request, body: bytes) -> None: - """Constructor of AsycnStarletteOpenAPIRequest. - - This constructor doesn't call super().__init__() to keep it async - """ - self.request = request - self.parameters = RequestParameters( - query=self.request.query_params, - header=self.request.headers, - cookie=self.request.cookies, - ) - self._body = body - - @property - def body(self) -> Optional[str]: - body = self._body - if body is None: - return None - return body.decode("utf-8") diff --git a/requirements/dev.txt b/requirements/dev.txt index de275b2c0..2947745fa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -33,12 +33,13 @@ asttokens==2.4.1 # -r requirements/run.txt # kytos # stack-data -attrs==21.4.0 +attrs==23.2.0 # via # -r requirements/run.txt # jsonschema # kytos # pytest + # referencing babel==2.9.0 # via sphinx black==23.3.0 @@ -53,8 +54,11 @@ certifi==2024.2.2 # httpx # kytos # requests -charset-normalizer==2.0.10 - # via requests +charset-normalizer==3.3.2 + # via + # -r requirements/run.txt + # kytos + # requests click==8.1.7 # via # -r requirements/run.txt @@ -178,19 +182,25 @@ jinja2==3.1.3 # kytos # sphinx # starlette -jsonschema==4.17.3 +jsonschema==4.21.1 # via # -r requirements/run.txt - # jsonschema-spec # kytos + # openapi-core # openapi-schema-validator # openapi-spec-validator -jsonschema-spec==0.1.4 +jsonschema-path==0.3.2 # via # -r requirements/run.txt # kytos # openapi-core # openapi-spec-validator +jsonschema-specifications==2023.12.1 + # via + # -r requirements/run.txt + # jsonschema + # kytos + # openapi-schema-validator lazy-object-proxy==1.7.1 # via # -r requirements/run.txt @@ -223,17 +233,17 @@ more-itertools==8.12.0 # openapi-core mypy-extensions==1.0.0 # via black -openapi-core==0.16.6 +openapi-core==0.19.0 # via # -r requirements/run.txt # kytos -openapi-schema-validator==0.4.4 +openapi-schema-validator==0.6.2 # via # -r requirements/run.txt # kytos # openapi-core # openapi-spec-validator -openapi-spec-validator==0.5.6 +openapi-spec-validator==0.7.1 # via # -r requirements/run.txt # kytos @@ -258,9 +268,8 @@ parso==0.8.3 pathable==0.4.3 # via # -r requirements/run.txt - # jsonschema-spec + # jsonschema-path # kytos - # openapi-core pathspec==0.11.1 # via black pexpect==4.9.0 @@ -324,11 +333,6 @@ pymongo==4.6.2 # kytos pyproject-hooks==1.0.0 # via build -pyrsistent==0.18.0 - # via - # -r requirements/run.txt - # jsonschema - # kytos pytest==7.2.1 # via # kytos @@ -357,17 +361,34 @@ pytz==2021.3 pyyaml==6.0 # via # -r requirements/run.txt - # jsonschema-spec + # jsonschema-path # kytos # starlette # uvicorn -requests==2.27.0 - # via sphinx +referencing==0.31.1 + # via + # -r requirements/run.txt + # jsonschema + # jsonschema-path + # jsonschema-specifications + # kytos +requests==2.31.0 + # via + # -r requirements/run.txt + # jsonschema-path + # kytos + # sphinx rfc3339-validator==0.1.4 # via # -r requirements/run.txt # kytos # openapi-schema-validator +rpds-py==0.18.0 + # via + # -r requirements/run.txt + # jsonschema + # kytos + # referencing six==1.16.0 # via # -r requirements/run.txt @@ -432,9 +453,7 @@ typing-extensions==4.5.0 # via # -r requirements/run.txt # janus - # jsonschema-spec # kytos - # openapi-core # pydantic urllib3==1.26.18 # via diff --git a/requirements/run.in b/requirements/run.in index 322f6dc2b..f288fda62 100644 --- a/requirements/run.in +++ b/requirements/run.in @@ -14,7 +14,7 @@ dnspython==2.6.1 email-validator==1.3.0 tenacity==8.2.3 elastic-apm==6.20.0 -openapi-core==0.16.6 +openapi-core==0.19.0 httpx==0.27.0 starlette[full]==0.37.1 uvicorn[standard]==0.27.1 diff --git a/requirements/run.txt b/requirements/run.txt index 6840802b6..a7cf3d1fa 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -13,13 +13,18 @@ asgiref==3.7.2 # via -r requirements/run.in asttokens==2.4.1 # via stack-data -attrs==21.4.0 - # via jsonschema +attrs==23.2.0 + # via + # jsonschema + # referencing certifi==2024.2.2 # via # elastic-apm # httpcore # httpx + # requests +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via uvicorn decorator==5.1.1 @@ -56,6 +61,7 @@ idna==3.6 # anyio # email-validator # httpx + # requests ipython==8.22.1 # via -r requirements/run.in isodate==0.6.1 @@ -70,15 +76,19 @@ jinja2==3.1.3 # via # -r requirements/run.in # starlette -jsonschema==4.17.3 +jsonschema==4.21.1 # via - # jsonschema-spec + # openapi-core # openapi-schema-validator # openapi-spec-validator -jsonschema-spec==0.1.4 +jsonschema-path==0.3.2 # via # openapi-core # openapi-spec-validator +jsonschema-specifications==2023.12.1 + # via + # jsonschema + # openapi-schema-validator lazy-object-proxy==1.7.1 # via openapi-spec-validator lockfile==0.12.2 @@ -91,22 +101,20 @@ matplotlib-inline==0.1.6 # via ipython more-itertools==8.12.0 # via openapi-core -openapi-core==0.16.6 +openapi-core==0.19.0 # via -r requirements/run.in -openapi-schema-validator==0.4.4 +openapi-schema-validator==0.6.2 # via # openapi-core # openapi-spec-validator -openapi-spec-validator==0.5.6 +openapi-spec-validator==0.7.1 # via openapi-core parse==1.19.0 # via openapi-core parso==0.8.3 # via jedi pathable==0.4.3 - # via - # jsonschema-spec - # openapi-core + # via jsonschema-path pexpect==4.9.0 # via ipython prompt-toolkit==3.0.43 @@ -123,8 +131,6 @@ pyjwt==2.8.0 # via -r requirements/run.in pymongo==4.6.2 # via -r requirements/run.in -pyrsistent==0.18.0 - # via jsonschema python-daemon==2.3.1 # via -r requirements/run.in python-dotenv==1.0.1 @@ -135,11 +141,22 @@ python-openflow # via -r requirements/run.in pyyaml==6.0 # via - # jsonschema-spec + # jsonschema-path # starlette # uvicorn +referencing==0.31.1 + # via + # jsonschema + # jsonschema-path + # jsonschema-specifications +requests==2.31.0 + # via jsonschema-path rfc3339-validator==0.1.4 # via openapi-schema-validator +rpds-py==0.18.0 + # via + # jsonschema + # referencing six==1.16.0 # via # asttokens @@ -162,11 +179,11 @@ traitlets==5.14.1 typing-extensions==4.5.0 # via # janus - # jsonschema-spec - # openapi-core # pydantic urllib3==1.26.18 - # via elastic-apm + # via + # elastic-apm + # requests uvicorn[standard]==0.27.1 # via -r requirements/run.in uvloop==0.19.0 diff --git a/tests/unit/test_core/test_helpers.py b/tests/unit/test_core/test_helpers.py index 0f1920600..eb3031d38 100644 --- a/tests/unit/test_core/test_helpers.py +++ b/tests/unit/test_core/test_helpers.py @@ -1,7 +1,6 @@ """Test kytos.core.helpers module.""" from unittest.mock import MagicMock, patch -from kytos.core import helpers from kytos.core.helpers import (alisten_to, ds_executors, executors, get_thread_pool_max_workers, get_time, listen_to, load_spec, run_on_thread) @@ -24,12 +23,11 @@ async def on_some_event(self, event): assert result == "some_response" -def test_load_spec(monkeypatch, minimal_openapi_spec_dict) -> None: +@patch('kytos.core.helpers.OpenAPI') +def test_load_spec(mock_open) -> None: """Test load spec.""" - monkeypatch.setattr(helpers, "_read_from_filename", - lambda x: minimal_openapi_spec_dict) - spec = load_spec("mocked_path") - assert spec.accessor.lookup == minimal_openapi_spec_dict + load_spec("mocked_path") + assert mock_open.from_file_path.call_count == 1 class TestHelpers: From dbc8b29cca3a999420ab72ef262449bd64d565c2 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Mon, 4 Mar 2024 13:18:01 -0500 Subject: [PATCH 06/22] Updated pydantic --- kytos/core/auth.py | 4 +- kytos/core/dead_letter.py | 4 +- kytos/core/user.py | 90 +++++++++++--------- requirements/dev.txt | 19 ++++- requirements/run.in | 6 +- requirements/run.txt | 13 ++- setup.py | 2 +- tests/conftest.py | 13 +++ tests/unit/test_core/test_auth.py | 10 +-- tests/unit/test_core/test_rest_api_routes.py | 6 +- tests/unit/test_core/test_user_controller.py | 2 +- tests/unit/test_core/test_users.py | 6 +- 12 files changed, 103 insertions(+), 72 deletions(-) diff --git a/kytos/core/auth.py b/kytos/core/auth.py index a9eabbbf7..11f3d80dc 100644 --- a/kytos/core/auth.py +++ b/kytos/core/auth.py @@ -25,7 +25,7 @@ content_type_json_or_415, error_msg, get_json_or_400) from kytos.core.retry import before_sleep, for_all_methods, retries -from kytos.core.user import HashSubDoc, UserDoc, UserDocUpdate +from kytos.core.user import HashSubDoc, UserDoc, UserDocUpdate, hashing __all__ = ['authenticated'] @@ -307,7 +307,7 @@ def _authenticate_user(self, request: Request) -> JSONResponse: user = self._find_user(username) if user["state"] != 'active': raise HTTPException(401, detail='This user is not active') - password_hashed = UserDoc.hashing(password, user["hash"]) + password_hashed = hashing(password, user["hash"]) if user["password"] != password_hashed: raise HTTPException(401, detail="Incorrect password") time_exp = datetime.datetime.utcnow() + datetime.timedelta( diff --git a/kytos/core/dead_letter.py b/kytos/core/dead_letter.py index 9a53db45d..00d128837 100644 --- a/kytos/core/dead_letter.py +++ b/kytos/core/dead_letter.py @@ -5,7 +5,7 @@ from typing import List # pylint: disable=no-name-in-module,invalid-name -from pydantic import BaseModel, ValidationError, constr +from pydantic import BaseModel, Field, ValidationError # pylint: enable=no-name-in-module from kytos.core.rest_api import (HTTPException, JSONResponse, Request, @@ -22,7 +22,7 @@ class KytosQueueBufferNames(str, Enum): class DeadLetterDeletePayload(BaseModel): """DeadLetterDeletePayload.""" - event_name: constr(min_length=1) + event_name: str = Field(min_length=1) ids: List[str] = [] diff --git a/kytos/core/user.py b/kytos/core/user.py index e741a98d6..5fde0b8b2 100644 --- a/kytos/core/user.py +++ b/kytos/core/user.py @@ -5,16 +5,18 @@ from datetime import datetime from typing import Literal, Optional -from pydantic import BaseModel, EmailStr, Field, constr, validator +from pydantic import (BaseModel, EmailStr, Field, ValidationInfo, + field_validator) +from typing_extensions import Annotated class DocumentBaseModel(BaseModel): """DocumentBaseModel""" id: str = Field(None, alias="_id") - inserted_at: Optional[datetime] - updated_at: Optional[datetime] - deleted_at: Optional[datetime] + inserted_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + deleted_at: Optional[datetime] = None def dict(self, **kwargs) -> dict: """Model to dict.""" @@ -26,14 +28,42 @@ def dict(self, **kwargs) -> dict: return values +def hashing(password: bytes, values: dict) -> str: + """Hash password and return it as string""" + return hashlib.scrypt(password=password, salt=values['salt'], + n=values['n'], r=values['r'], + p=values['p']).hex() + + +def validate_password(password: str, values: ValidationInfo): + """Check if password has at least a letter and a number""" + upper = False + lower = False + number = False + for char in password: + if char.isupper(): + upper = True + if char.isnumeric(): + number = True + if char.islower(): + lower = True + if number and upper and lower: + return hashing(password.encode(), values.data['hash'].dict()) + raise ValueError('value should contain ' + + 'minimun 8 characters, ' + + 'at least one upper case character, ' + + 'at least 1 numeric character [0-9]') + + class HashSubDoc(BaseModel): """HashSubDoc. Parameters for hash.scrypt function""" - salt: bytes = None + salt: bytes = Field(default=None, validate_default=True) n: int = 8192 r: int = 8 p: int = 1 - @validator('salt', pre=True, always=True) + @field_validator('salt', mode='before') + @classmethod def create_salt(cls, salt): """Create random salt value""" return salt or os.urandom(16) @@ -42,38 +72,15 @@ def create_salt(cls, salt): class UserDoc(DocumentBaseModel): """UserDocumentModel.""" - username: constr(min_length=1, max_length=64, regex="^[a-zA-Z0-9_-]+$") + username: str = Field( + min_length=1, max_length=64, pattern=r'^[a-zA-Z0-9_-]+$' + ) hash: HashSubDoc state: Literal['active', 'inactive'] = 'active' email: EmailStr - password: constr(min_length=8, max_length=64) - - @validator('password') - def validate_password(cls, password, values): - """Check if password has at least a letter and a number""" - upper = False - lower = False - number = False - for char in password: - if char.isupper(): - upper = True - if char.isnumeric(): - number = True - if char.islower(): - lower = True - if number and upper and lower: - return cls.hashing(password.encode(), values['hash'].dict()) - raise ValueError('value should contain ' + - 'minimun 8 characters, ' + - 'at least one upper case character, ' + - 'at least 1 numeric character [0-9]') + password: str = Field(min_length=8, max_length=64) - @staticmethod - def hashing(password: bytes, values: dict) -> str: - """Hash password and return it as string""" - return hashlib.scrypt(password=password, salt=values['salt'], - n=values['n'], r=values['r'], - p=values['p']).hex() + _validate_password = field_validator('password')(validate_password) @staticmethod def projection() -> dict: @@ -107,11 +114,12 @@ def projection_nopw() -> dict: class UserDocUpdate(DocumentBaseModel): "UserDocUpdate use to validate data before updating" - username: Optional[constr(min_length=1, max_length=64, - regex="^[a-zA-Z0-9_-]+$")] - email: Optional[EmailStr] - hash: Optional[HashSubDoc] - password: Optional[constr(min_length=8, max_length=64)] + username: Optional[Annotated[str, + Field(min_length=1, max_length=64, + pattern=r'^[a-zA-Z0-9_-]+$')]] = None + email: Optional[EmailStr] = None + hash: Optional[HashSubDoc] = None + password: Optional[Annotated[str, + Field(min_length=8, max_length=64)]] = None - _validate_password = validator('password', - allow_reuse=True)(UserDoc.validate_password) + _validate_password = field_validator('password')(validate_password) diff --git a/requirements/dev.txt b/requirements/dev.txt index 2947745fa..8feeef1a8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,6 +15,11 @@ # kytos alabaster==0.7.12 # via sphinx +annotated-types==0.6.0 + # via + # -r requirements/run.txt + # kytos + # pydantic anyio==4.3.0 # via # -r requirements/run.txt @@ -102,7 +107,7 @@ elastic-apm==6.20.0 # via # -r requirements/run.txt # kytos -email-validator==1.3.0 +email-validator==2.1.1 # via # -r requirements/run.txt # kytos @@ -309,10 +314,15 @@ pycodestyle==2.10.0 # via # kytos # yala -pydantic==1.9.0 +pydantic==2.6.3 # via # -r requirements/run.txt # kytos +pydantic-core==2.16.3 + # via + # -r requirements/run.txt + # kytos + # pydantic pygments==2.17.2 # via # -r requirements/run.txt @@ -342,7 +352,7 @@ pytest-asyncio==0.20.3 # via kytos pytest-cov==4.0.0 # via kytos -python-daemon==2.3.1 +python-daemon==3.0.1 # via # -r requirements/run.txt # kytos @@ -449,12 +459,13 @@ traitlets==5.14.1 # ipython # kytos # matplotlib-inline -typing-extensions==4.5.0 +typing-extensions==4.10.0 # via # -r requirements/run.txt # janus # kytos # pydantic + # pydantic-core urllib3==1.26.18 # via # -r requirements/run.txt diff --git a/requirements/run.in b/requirements/run.in index f288fda62..cbf50aaa8 100644 --- a/requirements/run.in +++ b/requirements/run.in @@ -3,15 +3,15 @@ ipython==8.22.1 lockfile==0.12.2 python-openflow -python-daemon==2.3.1 +python-daemon==3.0.1 janus==1.0.0 jinja2==3.1.3 watchdog==4.0.0 pyjwt==2.8.0 pymongo==4.6.2 -pydantic==1.9.0 +pydantic==2.6.3 dnspython==2.6.1 -email-validator==1.3.0 +email-validator==2.1.1 tenacity==8.2.3 elastic-apm==6.20.0 openapi-core==0.19.0 diff --git a/requirements/run.txt b/requirements/run.txt index a7cf3d1fa..83497ca15 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -4,6 +4,8 @@ # # pip-compile --output-file=requirements/run.txt requirements/run.in # +annotated-types==0.6.0 + # via pydantic anyio==4.3.0 # via # httpx @@ -40,7 +42,7 @@ ecs-logging==2.1.0 # via elastic-apm elastic-apm==6.20.0 # via -r requirements/run.in -email-validator==1.3.0 +email-validator==2.1.1 # via -r requirements/run.in executing==2.0.1 # via stack-data @@ -123,15 +125,17 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pydantic==1.9.0 +pydantic==2.6.3 # via -r requirements/run.in +pydantic-core==2.16.3 + # via pydantic pygments==2.17.2 # via ipython pyjwt==2.8.0 # via -r requirements/run.in pymongo==4.6.2 # via -r requirements/run.in -python-daemon==2.3.1 +python-daemon==3.0.1 # via -r requirements/run.in python-dotenv==1.0.1 # via uvicorn @@ -176,10 +180,11 @@ traitlets==5.14.1 # via # ipython # matplotlib-inline -typing-extensions==4.5.0 +typing-extensions==4.10.0 # via # janus # pydantic + # pydantic-core urllib3==1.26.18 # via # elastic-apm diff --git a/setup.py b/setup.py index e3c512bbd..c3c4ddd46 100644 --- a/setup.py +++ b/setup.py @@ -233,7 +233,7 @@ def run(self): 'yala==3.2.0', 'tox==3.28.0', 'virtualenv==20.21.0', - 'typing-extensions==4.5.0' + 'typing-extensions==4.10.0' ]}, cmdclass={ 'clean': Cleaner, diff --git a/tests/conftest.py b/tests/conftest.py index c269bd8b1..f1594759c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ """Unit test fixtures.""" # pylint: disable=redefined-outer-name +from unittest import mock + import pytest from httpx import AsyncClient @@ -52,3 +54,14 @@ def minimal_openapi_spec_dict(): } }, } + + +@pytest.fixture(scope='session', autouse=True) +def mock_hashing(): + """Mock hasinh method from user.py""" + def new_hashing(password: bytes, _hash) -> str: + if password == b"password": + return "some_hash" + return "wrong_hash" + with mock.patch("kytos.core.auth.hashing", wraps=new_hashing) as mock_all: + yield mock_all diff --git a/tests/unit/test_core/test_auth.py b/tests/unit/test_core/test_auth.py index bf86d08c1..2abd050a6 100644 --- a/tests/unit/test_core/test_auth.py +++ b/tests/unit/test_core/test_auth.py @@ -4,12 +4,11 @@ import pytest from httpx import AsyncClient # pylint: disable=no-name-in-module -from pydantic import BaseModel, ValidationError +from pydantic import ValidationError from pymongo.errors import DuplicateKeyError from kytos.core.auth import Auth from kytos.core.rest_api import HTTPException -from kytos.core.user import UserDoc # pylint: disable=unused-argument @@ -18,11 +17,6 @@ class TestAuth: def setup_method(self): """Instantiate a controller and an Auth.""" - def hashing(password: bytes, _hash) -> str: - if password == b"password": - return "some_hash" - return "wrong_hash" - UserDoc.hashing = hashing self.username, self.password = ("test", "password") self.user_data = { "username": "authtempuser", @@ -184,7 +178,7 @@ async def test_05_update_user_request_bad(self, auth, api_client, event_loop): """Test auth update user endpoint""" auth.controller.loop = event_loop - exc = ValidationError('', BaseModel) + exc = ValidationError.from_exception_data('', []) auth.user_controller.update_user.side_effect = exc endpoint = "kytos/core/auth/users/user5" headers = await self.auth_headers(auth, api_client) diff --git a/tests/unit/test_core/test_rest_api_routes.py b/tests/unit/test_core/test_rest_api_routes.py index fa5a8aa83..3aa2f5c80 100644 --- a/tests/unit/test_core/test_rest_api_routes.py +++ b/tests/unit/test_core/test_rest_api_routes.py @@ -92,9 +92,9 @@ async def test_error_msg(): with pytest.raises(ValidationError) as err: user.create_user(user_data) actual_msg = error_msg(err.value.errors()) - expected_msg = 'password: value should contain minimun 8 characters, ' \ - 'at least one upper case character, at least 1 ' \ - 'numeric character [0-9]' + expected_msg = 'password: Value error, value should contain minimun 8 ' \ + 'characters, at least one upper case character, at ' \ + 'least 1 numeric character [0-9]' assert actual_msg == expected_msg diff --git a/tests/unit/test_core/test_user_controller.py b/tests/unit/test_core/test_user_controller.py index 32397c938..86eaeb88a 100644 --- a/tests/unit/test_core/test_user_controller.py +++ b/tests/unit/test_core/test_user_controller.py @@ -96,7 +96,7 @@ def test_update_user_duplicate(self): db_base = self.user.db db_base.users.find_one_and_update.side_effect = DuplicateKeyError(0) with pytest.raises(DuplicateKeyError): - self.user.update_user({}, {}) + self.user.update_user({}, {"username": "mock"}) def test_get_user(self): """Test get_user""" diff --git a/tests/unit/test_core/test_users.py b/tests/unit/test_core/test_users.py index 2668f7af1..df1ce6901 100644 --- a/tests/unit/test_core/test_users.py +++ b/tests/unit/test_core/test_users.py @@ -6,7 +6,7 @@ import pytest from pydantic import ValidationError -from kytos.core.user import DocumentBaseModel, HashSubDoc, UserDoc +from kytos.core.user import DocumentBaseModel, HashSubDoc, UserDoc, hashing def test_document_base_model_dict(): @@ -98,6 +98,6 @@ def test_user_doc_projection_nopw(self): def test_user_doc_hashing(self): """Test UserDoc hashing of password""" user_doc = UserDoc(**self.user_data).dict() - pwd_hashed = UserDoc.hashing("Password123".encode(), - user_doc["hash"]) + pwd_hashed = hashing("Password123".encode(), + user_doc["hash"]) assert user_doc["password"] == pwd_hashed From 5eda4340f81187b92751e52889780d21c133db73 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 5 Mar 2024 07:22:41 -0500 Subject: [PATCH 07/22] Updated testing libraries --- kytos/core/api_server.py | 3 +-- kytos/core/connection.py | 1 + kytos/core/helpers.py | 4 ++-- requirements/dev.txt | 50 ++++++++++++++++++++++------------------ setup.py | 18 +++++++-------- tests/helper.py | 1 + 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/kytos/core/api_server.py b/kytos/core/api_server.py index 214a9cb7f..9a430ccd1 100644 --- a/kytos/core/api_server.py +++ b/kytos/core/api_server.py @@ -404,8 +404,7 @@ def _get_decorated_functions(napp): and isinstance(pub_attr.route_index, int) ): callables.append(pub_attr) - for pub_attr in sorted(callables, key=lambda f: f.route_index): - yield pub_attr + yield from sorted(callables, key=lambda f: f.route_index) @classmethod def get_absolute_rule(cls, rule, napp): diff --git a/kytos/core/connection.py b/kytos/core/connection.py index 6123b4d81..a9f2ecd39 100644 --- a/kytos/core/connection.py +++ b/kytos/core/connection.py @@ -74,6 +74,7 @@ def state(self): @state.setter def state(self, new_state): if new_state not in ConnectionState: + # pylint: disable=broad-exception-raised raise Exception('Unknown State', new_state) # pylint: disable=attribute-defined-outside-init self._state = new_state diff --git a/kytos/core/helpers.py b/kytos/core/helpers.py index ed8eab3df..a5e9236cb 100644 --- a/kytos/core/helpers.py +++ b/kytos/core/helpers.py @@ -177,7 +177,7 @@ def handler_context_apm(*args, apm_client=None): handler_func, kwargs = handler_context, {} if get_apm_name() == "es": handler_func = handler_context_apm - kwargs = dict(apm_client=ElasticAPM.get_client()) + kwargs = {"apm_client": ElasticAPM.get_client()} def get_executor(pool, event, default_pool="app", handler=handler): """Get executor.""" @@ -267,7 +267,7 @@ async def handler_context_apm(*args, apm_client=None): handler_func, kwargs = handler_context, {} if get_apm_name() == "es": handler_func = handler_context_apm - kwargs = dict(apm_client=ElasticAPM.get_client()) + kwargs = {"apm_client": ElasticAPM.get_client()} async def inner(*args): """Inner decorated with events attribute.""" diff --git a/requirements/dev.txt b/requirements/dev.txt index 8feeef1a8..9826781d4 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -31,7 +31,7 @@ asgiref==3.7.2 # via # -r requirements/run.txt # kytos -astroid==2.13.5 +astroid==3.1.0 # via pylint asttokens==2.4.1 # via @@ -43,14 +43,15 @@ attrs==23.2.0 # -r requirements/run.txt # jsonschema # kytos - # pytest # referencing babel==2.9.0 # via sphinx -black==23.3.0 +black==24.2.0 # via kytos build==0.10.0 # via pip-tools +cachetools==5.3.3 + # via tox certifi==2024.2.2 # via # -r requirements/run.txt @@ -59,6 +60,8 @@ certifi==2024.2.2 # httpx # kytos # requests +chardet==5.2.0 + # via tox charset-normalizer==3.3.2 # via # -r requirements/run.txt @@ -72,7 +75,9 @@ click==8.1.7 # pip-tools # uvicorn colorama==0.4.6 - # via sphinx-autobuild + # via + # sphinx-autobuild + # tox coverage[toml]==7.2.2 # via pytest-cov decorator==5.1.1 @@ -80,9 +85,9 @@ decorator==5.1.1 # -r requirements/run.txt # ipython # kytos -dill==0.3.4 +dill==0.3.8 # via pylint -distlib==0.3.6 +distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via @@ -116,7 +121,7 @@ executing==2.0.1 # -r requirements/run.txt # kytos # stack-data -filelock==3.10.7 +filelock==3.13.1 # via # tox # virtualenv @@ -162,7 +167,7 @@ isodate==0.6.1 # -r requirements/run.txt # kytos # openapi-core -isort==5.12.0 +isort==5.13.2 # via # kytos # pylint @@ -209,7 +214,6 @@ jsonschema-specifications==2023.12.1 lazy-object-proxy==1.7.1 # via # -r requirements/run.txt - # astroid # kytos # openapi-spec-validator livereload==2.6.3 @@ -253,10 +257,11 @@ openapi-spec-validator==0.7.1 # -r requirements/run.txt # kytos # openapi-core -packaging==23.0 +packaging==23.2 # via # black # build + # pyproject-api # pytest # sphinx # tox @@ -284,12 +289,13 @@ pexpect==4.9.0 # kytos pip-tools==6.12.3 # via kytos -platformdirs==3.2.0 +platformdirs==4.2.0 # via # black # pylint + # tox # virtualenv -pluggy==1.0.0 +pluggy==1.4.0 # via # pytest # tox @@ -308,9 +314,7 @@ pure-eval==0.2.2 # -r requirements/run.txt # kytos # stack-data -py==1.11.0 - # via tox -pycodestyle==2.10.0 +pycodestyle==2.11.1 # via # kytos # yala @@ -333,7 +337,7 @@ pyjwt==2.8.0 # via # -r requirements/run.txt # kytos -pylint==2.15.0 +pylint==3.1.0 # via # kytos # yala @@ -341,16 +345,18 @@ pymongo==4.6.2 # via # -r requirements/run.txt # kytos +pyproject-api==1.6.1 + # via tox pyproject-hooks==1.0.0 # via build -pytest==7.2.1 +pytest==8.0.1 # via # kytos # pytest-asyncio # pytest-cov -pytest-asyncio==0.20.3 +pytest-asyncio==0.23.5 # via kytos -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via kytos python-daemon==3.0.1 # via @@ -407,7 +413,6 @@ six==1.16.0 # kytos # livereload # rfc3339-validator - # tox sniffio==1.3.0 # via # -r requirements/run.txt @@ -451,7 +456,7 @@ tomlkit==0.11.7 # via pylint tornado==6.2 # via livereload -tox==3.28.0 +tox==4.13.0 # via kytos traitlets==5.14.1 # via @@ -481,7 +486,7 @@ uvloop==0.19.0 # -r requirements/run.txt # kytos # uvicorn -virtualenv==20.21.0 +virtualenv==20.25.1 # via # kytos # tox @@ -514,7 +519,6 @@ wheel==0.40.0 wrapt==1.14.1 # via # -r requirements/run.txt - # astroid # elastic-apm # kytos yala==3.2.0 diff --git a/setup.py b/setup.py index c3c4ddd46..5eb79172a 100644 --- a/setup.py +++ b/setup.py @@ -223,16 +223,16 @@ def run(self): install_requires=install_requires, extras_require={'dev': [ 'pip-tools >= 2.0', - 'pytest==7.2.1', - 'pytest-cov==4.0.0', - 'pytest-asyncio==0.20.3', - 'black==23.3.0', - 'isort==5.12.0', - 'pylint==2.15.0', - 'pycodestyle==2.10.0', + 'pytest==8.0.1', + 'pytest-cov==4.1.0', + 'pytest-asyncio==0.23.5', + 'black==24.2.0', + 'isort==5.13.2', + 'pylint==3.1.0', + 'pycodestyle==2.11.1', 'yala==3.2.0', - 'tox==3.28.0', - 'virtualenv==20.21.0', + 'tox==4.13.0', + 'virtualenv==20.25.1', 'typing-extensions==4.10.0' ]}, cmdclass={ diff --git a/tests/helper.py b/tests/helper.py index f5e7ab31e..774068642 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -125,6 +125,7 @@ def new_handshaked_client(options=None): return do_handshake(client) +# pylint: disable=broad-exception-raised def test_concurrently(times): """ Decorator to test concurrently many times. From 663bb5614c9abbd88c2be627b9278b2cd1121f5b Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 5 Mar 2024 08:21:50 -0500 Subject: [PATCH 08/22] Updated the rest of dependencies. --- requirements/dev.txt | 10 +++++----- requirements/run.txt | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 9826781d4..88755b85a 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -211,7 +211,7 @@ jsonschema-specifications==2023.12.1 # jsonschema # kytos # openapi-schema-validator -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.10.0 # via # -r requirements/run.txt # kytos @@ -235,7 +235,7 @@ matplotlib-inline==0.1.6 # kytos mccabe==0.7.0 # via pylint -more-itertools==8.12.0 +more-itertools==10.2.0 # via # -r requirements/run.txt # kytos @@ -265,7 +265,7 @@ packaging==23.2 # pytest # sphinx # tox -parse==1.19.0 +parse==1.20.1 # via # -r requirements/run.txt # kytos @@ -374,7 +374,7 @@ python-multipart==0.0.9 # starlette pytz==2021.3 # via babel -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/run.txt # jsonschema-path @@ -413,7 +413,7 @@ six==1.16.0 # kytos # livereload # rfc3339-validator -sniffio==1.3.0 +sniffio==1.3.1 # via # -r requirements/run.txt # anyio diff --git a/requirements/run.txt b/requirements/run.txt index 83497ca15..83535e47c 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -91,7 +91,7 @@ jsonschema-specifications==2023.12.1 # via # jsonschema # openapi-schema-validator -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.10.0 # via openapi-spec-validator lockfile==0.12.2 # via @@ -101,7 +101,7 @@ markupsafe==2.1.5 # via jinja2 matplotlib-inline==0.1.6 # via ipython -more-itertools==8.12.0 +more-itertools==10.2.0 # via openapi-core openapi-core==0.19.0 # via -r requirements/run.in @@ -111,7 +111,7 @@ openapi-schema-validator==0.6.2 # openapi-spec-validator openapi-spec-validator==0.7.1 # via openapi-core -parse==1.19.0 +parse==1.20.1 # via openapi-core parso==0.8.3 # via jedi @@ -143,7 +143,7 @@ python-multipart==0.0.9 # via starlette python-openflow # via -r requirements/run.in -pyyaml==6.0 +pyyaml==6.0.1 # via # jsonschema-path # starlette @@ -166,7 +166,7 @@ six==1.16.0 # asttokens # isodate # rfc3339-validator -sniffio==1.3.0 +sniffio==1.3.1 # via # anyio # httpx From 606f0f994523c542c83e92741831e15d3f1bd9d9 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Wed, 6 Mar 2024 06:44:26 -0500 Subject: [PATCH 09/22] Reverted change --- kytos/core/rest_api.py | 5 +++++ tox.ini | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 01a6a3fd0..1b19a51b4 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -9,6 +9,11 @@ from starlette.exceptions import HTTPException from starlette.requests import Request from starlette.responses import JSONResponse as StarletteJSONResponse +from starlette.responses import Response + +Request = Request +Response = Response +HTTPException = HTTPException def _json_serializer(obj): diff --git a/tox.ini b/tox.ini index 1d415c068..f1718a7a5 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ python = 3.11: py311 [testenv] -whitelist_externals= +allowlist_externals= rm make From 67de521ba95b90135422eea553cbe26a4691ad40 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 12 Mar 2024 06:31:31 -0400 Subject: [PATCH 10/22] Updated pydantic documents --- kytos/core/auth.py | 4 ++-- kytos/core/user.py | 6 +++--- requirements/dev.in | 2 +- requirements/dev.txt | 2 +- tests/unit/test_core/test_users.py | 12 ++++++------ 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/kytos/core/auth.py b/kytos/core/auth.py index 11f3d80dc..221295557 100644 --- a/kytos/core/auth.py +++ b/kytos/core/auth.py @@ -110,7 +110,7 @@ def create_user(self, user_data: dict) -> InsertOneResult: "email": user_data.get('email'), "inserted_at": utc_now, "updated_at": utc_now, - }).dict()) + }).model_dump()) except DuplicateKeyError as err: raise err except ValidationError as err: @@ -142,7 +142,7 @@ def update_user(self, username: str, data: dict) -> dict: "$set": UserDocUpdate(**{ **data, **{"updated_at": utc_now} - }).dict(exclude_none=True) + }).model_dump(exclude_none=True) }, return_document=ReturnDocument.AFTER ) diff --git a/kytos/core/user.py b/kytos/core/user.py index 5fde0b8b2..a3b91eca3 100644 --- a/kytos/core/user.py +++ b/kytos/core/user.py @@ -18,9 +18,9 @@ class DocumentBaseModel(BaseModel): updated_at: Optional[datetime] = None deleted_at: Optional[datetime] = None - def dict(self, **kwargs) -> dict: + def model_dump(self, **kwargs) -> dict: """Model to dict.""" - values = super().dict(**kwargs) + values = super().model_dump(**kwargs) if "id" in values and values["id"]: values["_id"] = values["id"] if "exclude" in kwargs and "_id" in kwargs["exclude"]: @@ -48,7 +48,7 @@ def validate_password(password: str, values: ValidationInfo): if char.islower(): lower = True if number and upper and lower: - return hashing(password.encode(), values.data['hash'].dict()) + return hashing(password.encode(), values.data['hash'].model_dump()) raise ValueError('value should contain ' + 'minimun 8 characters, ' + 'at least one upper case character, ' + diff --git a/requirements/dev.in b/requirements/dev.in index 6e1b2e9cb..4bdf27572 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,3 +1,3 @@ --e git+https://github.com/kytos-ng/python-openflow.git#egg=python-openflow +-e git+https://github.com/kytos-ng/python-openflow.git@upgrade/python#egg=python-openflow -e git+https://github.com/kytos-ng/sphinx-theme.git#egg=kytos-sphinx-theme -e .[dev] diff --git a/requirements/dev.txt b/requirements/dev.txt index 88755b85a..01283a95d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ # via -r requirements/dev.in -e git+https://github.com/kytos-ng/sphinx-theme.git#egg=kytos-sphinx-theme # via -r requirements/dev.in --e git+https://github.com/kytos-ng/python-openflow.git#egg=python-openflow +-e git+https://github.com/kytos-ng/python-openflow.git@upgrade/python#egg=python-openflow # via # -r requirements/dev.in # -r requirements/run.txt diff --git a/tests/unit/test_core/test_users.py b/tests/unit/test_core/test_users.py index df1ce6901..305c54b7a 100644 --- a/tests/unit/test_core/test_users.py +++ b/tests/unit/test_core/test_users.py @@ -10,13 +10,13 @@ def test_document_base_model_dict(): - """Test DocumentBaseModel.dict()""" + """Test DocumentBaseModel.model_dump()""" _id = "random_id" utc_now = datetime.utcnow() payload = {"_id": _id, "inserted_at": utc_now, "updated_at": utc_now} model = DocumentBaseModel(**payload) - assert model.dict(exclude_none=True) == {**payload, **{"id": _id}} - assert "_id" not in model.dict(exclude={"_id"}) + assert model.model_dump(exclude_none=True) == {**payload, **{"id": _id}} + assert "_id" not in model.model_dump(exclude={"_id"}) class TestUserDoc: @@ -32,13 +32,13 @@ def setup_method(self): } def test_user_doc_dict(self): - """Test UserDocModel.dict()""" + """Test UserDocModel.model_dump()""" correct_hash = { "n": 8192, "r": 8, "p": 1 } - user_doc = UserDoc(**self.user_data).dict() + user_doc = UserDoc(**self.user_data).model_dump() user_hash = user_doc["hash"] assert user_doc["username"] == self.user_data["username"] assert user_doc["email"] == self.user_data["email"] @@ -97,7 +97,7 @@ def test_user_doc_projection_nopw(self): def test_user_doc_hashing(self): """Test UserDoc hashing of password""" - user_doc = UserDoc(**self.user_data).dict() + user_doc = UserDoc(**self.user_data).model_dump() pwd_hashed = hashing("Password123".encode(), user_doc["hash"]) assert user_doc["password"] == pwd_hashed From 6b1076e32627fb8e7f71b3b0e792d35bf99009b0 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 12 Mar 2024 08:14:14 -0400 Subject: [PATCH 11/22] Update to gracefully close elastic --- kytos/core/controller.py | 8 ++++- tests/unit/test_core/test_auth.py | 32 +++++++++----------- tests/unit/test_core/test_controller.py | 5 ++- tests/unit/test_core/test_rest_api_routes.py | 8 ++--- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/kytos/core/controller.py b/kytos/core/controller.py index 733c24502..40af9b0fe 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -154,6 +154,9 @@ def __init__(self, options=None, loop: AbstractEventLoop = None): self._alisten_tasks = set() self.qmonitors: list[QueueMonitorWindow] = [] + #: APM client in memory to be closed when necessary + self.apm = None + self._register_endpoints() #: Adding the napps 'enabled' directory into the PATH #: Now you can access the enabled napps with: @@ -258,7 +261,7 @@ def start(self, restart=False): db_conn_wait(db_backend=self.options.database) self.start_auth() if self.options.apm: - init_apm(self.options.apm, app=self.api_server.app) + self.apm = init_apm(self.options.apm, app=self.api_server.app) if not restart: self.create_pidfile() self.start_controller() @@ -508,6 +511,9 @@ def stop_controller(self, graceful=True): # self.server.server_close() self.stop_queue_monitors() + if self.apm: + self.log.info("Stopping APM server...") + self.apm.close() self.log.info("Stopping API Server...") self.api_server.stop() self.log.info("Stopped API Server") diff --git a/tests/unit/test_core/test_auth.py b/tests/unit/test_core/test_auth.py index 2abd050a6..7cd655365 100644 --- a/tests/unit/test_core/test_auth.py +++ b/tests/unit/test_core/test_auth.py @@ -1,4 +1,5 @@ """Test kytos.core.auth module.""" +import asyncio import base64 import pytest @@ -97,9 +98,9 @@ async def test_02_list_users_request(self, auth, api_client): resp = await api_client.get(endpoint, headers=invalid_header) assert resp.status_code == 401 - async def test_03_create_user_request(self, auth, api_client, event_loop): + async def test_03_create_user_request(self, auth, api_client): """Test auth create user endpoint.""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() endpoint = "kytos/core/auth/users" headers = await self.auth_headers(auth, api_client) resp = await api_client.post(endpoint, json=self.user_data, @@ -108,10 +109,10 @@ async def test_03_create_user_request(self, auth, api_client, event_loop): assert "User successfully created" in resp.json() async def test_03_create_user_request_conflict( - self, auth, api_client, event_loop + self, auth, api_client ): """Test auth create user endpoint.""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() endpoint = "kytos/core/auth/users" auth.user_controller.create_user.side_effect = DuplicateKeyError("0") headers = await self.auth_headers(auth, api_client) @@ -121,10 +122,10 @@ async def test_03_create_user_request_conflict( assert "already exists" in resp.json()["description"] async def test_03_create_user_request_bad( - self, auth, api_client, event_loop + self, auth, api_client ): """Test auth create user endpoint.""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() data = "wrong_json" endpoint = "kytos/core/auth/users" headers = await self.auth_headers(auth, api_client) @@ -151,9 +152,9 @@ async def test_04_list_user_request_error(self, auth, api_client): assert resp.status_code == 404 assert "not found" in resp.json()["description"] - async def test_05_update_user_request(self, auth, api_client, event_loop): + async def test_05_update_user_request(self, auth, api_client): """Test auth update user endpoint.""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() data = {"email": "newemail_tempuser@kytos.io"} endpoint = f"kytos/core/auth/users/{self.user_data['username']}" headers = await self.auth_headers(auth, api_client) @@ -161,10 +162,9 @@ async def test_05_update_user_request(self, auth, api_client, event_loop): headers=headers) assert resp.status_code == 200 - async def test_05_update_user_request_not_found(self, auth, api_client, - event_loop): + async def test_05_update_user_request_not_found(self, auth, api_client): """Test auth update user endpoint.""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() data = {"email": "newemail_tempuser@kytos.io"} auth.user_controller.update_user.return_value = {} endpoint = "kytos/core/auth/users/user5" @@ -174,10 +174,9 @@ async def test_05_update_user_request_not_found(self, auth, api_client, assert resp.status_code == 404 assert "not found" in resp.json()["description"] - async def test_05_update_user_request_bad(self, auth, api_client, - event_loop): + async def test_05_update_user_request_bad(self, auth, api_client): """Test auth update user endpoint""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() exc = ValidationError.from_exception_data('', []) auth.user_controller.update_user.side_effect = exc endpoint = "kytos/core/auth/users/user5" @@ -186,10 +185,9 @@ async def test_05_update_user_request_bad(self, auth, api_client, headers=headers) assert resp.status_code == 400 - async def test_05_update_user_request_conflict(self, auth, api_client, - event_loop): + async def test_05_update_user_request_conflict(self, auth, api_client): """Test auth update user endpoint""" - auth.controller.loop = event_loop + auth.controller.loop = asyncio.get_running_loop() auth.user_controller.update_user.side_effect = DuplicateKeyError("0") endpoint = "kytos/core/auth/users/user5" headers = await self.auth_headers(auth, api_client) diff --git a/tests/unit/test_core/test_controller.py b/tests/unit/test_core/test_controller.py index 04d1edfe2..bda65fe4c 100644 --- a/tests/unit/test_core/test_controller.py +++ b/tests/unit/test_core/test_controller.py @@ -182,6 +182,7 @@ def test_start(self, *args): mock_start_controller.assert_called() mock_db_conn_wait.assert_not_called() mock_init_apm.assert_not_called() + assert self.controller.apm is None @patch('kytos.core.controller.sys') @patch('kytos.core.controller.init_apm') @@ -223,6 +224,7 @@ def test_start_with_mongodb_and_apm(self, *args): mock_start_controller.assert_called() mock_db_conn_wait.assert_called() mock_init_apm.assert_called() + assert self.controller.apm is not None @patch('kytos.core.controller.sys.exit') @patch('kytos.core.controller.Controller.create_pidfile') @@ -749,9 +751,10 @@ async def test_stop_controller(self, controller): controller.api_server = api_server controller.napp_dir_listener = napp_dir_listener controller.stop_queue_monitors = MagicMock() + controller.apm = MagicMock() controller.stop_controller() - + controller.apm.close.assert_called() controller.buffers.send_stop_signal.assert_called() api_server.stop.assert_called() napp_dir_listener.stop.assert_called() diff --git a/tests/unit/test_core/test_rest_api_routes.py b/tests/unit/test_core/test_rest_api_routes.py index 3aa2f5c80..ea69f92be 100644 --- a/tests/unit/test_core/test_rest_api_routes.py +++ b/tests/unit/test_core/test_rest_api_routes.py @@ -61,10 +61,10 @@ def test_api_500_traceback_by_default() -> None: assert KytosConfig().options["daemon"].api_traceback_on_500 -async def test_get_json_or_400(controller, api_client, event_loop) -> None: +async def test_get_json_or_400(controller, api_client) -> None: """Test get_json_or_400.""" - controller.loop = event_loop + controller.loop = asyncio.get_running_loop() def handler(request: Request) -> JSONResponse: body = get_json_or_400(request, controller.loop) @@ -98,9 +98,9 @@ async def test_error_msg(): assert actual_msg == expected_msg -async def test_get_body(controller, api_client, event_loop) -> None: +async def test_get_body(controller, api_client) -> None: """Test get_body (low level-ish usage for validators).""" - controller.loop = event_loop + controller.loop = asyncio.get_running_loop() def handler(request: Request) -> JSONResponse: body_bytes = get_body(request, controller.loop) From cdd07d05152f0da631537704a9d39988b3ae3792 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Wed, 13 Mar 2024 10:35:18 -0400 Subject: [PATCH 12/22] Installed `openapi-core==0.18.2` --- kytos/core/helpers.py | 73 +++++++++++++--------------- kytos/core/rest_api.py | 51 ++++++++++++++++++- requirements/dev.txt | 19 ++++++-- requirements/run.in | 4 +- requirements/run.txt | 28 +++++++---- tests/unit/test_core/test_helpers.py | 10 ++-- 6 files changed, 125 insertions(+), 60 deletions(-) diff --git a/kytos/core/helpers.py b/kytos/core/helpers.py index a5e9236cb..9b789e6ee 100644 --- a/kytos/core/helpers.py +++ b/kytos/core/helpers.py @@ -9,13 +9,15 @@ from pathlib import Path from threading import Thread -from openapi_core import OpenAPI -from openapi_core.contrib.starlette import StarletteOpenAPIRequest -from openapi_core.unmarshalling.request.datatypes import RequestUnmarshalResult +from openapi_core import Spec, unmarshal_request +from openapi_core.exceptions import OpenAPIError +from openapi_spec_validator import validate +from openapi_spec_validator.readers import read_from_filename from kytos.core.apm import ElasticAPM from kytos.core.config import KytosConfig -from kytos.core.rest_api import (HTTPException, Request, +from kytos.core.rest_api import (AStarletteOpenAPIRequest, HTTPException, + Request, StarletteOpenAPIRequest, content_type_json_or_415, get_body) __all__ = ['listen_to', 'now', 'run_on_thread', 'get_time'] @@ -347,44 +349,35 @@ def get_time(data=None): return date.replace(tzinfo=timezone.utc) +def _read_from_filename(yml_file_path: Path) -> dict: + """Read from yml filename.""" + spec_dict, _ = read_from_filename(yml_file_path) + return spec_dict + + def load_spec(yml_file_path: Path): """Load and validate spec object given a yml file path.""" - return OpenAPI.from_file_path(yml_file_path) + spec = _read_from_filename(yml_file_path) + validate(spec) + return Spec.from_dict(spec) -def _request_validation_result_or_400(result: RequestUnmarshalResult) -> None: - """Request validation result or raise HTTP 400.""" - if not result.errors: - return - error_response = ( - "The request body contains invalid API data." - ) - errors = result.errors[0] +def _request_validation_result_or_400(errors: OpenAPIError) -> None: + """Raise HTTP 400.""" + error_response = "The request body contains invalid API data." if not errors.__cause__: error_response = str(errors) - elif hasattr(errors.__cause__, "schema_errors"): - errors = errors.__cause__ - schema_errors = errors.schema_errors[0] - error_log = { - "error_message": schema_errors.message, - "error_validator": schema_errors.validator, - "error_validator_value": schema_errors.validator_value, - "error_path": list(schema_errors.path), - "error_schema": schema_errors.schema, - "error_schema_path": list(schema_errors.schema_path), - } - LOG.debug(f"Invalid request (API schema): {error_log}") - error_response += f" {schema_errors.message} for field" - error_response += ( - f" {'/'.join(map(str,schema_errors.path))}." - ) - else: - error_response = str(errors.__cause__) + elif (hasattr(errors.__cause__, "schema_errors") and + errors.__cause__.schema_errors): + schema_errors = errors.__cause__.schema_errors + error_response += f" {schema_errors[0].message}" + for error in schema_errors[1:]: + error_response += f", {error.message}" raise HTTPException(400, detail=error_response) def validate_openapi_request( - spec: OpenAPI, request: Request, loop: AbstractEventLoop + spec: Spec, request: Request, loop: AbstractEventLoop ) -> bytes: """Validate a Request given an OpenAPI spec. @@ -396,13 +389,15 @@ def validate_openapi_request( if body: content_type_json_or_415(request) openapi_request = StarletteOpenAPIRequest(request, body) - result = spec.unmarshal_request(openapi_request) - _request_validation_result_or_400(result) + try: + unmarshal_request(openapi_request, spec) + except OpenAPIError as err: + _request_validation_result_or_400(err) return body async def avalidate_openapi_request( - spec: OpenAPI, + spec: Spec, request: Request, ) -> bytes: """Async validate_openapi_request. @@ -420,9 +415,11 @@ async def avalidate_openapi_request( body = await request.body() if body: content_type_json_or_415(request) - openapi_request = StarletteOpenAPIRequest(request, body) - result = spec.unmarshal_request(openapi_request) - _request_validation_result_or_400(result) + openapi_request = AStarletteOpenAPIRequest(request, body) + try: + unmarshal_request(openapi_request, spec) + except OpenAPIError as err: + _request_validation_result_or_400(err) return body diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 1b19a51b4..0b7555419 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -1,11 +1,14 @@ """Rest API utilities module.""" -# pylint: disable=self-assigning-variable +# pylint: disable=self-assigning-variable, super-init-not-called import asyncio import json from asyncio import AbstractEventLoop from datetime import datetime from typing import Any, Optional +from openapi_core.contrib.starlette import \ + StarletteOpenAPIRequest as _StarletteOpenAPIRequest +from openapi_core.validation.request.datatypes import RequestParameters from starlette.exceptions import HTTPException from starlette.requests import Request from starlette.responses import JSONResponse as StarletteJSONResponse @@ -96,3 +99,49 @@ def render(self, content) -> bytes: separators=(",", ":"), default=_json_serializer, ).encode("utf-8") + + +class AStarletteOpenAPIRequest(_StarletteOpenAPIRequest): + """Async StarletteOpenAPIRequest.""" + + def __init__(self, request: Request, body: bytes) -> None: + """Constructor of AsycnStarletteOpenAPIRequest. + This constructor doesn't call super().__init__() to keep it async + """ + self.request = request + self.parameters = RequestParameters( + query=self.request.query_params, + header=self.request.headers, + cookie=self.request.cookies, + ) + self._body = body + + @property + def body(self) -> Optional[str]: + body = self._body + if body is None: + return None + return body.decode("utf-8") + + +class StarletteOpenAPIRequest(_StarletteOpenAPIRequest): + """Sync StarletteOpenAPIRequest.""" + + def __init__(self, request: Request, body: bytes) -> None: + """Constructor of AsycnStarletteOpenAPIRequest. + This constructor doesn't call super().__init__() to keep it async + """ + self.request = request + self.parameters = RequestParameters( + query=self.request.query_params, + header=self.request.headers, + cookie=self.request.cookies, + ) + self._body = body + + @property + def body(self) -> Optional[str]: + body = self._body + if body is None: + return None + return body.decode("utf-8") diff --git a/requirements/dev.txt b/requirements/dev.txt index 01283a95d..1fb41398e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -31,6 +31,7 @@ asgiref==3.7.2 # via # -r requirements/run.txt # kytos + # openapi-core astroid==3.1.0 # via pylint asttokens==2.4.1 @@ -203,9 +204,13 @@ jsonschema-path==0.3.2 # via # -r requirements/run.txt # kytos - # openapi-core # openapi-spec-validator -jsonschema-specifications==2023.12.1 +jsonschema-spec==0.2.4 + # via + # -r requirements/run.txt + # kytos + # openapi-core +jsonschema-specifications==2023.7.1 # via # -r requirements/run.txt # jsonschema @@ -242,7 +247,7 @@ more-itertools==10.2.0 # openapi-core mypy-extensions==1.0.0 # via black -openapi-core==0.19.0 +openapi-core==0.18.2 # via # -r requirements/run.txt # kytos @@ -279,6 +284,7 @@ pathable==0.4.3 # via # -r requirements/run.txt # jsonschema-path + # jsonschema-spec # kytos pathspec==0.11.1 # via black @@ -378,20 +384,23 @@ pyyaml==6.0.1 # via # -r requirements/run.txt # jsonschema-path + # jsonschema-spec # kytos # starlette # uvicorn -referencing==0.31.1 +referencing==0.30.2 # via # -r requirements/run.txt # jsonschema # jsonschema-path + # jsonschema-spec # jsonschema-specifications # kytos requests==2.31.0 # via # -r requirements/run.txt # jsonschema-path + # jsonschema-spec # kytos # sphinx rfc3339-validator==0.1.4 @@ -444,7 +453,7 @@ stack-data==0.6.3 # -r requirements/run.txt # ipython # kytos -starlette[full]==0.37.1 +starlette[full]==0.31.1 # via # -r requirements/run.txt # kytos diff --git a/requirements/run.in b/requirements/run.in index cbf50aaa8..aa7e30925 100644 --- a/requirements/run.in +++ b/requirements/run.in @@ -14,8 +14,8 @@ dnspython==2.6.1 email-validator==2.1.1 tenacity==8.2.3 elastic-apm==6.20.0 -openapi-core==0.19.0 +openapi-core==0.18.2 httpx==0.27.0 -starlette[full]==0.37.1 +starlette[full]==0.31.1 uvicorn[standard]==0.27.1 asgiref==3.7.2 diff --git a/requirements/run.txt b/requirements/run.txt index 83535e47c..c719a9bbb 100644 --- a/requirements/run.txt +++ b/requirements/run.txt @@ -12,7 +12,9 @@ anyio==4.3.0 # starlette # watchfiles asgiref==3.7.2 - # via -r requirements/run.in + # via + # -r requirements/run.in + # openapi-core asttokens==2.4.1 # via stack-data attrs==23.2.0 @@ -84,10 +86,10 @@ jsonschema==4.21.1 # openapi-schema-validator # openapi-spec-validator jsonschema-path==0.3.2 - # via - # openapi-core - # openapi-spec-validator -jsonschema-specifications==2023.12.1 + # via openapi-spec-validator +jsonschema-spec==0.2.4 + # via openapi-core +jsonschema-specifications==2023.7.1 # via # jsonschema # openapi-schema-validator @@ -103,7 +105,7 @@ matplotlib-inline==0.1.6 # via ipython more-itertools==10.2.0 # via openapi-core -openapi-core==0.19.0 +openapi-core==0.18.2 # via -r requirements/run.in openapi-schema-validator==0.6.2 # via @@ -116,7 +118,9 @@ parse==1.20.1 parso==0.8.3 # via jedi pathable==0.4.3 - # via jsonschema-path + # via + # jsonschema-path + # jsonschema-spec pexpect==4.9.0 # via ipython prompt-toolkit==3.0.43 @@ -146,15 +150,19 @@ python-openflow pyyaml==6.0.1 # via # jsonschema-path + # jsonschema-spec # starlette # uvicorn -referencing==0.31.1 +referencing==0.30.2 # via # jsonschema # jsonschema-path + # jsonschema-spec # jsonschema-specifications requests==2.31.0 - # via jsonschema-path + # via + # jsonschema-path + # jsonschema-spec rfc3339-validator==0.1.4 # via openapi-schema-validator rpds-py==0.18.0 @@ -172,7 +180,7 @@ sniffio==1.3.1 # httpx stack-data==0.6.3 # via ipython -starlette[full]==0.37.1 +starlette[full]==0.31.1 # via -r requirements/run.in tenacity==8.2.3 # via -r requirements/run.in diff --git a/tests/unit/test_core/test_helpers.py b/tests/unit/test_core/test_helpers.py index eb3031d38..0f1920600 100644 --- a/tests/unit/test_core/test_helpers.py +++ b/tests/unit/test_core/test_helpers.py @@ -1,6 +1,7 @@ """Test kytos.core.helpers module.""" from unittest.mock import MagicMock, patch +from kytos.core import helpers from kytos.core.helpers import (alisten_to, ds_executors, executors, get_thread_pool_max_workers, get_time, listen_to, load_spec, run_on_thread) @@ -23,11 +24,12 @@ async def on_some_event(self, event): assert result == "some_response" -@patch('kytos.core.helpers.OpenAPI') -def test_load_spec(mock_open) -> None: +def test_load_spec(monkeypatch, minimal_openapi_spec_dict) -> None: """Test load spec.""" - load_spec("mocked_path") - assert mock_open.from_file_path.call_count == 1 + monkeypatch.setattr(helpers, "_read_from_filename", + lambda x: minimal_openapi_spec_dict) + spec = load_spec("mocked_path") + assert spec.accessor.lookup == minimal_openapi_spec_dict class TestHelpers: From 0ddad413686ec80616e0a90a43eeffbaead85b95 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 14 Mar 2024 10:12:40 -0400 Subject: [PATCH 13/22] Overwritten property --- kytos/core/helpers.py | 4 ++-- kytos/core/rest_api.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kytos/core/helpers.py b/kytos/core/helpers.py index 9b789e6ee..0bcf202a3 100644 --- a/kytos/core/helpers.py +++ b/kytos/core/helpers.py @@ -11,7 +11,7 @@ from openapi_core import Spec, unmarshal_request from openapi_core.exceptions import OpenAPIError -from openapi_spec_validator import validate +from openapi_spec_validator import validate_spec from openapi_spec_validator.readers import read_from_filename from kytos.core.apm import ElasticAPM @@ -358,7 +358,7 @@ def _read_from_filename(yml_file_path: Path) -> dict: def load_spec(yml_file_path: Path): """Load and validate spec object given a yml file path.""" spec = _read_from_filename(yml_file_path) - validate(spec) + validate_spec(spec) return Spec.from_dict(spec) diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 0b7555419..4bc5b1832 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -145,3 +145,10 @@ def body(self) -> Optional[str]: if body is None: return None return body.decode("utf-8") + + @property + def mimetype(self) -> str: + return ( + self.request.headers.get("Content-Type") + or "application/octet-stream" + ) From 6ebb80d5c1a054d5a181325f06e294f09f8b192a Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 14 Mar 2024 13:28:27 -0400 Subject: [PATCH 14/22] Fixed lint --- kytos/core/rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 4bc5b1832..1ed1b52b3 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -145,7 +145,7 @@ def body(self) -> Optional[str]: if body is None: return None return body.decode("utf-8") - + @property def mimetype(self) -> str: return ( From 0a8487b73c56684990459e91181a9d4ea8aec888 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 19 Mar 2024 14:51:29 -0400 Subject: [PATCH 15/22] Updated changelog --- CHANGELOG.rst | 8 ++++++++ kytos/core/atcp_server.py | 3 +-- kytos/core/connection.py | 14 +++++--------- kytos/core/controller.py | 2 +- requirements/dev.in | 2 +- tests/conftest.py | 13 ------------- tests/unit/test_core/test_auth.py | 15 +++++++++++++-- tests/unit/test_core/test_connection.py | 14 +++++++------- 8 files changed, 36 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2946ea5d..f9bc46660 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,14 @@ All notable changes to the kytos project will be documented in this file. UNRELEASED - Under development ****************************** +Changed +======= +- Updated python environment installation from 3.9 to 3.11 +- Updated test dependencies + +[2023.2.0] - 2024-02-16 +*********************** + Added ===== - Added ``Interface.tag_ranges`` as ``dict[str, list[list[int]]]`` as replacement for ``vlan_pool`` settings. diff --git a/kytos/core/atcp_server.py b/kytos/core/atcp_server.py index 9aea60373..6fad47027 100644 --- a/kytos/core/atcp_server.py +++ b/kytos/core/atcp_server.py @@ -126,11 +126,10 @@ def connection_made(self, transport: asyncio.Transport): addr, port = transport.get_extra_info('peername') _, server_port = transport.get_extra_info('sockname') - socket = transport.get_extra_info('socket') LOG.info("New connection from %s:%s", addr, port) - self.connection = Connection(addr, port, socket, transport) + self.connection = Connection(addr, port, transport) # This allows someone to inherit from KytosServer and start a server # on another port to handle a different protocol. diff --git a/kytos/core/connection.py b/kytos/core/connection.py index a9f2ecd39..cdf5ece4f 100644 --- a/kytos/core/connection.py +++ b/kytos/core/connection.py @@ -1,9 +1,8 @@ """Module with main classes related to Connections.""" import logging -from asyncio import Transport, trsock +from asyncio import Transport from enum import Enum from errno import EBADF, ENOTCONN -from socket import SHUT_RDWR __all__ = ('Connection', 'ConnectionProtocol', 'ConnectionState') @@ -37,7 +36,6 @@ def __init__( self, address: str, port: int, - socket: trsock.TransportSocket, transport: Transport, switch=None ): @@ -52,7 +50,6 @@ def __init__( """ self.address = address self.port = port - self.socket = socket self.switch = switch self.transport = transport self.state = ConnectionState.NEW @@ -64,7 +61,7 @@ def __str__(self): def __repr__(self): return f"Connection({self.address!r}, {self.port!r}," + \ - f" {self.socket!r}, {self.switch!r}, {self.state!r})" + f" {self.transport!r}, {self.switch!r}, {self.state!r})" @property def state(self): @@ -101,7 +98,7 @@ def send(self, buffer): try: if self.is_alive(): self.transport.write(buffer) - except (OSError, TypeError) as exception: + except OSError as exception: LOG.debug('Could not send packet. Exception: %s', exception) self.close() raise @@ -112,9 +109,8 @@ def close(self): LOG.debug('Shutting down Connection %s', self.id) try: - self.socket.shutdown(SHUT_RDWR) self.transport.close() - self.socket = None + self.transport = None LOG.debug('Connection Closed: %s', self.id) except OSError as exception: if exception.errno not in (ENOTCONN, EBADF): @@ -124,7 +120,7 @@ def close(self): def is_alive(self): """Return True if the connection socket is alive. False otherwise.""" - return self.socket is not None and self.state not in ( + return self.transport is not None and self.state not in ( ConnectionState.FINISHED, ConnectionState.FAILED) def is_new(self): diff --git a/kytos/core/controller.py b/kytos/core/controller.py index 40af9b0fe..9bc889ad0 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -631,7 +631,7 @@ async def msg_out_event_handler(self): message.header.xid, packet.hex()) self.notify_listeners(triggered_event) - except (OSError, TypeError): + except OSError: await self.publish_connection_error(triggered_event) self.log.info("connection closed. Cannot send message") except PackException as err: diff --git a/requirements/dev.in b/requirements/dev.in index 4bdf27572..6e1b2e9cb 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,3 +1,3 @@ --e git+https://github.com/kytos-ng/python-openflow.git@upgrade/python#egg=python-openflow +-e git+https://github.com/kytos-ng/python-openflow.git#egg=python-openflow -e git+https://github.com/kytos-ng/sphinx-theme.git#egg=kytos-sphinx-theme -e .[dev] diff --git a/tests/conftest.py b/tests/conftest.py index f1594759c..c269bd8b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,6 @@ """Unit test fixtures.""" # pylint: disable=redefined-outer-name -from unittest import mock - import pytest from httpx import AsyncClient @@ -54,14 +52,3 @@ def minimal_openapi_spec_dict(): } }, } - - -@pytest.fixture(scope='session', autouse=True) -def mock_hashing(): - """Mock hasinh method from user.py""" - def new_hashing(password: bytes, _hash) -> str: - if password == b"password": - return "some_hash" - return "wrong_hash" - with mock.patch("kytos.core.auth.hashing", wraps=new_hashing) as mock_all: - yield mock_all diff --git a/tests/unit/test_core/test_auth.py b/tests/unit/test_core/test_auth.py index 7cd655365..39b762175 100644 --- a/tests/unit/test_core/test_auth.py +++ b/tests/unit/test_core/test_auth.py @@ -1,6 +1,7 @@ """Test kytos.core.auth module.""" import asyncio import base64 +from unittest import mock import pytest from httpx import AsyncClient @@ -25,12 +26,21 @@ def setup_method(self): "password": "password", } + @staticmethod + def new_hashing(password: bytes, _hash) -> str: + """Use this method to mock auth.hashing.""" + if password == b"password": + return "some_hash" + return "wrong_hash" + async def auth_headers(self, auth: Auth, api_client: AsyncClient) -> dict: """Get Authorization headers with token.""" + # pylint: disable=no-value-for-parameter token = await self._get_token(auth, api_client) return {"Authorization": f"Bearer {token}"} - async def _get_token(self, auth: Auth, api_client: AsyncClient) -> str: + @mock.patch("kytos.core.auth.hashing", wraps=new_hashing) + async def _get_token(self, auth: Auth, api_client: AsyncClient, _) -> str: """Make a request to get a token to be used in tests.""" valid_header = { "Authorization": "Basic " @@ -55,7 +65,8 @@ def _validate_schema(self, my_dict, check_against): return False return True - async def test_01_login_request(self, auth, api_client, monkeypatch): + @mock.patch("kytos.core.auth.hashing", wraps=new_hashing) + async def test_01_login_request(self, _, auth, api_client, monkeypatch): """Test auth login endpoint.""" valid_header = { "Authorization": "Basic " diff --git a/tests/unit/test_core/test_connection.py b/tests/unit/test_core/test_connection.py index aeaa94867..1e53a7b9c 100644 --- a/tests/unit/test_core/test_connection.py +++ b/tests/unit/test_core/test_connection.py @@ -23,10 +23,10 @@ def test__str__(self): def test__repr__(self): """Test __repr__ method.""" - self.connection.socket = 'socket' + self.connection.transport = 'transport' self.connection.switch = 'switch' - expected = "Connection('addr', 123, 'socket', 'switch', " + \ + expected = "Connection('addr', 123, 'transport', 'switch', " + \ ")" assert repr(self.connection) == expected @@ -63,25 +63,25 @@ def test_close(self): """Test close method.""" self.connection.close() - assert self.connection.socket is None + assert self.connection.transport is None assert self.connection is not None def test_close__os_error(self): """Test close method to OSError case.""" - self.connection.socket.shutdown.side_effect = OSError + self.connection.transport.close.side_effect = OSError with pytest.raises(OSError): self.connection.close() - assert self.connection.socket is not None + assert self.connection.transport is not None def test_close__attribute_error(self): """Test close method to AttributeError case.""" - self.connection.socket = None + self.connection.transport = None self.connection.close() - assert self.connection.socket is None + assert self.connection.transport is None def test_is_alive(self): """Test is_alive method to True and False returns.""" From ebc815ca0d4e839907953c61335a127bf926eeb2 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Tue, 19 Mar 2024 16:06:04 -0400 Subject: [PATCH 16/22] Improved API error message --- kytos/core/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kytos/core/helpers.py b/kytos/core/helpers.py index 0bcf202a3..792b31c5b 100644 --- a/kytos/core/helpers.py +++ b/kytos/core/helpers.py @@ -370,9 +370,9 @@ def _request_validation_result_or_400(errors: OpenAPIError) -> None: elif (hasattr(errors.__cause__, "schema_errors") and errors.__cause__.schema_errors): schema_errors = errors.__cause__.schema_errors - error_response += f" {schema_errors[0].message}" - for error in schema_errors[1:]: - error_response += f", {error.message}" + for error in schema_errors: + error_response += f", {error.message} for field" + error_response += f" {'/'.join(map(str,error.path))}." raise HTTPException(400, detail=error_response) From 17e694ab0ef3aefe2e8f91e635a12f005f01219d Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Wed, 20 Mar 2024 13:09:52 -0400 Subject: [PATCH 17/22] Reverted default mimetype request --- kytos/core/rest_api.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 1ed1b52b3..0b7555419 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -145,10 +145,3 @@ def body(self) -> Optional[str]: if body is None: return None return body.decode("utf-8") - - @property - def mimetype(self) -> str: - return ( - self.request.headers.get("Content-Type") - or "application/octet-stream" - ) From 1e3475c47d3ed4192ea948873098d9d69cc8daa8 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 21 Mar 2024 14:35:29 -0400 Subject: [PATCH 18/22] Updated mimetype --- kytos/core/rest_api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kytos/core/rest_api.py b/kytos/core/rest_api.py index 0b7555419..08947737b 100644 --- a/kytos/core/rest_api.py +++ b/kytos/core/rest_api.py @@ -145,3 +145,11 @@ def body(self) -> Optional[str]: if body is None: return None return body.decode("utf-8") + + @property + def mimetype(self) -> str: + content_type = self.request.headers.get("Content-Type") + if content_type: + return content_type.partition(";")[0] + + return "application/octet-stream" From 2cb3d3737b3dbb7fe55fa58c782f37b9e396433e Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 28 Mar 2024 10:11:26 -0400 Subject: [PATCH 19/22] Updated readme --- README.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 52307d2cb..57f8641e6 100644 --- a/README.rst +++ b/README.rst @@ -46,11 +46,11 @@ Installing ========== If you don't have Python 3 installed, please install it. Please make -sure that you're using ``python3.9``: +sure that you're using ``python3.11`` and dependencies: .. code-block:: shell - $ apt install python3.9 + $ apt install python3.11-dev python3.11-venv Then, the first step is to clone *kytos* repository: @@ -64,7 +64,10 @@ install procedure: .. code-block:: shell $ cd kytos - $ sudo python3 setup.py install + $ sudo python3 -m pip install . + +For a complete installation procedure visit our +`development environment setup `_. Configuring =========== From 5f4a7893414fbde19b0d6ccfc42e0c1111878668 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 28 Mar 2024 10:19:33 -0400 Subject: [PATCH 20/22] Updated dev.txt --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 1fb41398e..f9c42a355 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ # via -r requirements/dev.in -e git+https://github.com/kytos-ng/sphinx-theme.git#egg=kytos-sphinx-theme # via -r requirements/dev.in --e git+https://github.com/kytos-ng/python-openflow.git@upgrade/python#egg=python-openflow +-e git+https://github.com/kytos-ng/python-openflow.git#egg=python-openflow # via # -r requirements/dev.in # -r requirements/run.txt From 041a277bf7d8c5c6b12c69c77daa6592e66536f7 Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 28 Mar 2024 10:30:35 -0400 Subject: [PATCH 21/22] Updated readme python installation --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 57f8641e6..1ad2bea45 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ sure that you're using ``python3.11`` and dependencies: .. code-block:: shell - $ apt install python3.11-dev python3.11-venv + $ apt install python3.11 python3.11-dev python3.11-venv Then, the first step is to clone *kytos* repository: From a6b65bab23793c92dd894168836d1670fbf1c2ce Mon Sep 17 00:00:00 2001 From: Aldo Ortega Date: Thu, 28 Mar 2024 10:51:11 -0400 Subject: [PATCH 22/22] Update kytos/core/connection.py Co-authored-by: Vinicius Arcanjo --- kytos/core/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kytos/core/connection.py b/kytos/core/connection.py index cdf5ece4f..79cdf2d7f 100644 --- a/kytos/core/connection.py +++ b/kytos/core/connection.py @@ -44,7 +44,6 @@ def __init__( Args: address (|hw_address|): Source address. port (int): Port number. - socket (TransportSocket): socket. transport (Transport): transport. switch (:class:`~.Switch`): switch with this connection. """