diff --git a/requirements.txt b/requirements.txt index f34747a405..549bbe3975 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies fsspec +paramiko python_slugify diff --git a/setup/storage_backend_sftp/odoo/addons/storage_backend_sftp b/setup/storage_backend_sftp/odoo/addons/storage_backend_sftp new file mode 120000 index 0000000000..53ca923758 --- /dev/null +++ b/setup/storage_backend_sftp/odoo/addons/storage_backend_sftp @@ -0,0 +1 @@ +../../../../storage_backend_sftp \ No newline at end of file diff --git a/setup/storage_backend_sftp/setup.py b/setup/storage_backend_sftp/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/storage_backend_sftp/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) diff --git a/storage_backend_sftp/README.rst b/storage_backend_sftp/README.rst new file mode 100644 index 0000000000..9f082d8cca --- /dev/null +++ b/storage_backend_sftp/README.rst @@ -0,0 +1,44 @@ + +.. image:: https://img.shields.io/badge/licence-LGPL--3-blue.svg + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 + +===================== +Storage backend SFTP +===================== + +Add the possibility to store and get data from an SFTP for your storage backend + + + +Installation +============ + +To install this module, you need to: + +#. (root) pip install paramiko + + +Known issues / Roadmap +====================== + +Update README with the last model of README when migration to v11 in OCA + + +Credits +======= + + +Contributors +------------ + +* Sebastien Beau +* Raphaël Reverdy +* Cédric Pigeon +* Simone Orsi + + +Maintainer +---------- + +* Akretion diff --git a/storage_backend_sftp/__init__.py b/storage_backend_sftp/__init__.py new file mode 100644 index 0000000000..0f00a6730d --- /dev/null +++ b/storage_backend_sftp/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import components diff --git a/storage_backend_sftp/__manifest__.py b/storage_backend_sftp/__manifest__.py new file mode 100644 index 0000000000..e96019e723 --- /dev/null +++ b/storage_backend_sftp/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "Storage Backend SFTP", + "summary": "Implement SFTP Storage", + "version": "16.0.1.0.0", + "category": "Storage", + "website": "https://github.com/OCA/storage", + "author": " Akretion,Odoo Community Association (OCA)", + "license": "LGPL-3", + "installable": True, + "external_dependencies": {"python": ["paramiko"]}, + "depends": ["storage_backend"], + "data": ["views/backend_storage_view.xml"], +} diff --git a/storage_backend_sftp/components/__init__.py b/storage_backend_sftp/components/__init__.py new file mode 100644 index 0000000000..76ddd15267 --- /dev/null +++ b/storage_backend_sftp/components/__init__.py @@ -0,0 +1 @@ +from . import sftp_adapter diff --git a/storage_backend_sftp/components/sftp_adapter.py b/storage_backend_sftp/components/sftp_adapter.py new file mode 100644 index 0000000000..aaa65fe7f9 --- /dev/null +++ b/storage_backend_sftp/components/sftp_adapter.py @@ -0,0 +1,129 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# Copyright 2019 Camptocamp SA (http://www.camptocamp.com). +# Copyright 2020 ACSONE SA/NV () +# @author Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import errno +import logging +import os +from contextlib import contextmanager +from io import StringIO + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + +try: + import paramiko +except ImportError as err: # pragma: no cover + _logger.debug(err) + + +def sftp_mkdirs(client, path, mode=511): + try: + client.mkdir(path, mode) + except IOError as e: + if e.errno == errno.ENOENT and path: + sftp_mkdirs(client, os.path.dirname(path), mode=mode) + client.mkdir(path, mode) + else: + raise # pragma: no cover + + +def load_ssh_key(ssh_key_buffer): + for pkey_class in ( + paramiko.RSAKey, + paramiko.DSSKey, + paramiko.ECDSAKey, + paramiko.Ed25519Key, + ): + try: + return pkey_class.from_private_key(ssh_key_buffer) + except paramiko.SSHException: + ssh_key_buffer.seek(0) # reset the buffer "file" + raise Exception("Invalid ssh private key") + + +@contextmanager +def sftp(backend): + transport = paramiko.Transport((backend.sftp_server, backend.sftp_port)) + if backend.sftp_auth_method == "pwd": + transport.connect(username=backend.sftp_login, password=backend.sftp_password) + elif backend.sftp_auth_method == "ssh_key": + ssh_key_buffer = StringIO(backend.sftp_ssh_private_key) + private_key = load_ssh_key(ssh_key_buffer) + transport.connect(username=backend.sftp_login, pkey=private_key) + client = paramiko.SFTPClient.from_transport(transport) + yield client + transport.close() + + +class SFTPStorageBackendAdapter(Component): + _name = "sftp.adapter" + _inherit = "base.storage.adapter" + _usage = "sftp" + + def add(self, relative_path, data, **kwargs): + with sftp(self.collection) as client: + full_path = self._fullpath(relative_path) + dirname = os.path.dirname(full_path) + if dirname: + try: + client.stat(dirname) + except IOError as e: + if e.errno == errno.ENOENT: + sftp_mkdirs(client, dirname) + else: + raise # pragma: no cover + remote_file = client.open(full_path, "w") + remote_file.write(data) + remote_file.close() + + def get(self, relative_path, **kwargs): + full_path = self._fullpath(relative_path) + with sftp(self.collection) as client: + file_data = client.open(full_path, "r") + data = file_data.read() + # TODO: shouldn't we close the file? + return data + + def list(self, relative_path): + full_path = self._fullpath(relative_path) + with sftp(self.collection) as client: + try: + return client.listdir(full_path) + except IOError as e: + if e.errno == errno.ENOENT: + # The path do not exist return an empty list + return [] + else: + raise # pragma: no cover + + def move_files(self, files, destination_path): + _logger.debug("mv %s %s", files, destination_path) + fp = self._fullpath + with sftp(self.collection) as client: + for sftp_file in files: + dest_file_path = os.path.join( + destination_path, os.path.basename(sftp_file) + ) + # Remove existing file at the destination path (an error is raised + # otherwise) + try: + client.lstat(dest_file_path) + except FileNotFoundError: + _logger.debug("destination %s is free", dest_file_path) + else: + client.unlink(dest_file_path) + # Move the file using absolute filepaths + client.rename(fp(sftp_file), fp(dest_file_path)) + + def delete(self, relative_path): + full_path = self._fullpath(relative_path) + with sftp(self.collection) as client: + return client.remove(full_path) + + def validate_config(self): + with sftp(self.collection) as client: + client.listdir() diff --git a/storage_backend_sftp/i18n/storage_backend_sftp.pot b/storage_backend_sftp/i18n/storage_backend_sftp.pot new file mode 100644 index 0000000000..52092a9b4d --- /dev/null +++ b/storage_backend_sftp/i18n/storage_backend_sftp.pot @@ -0,0 +1,82 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * storage_backend_sftp +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 15.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__backend_type +msgid "Backend Type" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,help:storage_backend_sftp.field_storage_backend__sftp_ssh_private_key +msgid "" +"It's recommended to not store the key here but to provide it via secret env " +"variable. See `server_environment` docs." +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,help:storage_backend_sftp.field_storage_backend__sftp_login +msgid "Login to connect to sftp server" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields.selection,name:storage_backend_sftp.selection__storage_backend__sftp_auth_method__pwd +msgid "Password" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields.selection,name:storage_backend_sftp.selection__storage_backend__sftp_auth_method__ssh_key +msgid "Private key" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields.selection,name:storage_backend_sftp.selection__storage_backend__backend_type__sftp +#: model_terms:ir.ui.view,arch_db:storage_backend_sftp.storage_backend_view_form +msgid "SFTP" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_auth_method +msgid "SFTP Authentification Method" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_server +msgid "SFTP Host" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_login +msgid "SFTP Login" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_password +msgid "SFTP Password" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_port +msgid "SFTP Port" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model.fields,field_description:storage_backend_sftp.field_storage_backend__sftp_ssh_private_key +msgid "SSH private key" +msgstr "" + +#. module: storage_backend_sftp +#: model:ir.model,name:storage_backend_sftp.model_storage_backend +msgid "Storage Backend" +msgstr "" diff --git a/storage_backend_sftp/models/__init__.py b/storage_backend_sftp/models/__init__.py new file mode 100644 index 0000000000..f45f402268 --- /dev/null +++ b/storage_backend_sftp/models/__init__.py @@ -0,0 +1 @@ +from . import storage_backend diff --git a/storage_backend_sftp/models/storage_backend.py b/storage_backend_sftp/models/storage_backend.py new file mode 100644 index 0000000000..f79870fbc6 --- /dev/null +++ b/storage_backend_sftp/models/storage_backend.py @@ -0,0 +1,48 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# Copyright 2019 Camptocamp SA (http://www.camptocamp.com). +# @author Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo import fields, models + + +class StorageBackend(models.Model): + _inherit = "storage.backend" + + backend_type = fields.Selection( + selection_add=[("sftp", "SFTP")], ondelete={"sftp": "set default"} + ) + sftp_server = fields.Char(string="SFTP Host") + sftp_port = fields.Integer(string="SFTP Port", default=22) + sftp_auth_method = fields.Selection( + string="SFTP Authentification Method", + selection=[("pwd", "Password"), ("ssh_key", "Private key")], + default="pwd", + required=True, + ) + sftp_login = fields.Char( + string="SFTP Login", help="Login to connect to sftp server" + ) + sftp_password = fields.Char(string="SFTP Password") + sftp_ssh_private_key = fields.Text( + string="SSH private key", + help="It's recommended to not store the key here " + "but to provide it via secret env variable. " + "See `server_environment` docs.", + ) + + @property + def _server_env_fields(self): + env_fields = super()._server_env_fields + env_fields.update( + { + "sftp_password": {}, + "sftp_login": {}, + "sftp_server": {}, + "sftp_port": {}, + "sftp_auth_method": {}, + "sftp_ssh_private_key": {}, + } + ) + return env_fields diff --git a/storage_backend_sftp/static/description/icon.png b/storage_backend_sftp/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/storage_backend_sftp/static/description/icon.png differ diff --git a/storage_backend_sftp/tests/__init__.py b/storage_backend_sftp/tests/__init__.py new file mode 100644 index 0000000000..c923f0a81e --- /dev/null +++ b/storage_backend_sftp/tests/__init__.py @@ -0,0 +1 @@ +from . import test_sftp diff --git a/storage_backend_sftp/tests/test_sftp.py b/storage_backend_sftp/tests/test_sftp.py new file mode 100644 index 0000000000..26ec9c9df9 --- /dev/null +++ b/storage_backend_sftp/tests/test_sftp.py @@ -0,0 +1,121 @@ +# Copyright 2017 Akretion (http://www.akretion.com). +# @author Sébastien BEAU +# Copyright 2019 Camptocamp (http://www.camptocamp.com). +# @author Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +# pylint: disable=missing-manifest-dependency +# disable warning on 'vcr' missing in manifest: this is only a dependency for +# dev/tests + +import errno +import logging +import os + +import mock + +from odoo.addons.storage_backend.tests.common import BackendStorageTestMixin, CommonCase + +_logger = logging.getLogger(__name__) + +MOD_PATH = "odoo.addons.storage_backend_sftp.components.sftp_adapter" +ADAPTER_PATH = MOD_PATH + ".SFTPStorageBackendAdapter" +PARAMIKO_PATH = MOD_PATH + ".paramiko" + + +class SftpCase(CommonCase, BackendStorageTestMixin): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.backend.write( + { + "backend_type": "sftp", + "sftp_login": "foo", + "sftp_password": "pass", + "sftp_server": os.environ.get("SFTP_HOST", "localhost"), + "sftp_port": os.environ.get("SFTP_PORT", "2222"), + "directory_path": "upload", + } + ) + cls.case_with_subdirectory = "upload/subdirectory/here" + + @mock.patch(MOD_PATH + ".sftp_mkdirs") + @mock.patch(PARAMIKO_PATH) + def test_add(self, mocked_paramiko, mocked_mkdirs): + client = mocked_paramiko.SFTPClient.from_transport() + # simulate errors + exc = IOError() + # general + client.stat.side_effect = exc + with self.assertRaises(IOError): + self.backend.add("fake/path", b"fake data") + # not found + exc.errno = errno.ENOENT + client.stat.side_effect = exc + fakefile = open("/tmp/fakefile.txt", "w+b") + client.open.return_value = fakefile + self.backend.add("fake/path", b"fake data") + # mkdirs has been called + mocked_mkdirs.assert_called() + # file has been written and closed + self.assertTrue(fakefile.closed) + with open("/tmp/fakefile.txt", "r") as thefile: + self.assertEqual(thefile.read(), "fake data") + + @mock.patch(PARAMIKO_PATH) + def test_get(self, mocked_paramiko): + client = mocked_paramiko.SFTPClient.from_transport() + with open("/tmp/fakefile2.txt", "w+b") as fakefile: + fakefile.write(b"filecontent") + client.open.return_value = open("/tmp/fakefile2.txt", "r") + self.assertEqual(self.backend.get("fake/path"), "filecontent") + + @mock.patch(PARAMIKO_PATH) + def test_list(self, mocked_paramiko): + client = mocked_paramiko.SFTPClient.from_transport() + # simulate errors + exc = IOError() + # general + client.listdir.side_effect = exc + with self.assertRaises(IOError): + self.backend.list_files() + # not found + exc.errno = errno.ENOENT + client.listdir.side_effect = exc + self.assertEqual(self.backend.list_files(), []) + + def test_find_files(self): + good_filepaths = ["somepath/file%d.good" % x for x in range(1, 10)] + bad_filepaths = ["somepath/file%d.bad" % x for x in range(1, 10)] + mocked_filepaths = bad_filepaths + good_filepaths + backend = self.backend.sudo() + expected = good_filepaths[:] + expected = [backend.directory_path + "/" + path for path in good_filepaths] + self._test_find_files( + backend, ADAPTER_PATH, mocked_filepaths, r".*\.good$", expected + ) + + @mock.patch(PARAMIKO_PATH) + def test_move_files(self, mocked_paramiko): + client = mocked_paramiko.SFTPClient.from_transport() + # simulate file is not already there + client.lstat.side_effect = FileNotFoundError() + to_move = "move/from/path/myfile.txt" + to_path = "move/to/path" + self.backend.move_files([to_move], to_path) + # no need to delete it + client.unlink.assert_not_called() + # rename gets called + client.rename.assert_called_with( + "upload/" + to_move, "upload/" + to_move.replace("from", "to") + ) + # now try to override destination + client.lstat.side_effect = None + client.lstat.return_value = True + self.backend.move_files([to_move], to_path) + # client will delete it first + client.unlink.assert_called_with(to_move.replace("from", "to")) + # then move it + client.rename.assert_called_with( + "upload/" + to_move, "upload/" + to_move.replace("from", "to") + ) diff --git a/storage_backend_sftp/views/backend_storage_view.xml b/storage_backend_sftp/views/backend_storage_view.xml new file mode 100644 index 0000000000..341631ffda --- /dev/null +++ b/storage_backend_sftp/views/backend_storage_view.xml @@ -0,0 +1,31 @@ + + + + storage.backend + + + + + + + + + + + + + + +