diff --git a/setup/sql_export_delta/odoo/addons/sql_export_delta b/setup/sql_export_delta/odoo/addons/sql_export_delta new file mode 120000 index 0000000000..314bca0f9e --- /dev/null +++ b/setup/sql_export_delta/odoo/addons/sql_export_delta @@ -0,0 +1 @@ +../../../../sql_export_delta \ No newline at end of file diff --git a/setup/sql_export_delta/setup.py b/setup/sql_export_delta/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/sql_export_delta/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) diff --git a/sql_export/models/sql_export.py b/sql_export/models/sql_export.py index d9a8a9395f..34d105dae2 100644 --- a/sql_export/models/sql_export.py +++ b/sql_export/models/sql_export.py @@ -41,6 +41,10 @@ class SqlExport(models.Model): default="utf-8", ) + keep_generated_file = fields.Boolean( + help="Check this to keep generated export files as attachments" + ) + def _compute_use_properties(self): for rec in self: rec.use_properties = bool(rec.query_properties_definition) diff --git a/sql_export/tests/test_sql_query.py b/sql_export/tests/test_sql_query.py index c970a38255..86339e6409 100644 --- a/sql_export/tests/test_sql_query.py +++ b/sql_export/tests/test_sql_query.py @@ -101,3 +101,16 @@ def test_sql_query_with_params(self): wizard.export_sql() export = base64.b64decode(wizard.binary_file) self.assertTrue(export) + + def test_keep_generated_file(self): + """Test that we keep generated files""" + self.sql_report_demo.keep_generated_file = True + wizard = self.wizard_obj.create( + { + "sql_export_id": self.sql_report_demo.id, + } + ) + wizard.export_sql() + attachment = wizard._get_field_attachment() + wizard.sudo().unlink() + self.assertTrue(attachment.exists()) diff --git a/sql_export/views/sql_export_view.xml b/sql_export/views/sql_export_view.xml index f377a6daff..f57386f75e 100644 --- a/sql_export/views/sql_export_view.xml +++ b/sql_export/views/sql_export_view.xml @@ -42,6 +42,11 @@ name="file_format" attrs="{'readonly': [('state', '!=', 'draft')]}" /> + `_ + +**Table of contents** + +.. contents:: + :local: + +Configuration +============= + +To configure this module, you need to: + +1. Go to an export in draft mode +2. Check the flag 'Delta' + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Hunki Enterprises BV + +Contributors +------------ + +- Holger Brunn + (https://hunki-enterprises.com) + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-hbrunn| image:: https://github.com/hbrunn.png?size=40px + :target: https://github.com/hbrunn + :alt: hbrunn + +Current `maintainer `__: + +|maintainer-hbrunn| + +This module is part of the `OCA/reporting-engine `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/sql_export_delta/__init__.py b/sql_export_delta/__init__.py new file mode 100644 index 0000000000..5f67c80400 --- /dev/null +++ b/sql_export_delta/__init__.py @@ -0,0 +1,3 @@ +from . import models +from . import wizards +from .hooks import uninstall_hook diff --git a/sql_export_delta/__manifest__.py b/sql_export_delta/__manifest__.py new file mode 100644 index 0000000000..1382133bce --- /dev/null +++ b/sql_export_delta/__manifest__.py @@ -0,0 +1,26 @@ +# Copyright 2024 Hunki Enterprises BV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl-3.0) + +{ + "name": "SQL Export (delta support)", + "summary": "Support exporting only the changes from last export", + "version": "16.0.1.0.0", + "development_status": "Alpha", + "category": "Extra Tools", + "website": "https://github.com/OCA/reporting-engine", + "author": "Hunki Enterprises BV, Odoo Community Association (OCA)", + "maintainers": ["hbrunn"], + "license": "AGPL-3", + "uninstall_hook": "uninstall_hook", + "external_dependencies": { + "python": [], + "bin": [], + }, + "depends": [ + "sql_export", + ], + "data": [ + "views/sql_export.xml", + ], + "demo": [], +} diff --git a/sql_export_delta/hooks.py b/sql_export_delta/hooks.py new file mode 100644 index 0000000000..9a422d018b --- /dev/null +++ b/sql_export_delta/hooks.py @@ -0,0 +1,9 @@ +# Copyright 2024 Hunki Enterprises BV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl-3.0) +from odoo import SUPERUSER_ID, api + + +def uninstall_hook(cr, registry): + env = api.Environment(cr, SUPERUSER_ID, {}) + for export in env["sql.export"].search([]): + export._export_delta_cleanup(keep_last=False) diff --git a/sql_export_delta/models/__init__.py b/sql_export_delta/models/__init__.py new file mode 100644 index 0000000000..0144620622 --- /dev/null +++ b/sql_export_delta/models/__init__.py @@ -0,0 +1 @@ +from . import sql_export diff --git a/sql_export_delta/models/sql_export.py b/sql_export_delta/models/sql_export.py new file mode 100644 index 0000000000..7aaf986eec --- /dev/null +++ b/sql_export_delta/models/sql_export.py @@ -0,0 +1,98 @@ +# Copyright 2024 Hunki Enterprises BV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl-3.0) + +from psycopg2.sql import SQL, Identifier + +from odoo import fields, models + + +class SqlExport(models.Model): + _inherit = "sql.export" + + export_delta = fields.Boolean( + string="Delta", + help="With this checked, the full result of the query " + "will be stored as table in the database, but the file generated will " + "only contain rows not existing in the n-1st export", + ) + + def write(self, vals): + """Delete previous results when we change the query""" + if "query" in vals: + for this in self: + this._export_delta_cleanup(keep_last=False) + return super().write(vals) + + def _execute_sql_request( + self, + params=None, + mode="fetchall", + rollback=True, + view_name=False, + copy_options="CSV HEADER DELIMITER ';'", + header=False, + ): + delta_id = self.env.context.get("export_delta_id") + + if delta_id: + original_query = self.env.cr.mogrify(self.query, params).decode("utf-8") + result_table = self._export_delta_table_name(delta_id) + table_query = SQL( + "WITH result as ({0}) SELECT * INTO TABLE {1} FROM result" + ).format(SQL(original_query), Identifier(result_table)) + previous_result_table = self._export_delta_existing_tables()[-1:] + if previous_result_table: + result_query = SQL("SELECT * FROM {0} EXCEPT SELECT * FROM {1}").format( + Identifier(result_table), + Identifier(previous_result_table[0]), + ) + else: + result_query = SQL("SELECT * FROM {0}").format(Identifier(result_table)) + self.env.cr.execute(table_query) + # inject new query in cache for super to use + self._cache["query"] = result_query + result = super()._execute_sql_request( + params=None, + mode=mode, + rollback=rollback, + view_name=view_name, + copy_options=copy_options, + header=header, + ) + self.invalidate_recordset(["query"]) + self._export_delta_cleanup(keep_last=True) + else: + result = super()._execute_sql_request( + params=params, + mode=mode, + rollback=rollback, + view_name=view_name, + copy_options=copy_options, + header=header, + ) + + return result + + def _export_delta_table_name(self, identifier): + """ + Return the name of a table to store data for delta export, must end with + {identifier} + """ + return f"sql_export_delta_{self.id}_{identifier}" + + def _export_delta_existing_tables(self): + """Return all table names used for storing data for delta export""" + self.env.cr.execute( + "SELECT table_name FROM information_schema.tables WHERE table_name LIKE %s", + (self._export_delta_table_name("%"),), + ) + return sorted( + [name for name, in self.env.cr.fetchall()], + key=lambda name: int(name[len(self._export_delta_table_name("")) :]), + ) + + def _export_delta_cleanup(self, keep_last=True): + """Delete tables storing data for delta export""" + table_names = self._export_delta_existing_tables()[: -1 if keep_last else None] + for table_name in table_names: + self.env.cr.execute(SQL("DROP TABLE {0}").format(Identifier(table_name))) diff --git a/sql_export_delta/readme/CONFIGURE.md b/sql_export_delta/readme/CONFIGURE.md new file mode 100644 index 0000000000..42bbd0aa47 --- /dev/null +++ b/sql_export_delta/readme/CONFIGURE.md @@ -0,0 +1,4 @@ +To configure this module, you need to: + +1. Go to an export in draft mode +2. Check the flag 'Delta' diff --git a/sql_export_delta/readme/CONTRIBUTORS.md b/sql_export_delta/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..b28199e1f4 --- /dev/null +++ b/sql_export_delta/readme/CONTRIBUTORS.md @@ -0,0 +1 @@ +- Holger Brunn \ (https://hunki-enterprises.com) diff --git a/sql_export_delta/readme/DESCRIPTION.md b/sql_export_delta/readme/DESCRIPTION.md new file mode 100644 index 0000000000..51811e323a --- /dev/null +++ b/sql_export_delta/readme/DESCRIPTION.md @@ -0,0 +1 @@ +This module extends the functionality of ``sql_export`` to allow you to only export changes between export runs. diff --git a/sql_export_delta/static/description/icon.png b/sql_export_delta/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/sql_export_delta/static/description/icon.png differ diff --git a/sql_export_delta/static/description/index.html b/sql_export_delta/static/description/index.html new file mode 100644 index 0000000000..d13d52b607 --- /dev/null +++ b/sql_export_delta/static/description/index.html @@ -0,0 +1,440 @@ + + + + + + +SQL Export (delta support) + + + +
+

SQL Export (delta support)

+ + +

Alpha License: AGPL-3 OCA/reporting-engine Translate me on Weblate Try me on Runboat

+

This module extends the functionality of sql_export to allow you to +only export changes between export runs.

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Configuration

+

To configure this module, you need to:

+
    +
  1. Go to an export in draft mode
    2. +
  2. Check the flag ‘Delta’
    3. +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Hunki Enterprises BV
    • +
+
+ +
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

hbrunn

+

This module is part of the OCA/reporting-engine project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/sql_export_delta/tests/__init__.py b/sql_export_delta/tests/__init__.py new file mode 100644 index 0000000000..b9039bdce0 --- /dev/null +++ b/sql_export_delta/tests/__init__.py @@ -0,0 +1 @@ +from . import test_sql_export_delta diff --git a/sql_export_delta/tests/test_sql_export_delta.py b/sql_export_delta/tests/test_sql_export_delta.py new file mode 100644 index 0000000000..f8e13a82c0 --- /dev/null +++ b/sql_export_delta/tests/test_sql_export_delta.py @@ -0,0 +1,26 @@ +# Copyright 2024 Hunki Enterprises BV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl-3.0) + +from odoo.tests.common import tagged + +from odoo.addons.sql_export.tests import test_sql_query + +from ..hooks import uninstall_hook + + +@tagged("post_install", "-at_install") +class TestSqlExportDelta(test_sql_query.TestExportSqlQuery): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.sql_report_demo.export_delta = True + + def test_sql_query(self): + """Test that exporting demo report twice exports empty delta""" + result = super().test_sql_query() + with self.assertRaises(AssertionError): + super().test_sql_query() + self.assertTrue(self.sql_report_demo._export_delta_existing_tables()) + uninstall_hook(self.env.cr, self.env.registry) + self.assertFalse(self.sql_report_demo._export_delta_existing_tables()) + return result diff --git a/sql_export_delta/views/sql_export.xml b/sql_export_delta/views/sql_export.xml new file mode 100644 index 0000000000..68861338e2 --- /dev/null +++ b/sql_export_delta/views/sql_export.xml @@ -0,0 +1,17 @@ + + + + + sql.export + + + + + + + + diff --git a/sql_export_delta/wizards/__init__.py b/sql_export_delta/wizards/__init__.py new file mode 100644 index 0000000000..f9ef62c395 --- /dev/null +++ b/sql_export_delta/wizards/__init__.py @@ -0,0 +1 @@ +from . import sql_file_wizard diff --git a/sql_export_delta/wizards/sql_file_wizard.py b/sql_export_delta/wizards/sql_file_wizard.py new file mode 100644 index 0000000000..a748ac8f03 --- /dev/null +++ b/sql_export_delta/wizards/sql_file_wizard.py @@ -0,0 +1,14 @@ +# Copyright 2024 Hunki Enterprises BV +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl-3.0) + +from odoo import models + + +class SqlFileWizard(models.TransientModel): + _inherit = "sql.file.wizard" + + def export_sql(self): + self.ensure_one() + if self.sql_export_id.export_delta: + self = self.with_context(export_delta_id=self.id) + return super(SqlFileWizard, self).export_sql()