From c1a50a9ac9d8955af89672b243cdec4c7529c80c Mon Sep 17 00:00:00 2001 From: buhtz Date: Sat, 9 Mar 2024 21:48:05 +0100 Subject: [PATCH] test: User callback log output Related to #1648 and #1269 --- common/config.py | 22 ++- common/pluginmanager.py | 5 + common/plugins/usercallbackplugin.py | 83 +++++---- common/snapshots.py | 7 +- common/test/test_lint.py | 1 + common/test/test_plugin_usercallback.py | 214 ++++++++++++++++++++++-- common/test/test_takeSnapshot.py | 1 + qt/test/test_lint.py | 1 + 8 files changed, 280 insertions(+), 54 deletions(-) diff --git a/common/config.py b/common/config.py index 2530d4386..1d01f52a6 100644 --- a/common/config.py +++ b/common/config.py @@ -140,6 +140,13 @@ class Config(configfile.ConfigFileWithProfiles): PLUGIN_MANAGER = pluginmanager.PluginManager() def __init__(self, config_path=None, data_path=None): + """Back In Time configuration (and much more then this). + + Args: + config_path (str): Full path to the config file + (default: `~/.config/backintime/config`). + data_path (str): It is $XDG_DATA_HOME (default: `~/.local/share`). + """ # Note: The main profiles name here is translated using the systems # current locale because the language code in the config file wasn't # read yet. @@ -174,13 +181,16 @@ def __init__(self, config_path=None, data_path=None): else: self._LOCAL_CONFIG_PATH = os.path.abspath(config_path) self._LOCAL_CONFIG_FOLDER = os.path.dirname(self._LOCAL_CONFIG_PATH) - old_path = os.path.join(self._LOCAL_CONFIG_FOLDER, 'config2') - if os.path.exists(old_path): - if os.path.exists(self._LOCAL_CONFIG_PATH): - os.remove(old_path) - else: - os.rename(old_path, self._LOCAL_CONFIG_PATH) + # (buhtz) Introduced in 2009 via commit 5b26575be4. + # Ready to remove after 15 years. + # old_path = os.path.join(self._LOCAL_CONFIG_FOLDER, 'config2') + + # if os.path.exists(old_path): + # if os.path.exists(self._LOCAL_CONFIG_PATH): + # os.remove(old_path) + # else: + # os.rename(old_path, self._LOCAL_CONFIG_PATH) # Load global config file self.load(self._GLOBAL_CONFIG_PATH) diff --git a/common/pluginmanager.py b/common/pluginmanager.py index 7e33f99a2..8fd8d13ca 100644 --- a/common/pluginmanager.py +++ b/common/pluginmanager.py @@ -238,15 +238,20 @@ def load(self, snapshots = None, cfg = None, force = False): self.hasGuiPlugins = False loadedPlugins = [] + # TODO 09/28/2022: Move hard coded plugin folders to configuration for path in ('plugins', 'common/plugins', 'qt/plugins'): fullPath = tools.backintimePath(path) + if os.path.isdir(fullPath): logger.debug('Register plugin path %s' %fullPath, self) tools.registerBackintimePath(path) + for f in os.listdir(fullPath): + if f not in loadedPlugins and f.endswith('.py') and not f.startswith('__'): logger.debug('Probing plugin %s' % f, self) + try: module = __import__(f[: -3]) module_dict = module.__dict__ diff --git a/common/plugins/usercallbackplugin.py b/common/plugins/usercallbackplugin.py index 72a8c92dd..edb8ab1fd 100644 --- a/common/plugins/usercallbackplugin.py +++ b/common/plugins/usercallbackplugin.py @@ -1,20 +1,20 @@ -# Back In Time -# Copyright (C) 2008-2022 Oprea Dan, Bart de Koning, Richard Bailey, Germar Reitze +# Back In Time +# Copyright (C) 2008-2022 Oprea Dan, Bart de Koning, Richard Bailey, +# Germar Reitze # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import pluginmanager @@ -23,7 +23,7 @@ from subprocess import Popen, PIPE from exceptions import StopException -_=gettext.gettext +_ = gettext.gettext class UserCallbackPlugin(pluginmanager.Plugin): @@ -33,14 +33,11 @@ class UserCallbackPlugin(pluginmanager.Plugin): files) about different steps ("events") in the backup process via the :py:class:`pluginmanager.PluginManager`. - This plugin calls a user-defined script file ("user-callback") - that is located in this folder: - - $XDG_CONFIG_HOME/backintime/user-callback - (by default $XDG_CONFIG_HOME is ~/.config) + This plugin calls a user-defined script file ("user-callback"). By default + that file is located in the config folder `$XDG_CONFIG_HOME/backintime/` + or another folder if command line optioni `--config` is used. - The user-callback script is called with up to five - positional arguments: + The user-callback script is called with up to five positional arguments: 1. The profile ID from the config file (1=Main Profile, ...) 2. The profile name (from the config file) @@ -67,37 +64,53 @@ def __init__(self): def init(self, snapshots): self.config = snapshots.config self.script = self.config.takeSnapshotUserCallback() - if not os.path.exists(self.script): - return False - return True - # TODO 09/28/2022: This method should be private (__callback) + return os.path.exists(self.script) + + # TODO 09/28/2022: This method should be private (_callback) def callback(self, *args, profileID = None): if profileID is None: profileID = self.config.currentProfile() + profileName = self.config.profileName(profileID) cmd = [self.script, profileID, profileName] - cmd.extend([str(x) for x in args]) - logger.debug('Call user-callback: %s' %' '.join(cmd), self) + cmd.extend(str(x) for x in args) + + logger.debug(f'Call user-callback: {" ".join(cmd)}', self) + if self.config.userCallbackNoLogging(): stdout, stderr = None, None else: stdout, stderr = PIPE, PIPE + try: callback = Popen(cmd, - stdout = stdout, - stderr = stderr, - universal_newlines = True) + stdout=stdout, + stderr=stderr, + universal_newlines=True) output = callback.communicate() + + # Stdout if output[0]: - logger.info('user-callback returned \'%s\'' %output[0].strip('\n'), self) + logger.info("user-callback returned '" + + output[0].strip('\n') + "'", + self) + + # Stderr if output[1]: - logger.error('user-callback returned \'%s\'' %output[1].strip('\n'), self) + logger.error("user-callback returned '" + + output[1].strip('\n') + "'", + self) + if callback.returncode != 0: - logger.warning('user-callback returncode: %s' %callback.returncode, self) + logger.warning( + f'user-callback returncode: {callback.returncode}', self) raise StopException() + except OSError as e: - logger.error("Exception when trying to run user callback: %s" % e.strerror, self) + logger.error( + f'Exception when trying to run user callback: {e.strerror}', + self) def processBegin(self): self.callback('1') diff --git a/common/snapshots.py b/common/snapshots.py index 9709db095..ab6a90644 100644 --- a/common/snapshots.py +++ b/common/snapshots.py @@ -48,10 +48,14 @@ class Snapshots: """ Collection of take-snapshot and restore commands. - BUHTZ 2022-10-09: In my understanding this the representation of a + BUHTZ 2022-10-09: In my understanding this is the representation of a snapshot in the "application layer". This seems to be the difference to the class `SID` which represents a snapshot in the "data layer". + BUHTZ 2024-02-23: Not sure but it seems to be one concret snapshot and + not a collection of snapshots. In this case the class name is missleading + because it is in plural form. + Args: cfg (config.Config): current config """ @@ -1370,6 +1374,7 @@ def takeSnapshot(self, sid, now, include_folders): # TODO # Process return value with rsync exit code to recognize errors that # cannot be recognized by parsing the rsync output currently + rsync_exit_code = proc.run() # Fix for #1491 and #489 # Note that the return value (containing the exit code) of the diff --git a/common/test/test_lint.py b/common/test/test_lint.py index 982ea249d..5213da0d7 100644 --- a/common/test/test_lint.py +++ b/common/test/test_lint.py @@ -77,6 +77,7 @@ def test_with_pylint(self): 'E1101', # no-member 'W1401', # anomalous-backslash-in-string (invalid escape sequence) 'E0401', # import-error + 'I0021', # useless-suppression ] cmd.append('--enable=' + ','.join(err_codes)) diff --git a/common/test/test_plugin_usercallback.py b/common/test/test_plugin_usercallback.py index 5a3b2f6e2..ce2c6a1cf 100644 --- a/common/test/test_plugin_usercallback.py +++ b/common/test/test_plugin_usercallback.py @@ -1,19 +1,23 @@ import sys -import pathlib +import inspect +import tempfile +from pathlib import Path +import stat +import io +from datetime import datetime import unittest import unittest.mock as mock import json +from contextlib import redirect_stdout, redirect_stderr # This workaround will become obsolet when migrating to src-layout -sys.path.append(str(pathlib.Path(__file__).parent)) -sys.path.append(str(pathlib.Path(__file__).parent / 'plugins')) -import config +sys.path.append(str(Path(__file__).parent)) +sys.path.append(str(Path(__file__).parent / 'plugins')) +import logger +from config import Config +from snapshots import Snapshots, SID from usercallbackplugin import UserCallbackPlugin -# TODO -# Create a surrogate instance of config.Config -# I assume it is well usable for some other tests, too. -# But it will be always a workaround until the code is cleaner. class UserCallback(unittest.TestCase): """Simple test related to to UserCallbackPlugin class. @@ -31,7 +35,7 @@ class UserCallback(unittest.TestCase): """ def _generic_called_with(self, the_step, reason, *args): sut = UserCallbackPlugin() - sut.config = config.Config() + sut.config = Config() sut.script = '' mock_name = 'usercallbackplugin.UserCallbackPlugin.callback' @@ -51,7 +55,7 @@ def test_reason_processnewSnapshot(self): def test_reason_error(self): sut = UserCallbackPlugin() - sut.config = config.Config() + sut.config = Config() sut.script = '' mock_name = 'usercallbackplugin.UserCallbackPlugin.callback' @@ -76,7 +80,7 @@ def test_reason_appExit(self): def test_reason_mount(self): sut = UserCallbackPlugin() - sut.config = config.Config() + sut.config = Config() sut.script = '' mock_name = 'usercallbackplugin.UserCallbackPlugin.callback' @@ -95,7 +99,7 @@ def test_reason_mount(self): def test_reason_unmount(self): sut = UserCallbackPlugin() - sut.config = config.Config() + sut.config = Config() sut.script = '' mock_name = 'usercallbackplugin.UserCallbackPlugin.callback' @@ -111,3 +115,189 @@ def test_reason_unmount(self): sut.unmount('987') func_callback.assert_called_once() func_callback.assert_called_with('8', profileID='987') + + +class SystemTest(unittest.TestCase): + """Full backup run and parsing the log output for the expected + user-callback returns in correct order. + + Create and use your own config file and take it over via `--config` + option. Place your own user-callback script in the same folder as + this config file. + """ + + @classmethod + def _create_user_callback_file(cls, parent_path): + content = inspect.cleandoc(''' + #!/usr/bin/env python3 + import sys + response = sys.argv[1:] + print(response) + ''') + + callback_fp = parent_path / 'user-callback' + callback_fp.write_text(content, 'utf-8') + callback_fp.chmod(stat.S_IRWXU) + + # Name of folder with files to backup. + NAME_SOURCE = 'src' + # Name of folder where snapshots (backups) are stored in. + NAME_DESTINATION = 'dest' + + @classmethod + def _extract_callback_responses(cls, output): + """Extract response of user-callback script out of log output. + + See https://github.com/bit-team/user-callback for documentation about + user-callback and the response codes. + + Example :: + # Raw output + INFO: user-callback returned '['1', 'Main profile', '2']' + INFO: Something else + INFO: user-callback returned '['1', 'Main profile', '8']' + + # Result in a two entry list + [ + ['1', 'Main profile', '2'] + ['1', 'Main profile', '8'] + ] + + Returns: + A list of response values as lists. First entry is profile, second + is profile id, third is reason code. If available further entries + could be contained. + """ + + if isinstance(output, str): + output = output.splitlines() + + # only log lines related to user-callback + response_lines = filter( + lambda line: 'user-callback returned' in line, output) + + callback_responses = [] + + for line in response_lines: + callback_responses.append( + eval(line[line.index("'")+1:line.rindex("'")]) + ) + + # Workaround: Cast profile-id and response-code to integers + for idx in range(len(callback_responses)): + callback_responses[idx][0] = int(callback_responses[idx][0]) + callback_responses[idx][2] = int(callback_responses[idx][2]) + + return callback_responses + + @classmethod + def _create_source_and_destination_folders(cls, parent_path): + # Folder to backup + src_path = parent_path / cls.NAME_SOURCE + src_path.mkdir() + + # Files and folders as backup content + (src_path / 'one').write_bytes(b'0123') + (src_path / 'subfolder').mkdir() + (src_path / 'subfolder' / 'two').write_bytes(b'4567') + + # Folder to store backup + dest_path = parent_path / cls.NAME_DESTINATION + dest_path.mkdir() + + @classmethod + def _create_config_file(cls, parent_path): + cfg_content = inspect.cleandoc(''' + config.version=6 + profile1.snapshots.include.1.type=0 + profile1.snapshots.include.1.value={rootpath}/{source} + profile1.snapshots.include.size=1 + profile1.snapshots.no_on_battery=false + profile1.snapshots.notify.enabled=true + profile1.snapshots.path={rootpath}/{destination} + profile1.snapshots.path.host=test-host + profile1.snapshots.path.profile=1 + profile1.snapshots.path.user=test-user + profile1.snapshots.preserve_acl=false + profile1.snapshots.preserve_xattr=false + profile1.snapshots.remove_old_snapshots.enabled=true + profile1.snapshots.remove_old_snapshots.unit=80 + profile1.snapshots.remove_old_snapshots.value=10 + profile1.snapshots.rsync_options.enabled=false + profile1.snapshots.rsync_options.value= + profiles.version=1 + ''') + + cfg_content = cfg_content.format( + rootpath=parent_path, + source=cls.NAME_SOURCE, + destination=cls.NAME_DESTINATION + ) + + # config file location + config_fp = parent_path / 'config_path' / 'config' + config_fp.parent.mkdir() + config_fp.write_text(cfg_content, 'utf-8') + + return config_fp + + def setUp(self): + """Setup a local snapshot profile including a user-callback""" + # cleanup() happens automatically + self._temp_dir = tempfile.TemporaryDirectory(prefix='bit.') + # Workaround: tempfile and pathlib not compatible yet + self.temp_path = Path(self._temp_dir.name) + + self._create_source_and_destination_folders(self.temp_path) + self.config_fp = self._create_config_file(self.temp_path) + self._create_user_callback_file(self.config_fp.parent) + + def test_local_snapshot(self): + """User-callback response while doing a local snapshot""" + + config = Config( + config_path=str(self.config_fp), + data_path=str(self.temp_path / '.local' / 'share') + ) + + full_snapshot_path = config.snapshotsFullPath() + Path(full_snapshot_path).mkdir(parents=True) + + snapshot = Snapshots(config) + + # DevNote : Because BIT don't use Python's logging module there is + # no way to use assertLogs(). Current solution is to capture + # stdout/stderr. + stdout = io.StringIO() + stderr = io.StringIO() + + with redirect_stdout(stdout), redirect_stderr(stderr): + # Result is inverted. 'True' means there was an error. + self.assertFalse(snapshot.backup()) + + # Empty STDOUT output + self.assertFalse(stdout.getvalue()) + + responses = self._extract_callback_responses(stderr.getvalue()) + + # Number of responses + self.assertEqual(5, len(responses)) + + # Test Name and ID + self.assertEqual( + {(1, 'Main profile')}, + # de-duplicate (using set() )by first two elements in each entry + {(entry[0], entry[1]) for entry in responses} + ) + + # Order of response codes + self.assertEqual( + [ + 7, # Mount + 1, # Backup begins + 3, # New snapshot was taken + 2, # Backup ends + 8, # Unmount + ], + [entry[2] for entry in responses] + ) diff --git a/common/test/test_takeSnapshot.py b/common/test/test_takeSnapshot.py index e2f9a6540..2bb5862ab 100644 --- a/common/test/test_takeSnapshot.py +++ b/common/test/test_takeSnapshot.py @@ -28,6 +28,7 @@ import snapshots import mount + class TestTakeSnapshot(generic.SnapshotsTestCase): def setUp(self): super(TestTakeSnapshot, self).setUp() diff --git a/qt/test/test_lint.py b/qt/test/test_lint.py index f8b1c684e..781f4a7f2 100644 --- a/qt/test/test_lint.py +++ b/qt/test/test_lint.py @@ -77,6 +77,7 @@ def test_with_pylint(self): 'E1101', # no-member 'W1401', # anomalous-backslash-in-string (invalid escape sequence) 'E0401', # import-error + 'I0021', # useless-suppression ] cmd.append('--enable=' + ','.join(err_codes))