From a1ff4a58688ac4eedaf1338719c32de5ffa92f97 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 16:58:56 +0100 Subject: [PATCH 01/15] doc: Fix dead link. --- doc/server/plugins/probes/ohai.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/server/plugins/probes/ohai.txt b/doc/server/plugins/probes/ohai.txt index 02ac8c694e..a771479867 100644 --- a/doc/server/plugins/probes/ohai.txt +++ b/doc/server/plugins/probes/ohai.txt @@ -5,8 +5,7 @@ Ohai ==== -.. _Ohai: http://wiki.opscode.com/display/chef/Ohai -.. _Ohai-Install: http://wiki.opscode.com/display/chef/Ohai+Installation+and+Use +.. _Ohai: https://docs.chef.io/ohai.html The `Ohai`_ plugin is used to detect information about the client operating system. The data is reported back to the server using JSON. @@ -14,8 +13,7 @@ operating system. The data is reported back to the server using JSON. Client prerequisites -------------------- -On the client, you need to install `Ohai`_. See `Ohai-Install`_ for more -information. +On the client, you need to install `Ohai`_. Server prerequisites -------------------- From 1cc8c961c057ba718e338a5808ba73caad406fa9 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:19:06 +0100 Subject: [PATCH 02/15] Server: Bump required python version to 2.6. --- doc/development/core.txt | 11 +++--- doc/installation/prerequisites.txt | 44 ++++++++++----------- doc/reports/dynamic.txt | 1 - doc/server/configuration.txt | 4 +- doc/server/plugins/probes/ohai.txt | 9 ----- osx/Makefile | 2 - setup.py | 4 -- src/lib/Bcfg2/Server/MultiprocessingCore.py | 2 +- src/lib/Bcfg2/Server/SSLServer.py | 10 +---- src/lib/Bcfg2/Server/Test.py | 18 +-------- 10 files changed, 33 insertions(+), 72 deletions(-) diff --git a/doc/development/core.txt b/doc/development/core.txt index f5cc7de67a..ca7a247328 100644 --- a/doc/development/core.txt +++ b/doc/development/core.txt @@ -12,12 +12,11 @@ Bcfg2 1.3 added a pluggable server core system so that the server core itself can be easily swapped out to use different technologies. It currently ships with several backends: a builtin core written from scratch using the various server tools in the Python standard library; -a variant on the builtin core that uses Python 2.6's -:mod:`multiprocessing` library to process requests in parallel; and an -experimental `CherryPy `_ based core. This -page documents the server core interface so that other cores can be -written to take advantage of other technologies, e.g., `Tornado -`_ or `Twisted +a variant on the builtin core that uses the Python :mod:`multiprocessing` +library to process requests in parallel; and an experimental `CherryPy +`_ based core. This page documents the server core +interface so that other cores can be written to take advantage of other +technologies, e.g., `Tornado `_ or `Twisted `_. A core implementation needs to: diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index e6de2a9f6f..7548062845 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -41,28 +41,28 @@ Bcfg2 Client Bcfg2 Server ------------ -+-------------------------------+----------+--------------------------------+ -| Software | Version | Requires | -+===============================+==========+================================+ -| libxml2 | 2.6.24+ | | -+-------------------------------+----------+--------------------------------+ -| libxslt | Any | libxml2 | -+-------------------------------+----------+--------------------------------+ -| python | 2.2-2.7 | | -+-------------------------------+----------+--------------------------------+ -| lxml | 0.9+ | lxml: libxml2, libxslt, python | -+-------------------------------+----------+--------------------------------+ -| gamin or inotify | Any | | -+-------------------------------+----------+--------------------------------+ -| python-gamin or pyinotify | Any | gamin or inotify, python | -+-------------------------------+----------+--------------------------------+ -| python-ssl (this is included | Any | | -| in Python2.6 and later) | | | -+-------------------------------+----------+--------------------------------+ -| python-setuptools | Any | | -+-------------------------------+----------+--------------------------------+ -| python-genshi | Any | | -+-------------------------------+----------+--------------------------------+ ++-------------------------------+-----------------+--------------------------------+ +| Software | Version | Requires | ++===============================+=================+================================+ +| libxml2 | 2.6.24+ | | ++-------------------------------+-----------------+--------------------------------+ +| libxslt | Any | libxml2 | ++-------------------------------+-----------------+--------------------------------+ +| python | 2.6 and greater | | ++-------------------------------+-----------------+--------------------------------+ +| lxml | 0.9+ | lxml: libxml2, libxslt, python | ++-------------------------------+-----------------+--------------------------------+ +| gamin or inotify | Any | | ++-------------------------------+-----------------+--------------------------------+ +| python-gamin or pyinotify | Any | gamin or inotify, python | ++-------------------------------+-----------------+--------------------------------+ +| python-ssl (this is included | Any | | +| in Python2.6 and later) | | | ++-------------------------------+-----------------+--------------------------------+ +| python-setuptools | Any | | ++-------------------------------+-----------------+--------------------------------+ +| python-genshi | Any | | ++-------------------------------+-----------------+--------------------------------+ Bcfg2 Reporting --------------- diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 38d4c7e3aa..d71d4f873a 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -39,7 +39,6 @@ Prerequisites ------------- * sqlite3 -* pysqlite2 (if using python 2.4) * `Django `_ >= 1.3 * mod-wsgi diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt index 79d732f6d5..f4345bb0d3 100644 --- a/doc/server/configuration.txt +++ b/doc/server/configuration.txt @@ -180,7 +180,6 @@ The builtin server: * Is very stable and mature; * Supports certificate authentication; -* Works on Python 2.4; * Is slow with larger numbers of clients. The multiprocessing server: @@ -190,7 +189,6 @@ The multiprocessing server: * Introduces concurrent processing to Bcfg2, which may break in various edge cases; * Supports certificate authentication; -* Requires Python 2.6; * Is faster with large numbers of concurrent runs. The CherryPy server: @@ -198,7 +196,7 @@ The CherryPy server: * Is very new and potentially buggy; * Does not support certificate authentication yet, only password authentication; -* Requires CherryPy 3.3, which requires Python 2.5; +* Requires CherryPy 3.3; * Is smarter about daemonization, particularly if you are :ref:`server-dropping-privs`; * Is faster with large numbers of clients. diff --git a/doc/server/plugins/probes/ohai.txt b/doc/server/plugins/probes/ohai.txt index a771479867..6d6ca86b60 100644 --- a/doc/server/plugins/probes/ohai.txt +++ b/doc/server/plugins/probes/ohai.txt @@ -15,15 +15,6 @@ Client prerequisites On the client, you need to install `Ohai`_. -Server prerequisites --------------------- - -If you have python 2.6 or later installed, you can continue on to -:ref:`ohai-setup`. Otherwise, you will need to install the -python-simplejson module found packaged in most distributions. - -.. _ohai-setup: - Setup ----- diff --git a/osx/Makefile b/osx/Makefile index e2fcf0c195..84b84d0ae0 100644 --- a/osx/Makefile +++ b/osx/Makefile @@ -71,8 +71,6 @@ prepare: install chmod 0770 "${PKGTMP}/scripts/${PREFLIGHT}" chmod 0770 "${PKGTMP}/scripts/${POSTFLIGHT}" - # add in M2Crypto if python version is less than 2.6 - if [[ ${PYMAJORVERSION} == 2 ]]; then if [[ ${PYMINORVERSION} -lt 6 ]]; then cp "M2Crypto-0.20.2-py2.5-macosx-10.5-ppc.egg" "${PKGROOT}/${SITELIBDIR}" && cp "easy-install.pth" "${PKGROOT}/${SITELIBDIR}"; fi ; fi # add default bcfg2.conf mkdir -p "${PKGROOT}/etc" cp "${CONF}" "${PKGROOT}/etc/${CONF}" diff --git a/setup.py b/setup.py index 7a2dc7efd7..aab1d5bbaf 100755 --- a/setup.py +++ b/setup.py @@ -19,10 +19,6 @@ 'argparse' ] -# Use the backported ssl module on < python2.6 -if sys.version_info[:2] < (2, 6): - inst_reqs.append('ssl') - setup(name="Bcfg2", version=__version__, # Defined in src/lib/Bcfg2/version.py description="Bcfg2 Server", diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 4bf3e4a273..6049d59378 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -1,7 +1,7 @@ """ The multiprocessing server core is a reimplementation of the :mod:`Bcfg2.Server.BuiltinCore` that uses the Python :mod:`multiprocessing` library to offload work to multiple child -processes. As such, it requires Python 2.6+. +processes. The parent communicates with the children over :class:`multiprocessing.Queue` objects via a diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py index 1f8febd0ed..16bf1aefe3 100644 --- a/src/lib/Bcfg2/Server/SSLServer.py +++ b/src/lib/Bcfg2/Server/SSLServer.py @@ -22,14 +22,8 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): """ An XML-RPC dispatcher. """ def __init__(self, allow_none, encoding): - try: - SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self, - allow_none, - encoding) - except: - # Python 2.4? - SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self) - + SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__( + self, allow_none, encoding) self.logger = logging.getLogger(self.__class__.__name__) self.allow_none = allow_none self.encoding = encoding diff --git a/src/lib/Bcfg2/Server/Test.py b/src/lib/Bcfg2/Server/Test.py index 9a1d43d12d..98c511960f 100644 --- a/src/lib/Bcfg2/Server/Test.py +++ b/src/lib/Bcfg2/Server/Test.py @@ -12,16 +12,7 @@ from nose.core import TestProgram from nose.suite import LazySuite from unittest import TestCase - -try: - from multiprocessing import Process, Queue, active_children - HAS_MULTIPROC = True -except ImportError: - HAS_MULTIPROC = False - - def active_children(): - """active_children() when multiprocessing lib is missing.""" - return [] +from multiprocessing import Process, Queue, active_children def get_sigint_handler(core): @@ -184,7 +175,7 @@ class CLI(object): help='Ignore these entries if they fail to build'), Bcfg2.Options.Option( "--children", cf=('bcfg2_test', 'children'), default=0, type=int, - help='Spawn this number of children for bcfg2-test (python 2.6+)')] + help='Spawn this number of children for bcfg2-test')] def __init__(self): parser = Bcfg2.Options.get_parser( @@ -193,11 +184,6 @@ def __init__(self): parser.parse() self.logger = logging.getLogger(parser.prog) - if Bcfg2.Options.setup.children and not HAS_MULTIPROC: - self.logger.warning("Python multiprocessing library not found, " - "running with no children") - Bcfg2.Options.setup.children = 0 - def get_core(self): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.Core() From 04f438e4557947c1addec9bec3111e481cf351b1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:27:59 +0100 Subject: [PATCH 03/15] Remove nested try/except/finally. We now require Python 2.6 on the server and can use PEP 341 (Unified try/except/finally). --- src/lib/Bcfg2/Reporting/Collector.py | 21 ++++++++++---------- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 9 ++++----- src/lib/Bcfg2/Server/BuiltinCore.py | 13 ++++++------ 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index f05a257328..056f7872ca 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -43,17 +43,16 @@ def __init__(self, interaction, storage, group=None, target=None, def run(self): """Call the database storage procedure (aka import)""" try: - try: - start = time.time() - self.storage.import_interaction(self.interaction) - self.logger.info("Imported interaction for %s in %ss" % - (self.interaction.get('hostname', - ''), - time.time() - start)) - except: - #TODO requeue? - self.logger.error("Unhandled exception in import thread %s" % - sys.exc_info()[1]) + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', + ''), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + sys.exc_info()[1]) finally: if self.semaphore: self.semaphore.release() diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 6fe69c0fa2..5c3f70147f 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -400,11 +400,10 @@ def _import_interaction(self, interaction): def import_interaction(self, interaction): """Import the data into the backend""" try: - try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index dc5cc46fba..be8fa85bdc 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -86,14 +86,9 @@ def _dispatch(self, method, args, dispatch_dict): raise xmlrpclib.Fault(xmlrpclib.METHOD_NOT_FOUND, "Unknown method %s" % method) + method_start = time.time() try: - method_start = time.time() - try: - return method_func(*args) - finally: - Bcfg2.Server.Statistics.stats.add_value( - method, - time.time() - method_start) + return method_func(*args) except xmlrpclib.Fault: raise except Exception: @@ -101,6 +96,10 @@ def _dispatch(self, method, args, dispatch_dict): if getattr(err, "log", True): self.logger.error(err, exc_info=True) raise xmlrpclib.Fault(getattr(err, "fault_code", 1), str(err)) + finally: + Bcfg2.Server.Statistics.stats.add_value( + method, + time.time() - method_start) def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID From f05a2845530b33734e0ba40c33758bdec91f63f8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:37:26 +0100 Subject: [PATCH 04/15] Compat: Remove MutableMapping. --- doc/development/compat.txt | 15 --------------- src/lib/Bcfg2/Compat.py | 5 ----- src/lib/Bcfg2/Server/Cache.py | 2 +- src/lib/Bcfg2/Server/Plugin/helpers.py | 4 +++- src/lib/Bcfg2/Server/Plugins/Metadata.py | 11 ++++++----- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index a6a85cbab3..04e33312d6 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -17,7 +17,6 @@ tool), this documentation is *not* automatically generated. There are several basic types of symbols found in ``Bcfg2.Compat``: -* Symbols whose names have changed, e.g., ``MutableMapping``. * Symbols whose modules have changed names, e.g., ``urljoin``/``urlparse`` * Symbols that have been added or removed, e.g., ``any``/``all``, ``reduce``, ``walk_packages`` @@ -127,8 +126,6 @@ below, since some of these implementations may be feature-incomplete. +----------------+--------------------------------+--------------------------------------------+ | md5 | :func:`md5.md5` | :class:`hashlib.md5` | +----------------+--------------------------------+--------------------------------------------+ -| MutableMapping | :class:`UserDict.DictMixin` | :class:`collections.MutableMapping` (2.6+) | -+----------------+--------------------------------+--------------------------------------------+ | literal_eval | :func:`eval` | :func:`ast.literal_eval` (2.6+) | +----------------+--------------------------------+--------------------------------------------+ @@ -161,18 +158,6 @@ is in use, since they are not equivalent. For the majority of simple cases -- finding the MD5 sum of a string -- they are equivalent enough. -MutableMapping -~~~~~~~~~~~~~~ - -:class:`collections.MutableMapping` provides a subset of the -functionality of :class:`UserDict.DictMixin`; that is, any object that -is written to work with ``MutableMapping`` will also work with -``DictMixin``, so you should write classes with ``MutableMapping`` in -mind. - -:class:`collections.MutableMapping` is available in Python 2.6+, and -will be used if available. - literal_eval ~~~~~~~~~~~~ diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 1c2420ccf9..b053ab2aab 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -123,11 +123,6 @@ def b64decode(val, **kwargs): # pylint: disable=C0111 except NameError: from functools import reduce -try: - from collections import MutableMapping -except ImportError: - from UserDict import DictMixin as MutableMapping - class CmpMixin(object): """ In Py3K, :meth:`object.__cmp__` is no longer magical, so this diff --git a/src/lib/Bcfg2/Server/Cache.py b/src/lib/Bcfg2/Server/Cache.py index b3b906b2c2..4ee79ec7c0 100644 --- a/src/lib/Bcfg2/Server/Cache.py +++ b/src/lib/Bcfg2/Server/Cache.py @@ -64,7 +64,7 @@ """ -from Bcfg2.Compat import MutableMapping +from collections import MutableMapping class _Cache(MutableMapping): diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 762d018ebc..ed524600ad 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -9,11 +9,13 @@ import genshi import operator import lxml.etree +from collections import MutableMapping + import Bcfg2.Server import Bcfg2.Options import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable -from Bcfg2.Compat import CmpMixin, MutableMapping, wraps +from Bcfg2.Compat import CmpMixin, wraps from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b912d3725e..729a3fa126 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -10,6 +10,8 @@ import socket import logging import lxml.etree +from collections import MutableMapping + import Bcfg2.Server import Bcfg2.Options import Bcfg2.Server.Plugin @@ -17,7 +19,7 @@ from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache # pylint: disable=W0622 -from Bcfg2.Compat import MutableMapping, all, any, wraps +from Bcfg2.Compat import all, any, wraps # pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo @@ -71,10 +73,9 @@ def __setitem__(self, key, value): @Bcfg2.Server.Plugin.DatabaseBacked.get_db_lock def __delitem__(self, key): - # UserDict didn't require __delitem__, but MutableMapping - # does. we don't want deleting a client version record to - # delete the client, so we just set the version to None, - # which is kinda like deleting it, but not really. + # We don't want deleting a client version record to delete the + # client, so we just set the version to None which is kinda like + # deleting it, but not really. try: client = MetadataClientModel.objects.get(hostname=key) except MetadataClientModel.DoesNotExist: From a899b1d47137db845d280ea32cf28942a29929ad Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:42:06 +0100 Subject: [PATCH 05/15] Compat: Remove formatdate. --- doc/development/compat.txt | 2 -- src/lib/Bcfg2/Compat.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index 04e33312d6..439c1a8316 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -114,8 +114,6 @@ below, since some of these implementations may be feature-incomplete. +----------------+--------------------------------+--------------------------------------------+ | Name | Python 2.4 | Python 2.5+ | +================+================================+============================================+ -| formatdate | :func:`email.Utils.formatdate` | :func:`email.utils.formatdate` | -+----------------+--------------------------------+--------------------------------------------+ | walk_packages | Not implemented | :func:`pkgutil.walk_packages` (2.6+) | +----------------+--------------------------------+--------------------------------------------+ | any | Not implemented | :func:`any` | diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index b053ab2aab..da8edcff38 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -12,11 +12,6 @@ # pylint: disable=E0601,E0602,E0611,W0611,W0622,C0103 -try: - from email.Utils import formatdate -except ImportError: - from email.utils import formatdate - # urllib imports try: from urllib import quote_plus From a9b4c0eb6463c0f94723e15dd134f7d09a3b916e Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 18:01:33 +0100 Subject: [PATCH 06/15] Compat: Remove md5. --- doc/development/compat.txt | 12 ------------ src/lib/Bcfg2/Compat.py | 5 ----- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 3 ++- src/lib/Bcfg2/Server/Encryption.py | 4 +++- src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 4 +++- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 +++- 6 files changed, 11 insertions(+), 21 deletions(-) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index 439c1a8316..186fd9ba1b 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -122,8 +122,6 @@ below, since some of these implementations may be feature-incomplete. +----------------+--------------------------------+--------------------------------------------+ | wraps | Not implemented | :func:`functools.wraps` | +----------------+--------------------------------+--------------------------------------------+ -| md5 | :func:`md5.md5` | :class:`hashlib.md5` | -+----------------+--------------------------------+--------------------------------------------+ | literal_eval | :func:`eval` | :func:`ast.literal_eval` (2.6+) | +----------------+--------------------------------+--------------------------------------------+ @@ -146,16 +144,6 @@ The ``wraps`` implementation for Python 2.4 is a no-op. It does not attempt to copy the docstring or other details from the original function to the wrapped function. -md5 -~~~ - -:mod:`hashlib` is available for Python 2.4, but it is not part of the -standard base. If it is installed, it will be used. If you are doing -something fancy with MD5 sums, you may need to determine which object -is in use, since they are not equivalent. For the majority of simple -cases -- finding the MD5 sum of a string -- they are equivalent -enough. - literal_eval ~~~~~~~~~~~~ diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index da8edcff38..8bd62f5000 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -242,11 +242,6 @@ def any(iterable): return True return False -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - def oct_mode(mode): """ Convert a decimal number describing a POSIX permissions mode diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 5c3f70147f..6c123d9056 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -7,6 +7,7 @@ from datetime import datetime from time import strptime from lxml import etree +from hashlib import md5 import sys import django @@ -16,7 +17,7 @@ import Bcfg2.Options import Bcfg2.DBSettings -from Bcfg2.Compat import b64decode, md5 +from Bcfg2.Compat import b64decode from Bcfg2.Reporting.Compat import transaction from Bcfg2.Reporting.Storage.base import StorageBase, StorageError from Bcfg2.Server.Plugin.exceptions import PluginExecutionError diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py index c6cd4232e5..03a3e10dd0 100755 --- a/src/lib/Bcfg2/Server/Encryption.py +++ b/src/lib/Bcfg2/Server/Encryption.py @@ -7,13 +7,15 @@ import copy import logging import lxml.etree +from hashlib import md5 + import Bcfg2.Logger import Bcfg2.Options from M2Crypto import Rand from M2Crypto.EVP import Cipher, EVPError from Bcfg2.Utils import safe_input from Bcfg2.Server import XMLParser -from Bcfg2.Compat import md5, b64encode, b64decode, StringIO +from Bcfg2.Compat import b64encode, b64decode, StringIO #: Constant representing the encryption operation for #: :class:`M2Crypto.EVP.Cipher`, which uses a simple integer. This diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 004e278749..988663f037 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -75,10 +75,12 @@ import copy import lxml.etree +from hashlib import md5 + import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Logger import Debuggable -from Bcfg2.Compat import any, md5 # pylint: disable=W0622 +from Bcfg2.Compat import any # pylint: disable=W0622 from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Server.Statistics import track_statistics diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 86f7698f72..21dd7bee80 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -49,10 +49,12 @@ import os import re import sys +from hashlib import md5 + from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ - cPickle, md5 + cPickle from Bcfg2.Server.Statistics import track_statistics From 4d59fcd769d99c40888f89deebf1263885cbbdb8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:50:55 +0100 Subject: [PATCH 07/15] doc: Add note about 2.4 compat stuff. --- doc/development/compat.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index 186fd9ba1b..22c4f4d1a6 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -111,6 +111,9 @@ The following symbols are imported or defined to provide compatibility with Python 2.4 (and occasionally 2.5). Be sure to read the notes below, since some of these implementations may be feature-incomplete. +As we require Python 2.6 on the server side, you should only use these +compat methods in common or client code. + +----------------+--------------------------------+--------------------------------------------+ | Name | Python 2.4 | Python 2.5+ | +================+================================+============================================+ From 0fbd1d5610934e48aac630f224f11efc9c7ebaa0 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:45:37 +0100 Subject: [PATCH 08/15] Compat: Do not use "any" from Compat on the server. --- src/lib/Bcfg2/Server/Info.py | 2 +- src/lib/Bcfg2/Server/Lint/TemplateAbuse.py | 1 - src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 2 +- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 1 - src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 2 +- 7 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/lib/Bcfg2/Server/Info.py b/src/lib/Bcfg2/Server/Info.py index 044dcdf0c2..c2109bffbe 100644 --- a/src/lib/Bcfg2/Server/Info.py +++ b/src/lib/Bcfg2/Server/Info.py @@ -19,7 +19,7 @@ import Bcfg2.Server.Core import Bcfg2.Server.Plugin import Bcfg2.Client.Tools.POSIX -from Bcfg2.Compat import any # pylint: disable=W0622 + try: try: diff --git a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py index a437c13182..5f9af179ac 100644 --- a/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py +++ b/src/lib/Bcfg2/Server/Lint/TemplateAbuse.py @@ -3,7 +3,6 @@ import os import stat import Bcfg2.Server.Lint -from Bcfg2.Compat import any # pylint: disable=W0622 from Bcfg2.Server.Plugin import default_path_metadata from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index dae03144af..13ce50b97f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,7 +10,7 @@ import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, any, walk_packages +from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages # pylint: enable=W0622 try: diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 729a3fa126..44046b352a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -19,7 +19,7 @@ from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache # pylint: disable=W0622 -from Bcfg2.Compat import all, any, wraps +from Bcfg2.Compat import all, wraps # pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 988663f037..dd768a8933 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -80,7 +80,6 @@ import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Logger import Debuggable -from Bcfg2.Compat import any # pylint: disable=W0622 from Bcfg2.Server.FileMonitor import get_fam from Bcfg2.Server.Statistics import track_statistics diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 0a0ed9414a..eb0da8221d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -10,7 +10,7 @@ import Bcfg2.Server import Bcfg2.Server.Cache import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode, any # pylint: disable=W0622 +from Bcfg2.Compat import unicode # pylint: disable=W0622 import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable from Bcfg2.Server.Statistics import track_statistics diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 08acc4d8dd..8a3adacea4 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -12,7 +12,7 @@ from itertools import chain from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import PluginExecutionError -from Bcfg2.Compat import any, u_str, b64encode # pylint: disable=W0622 +from Bcfg2.Compat import u_str, b64encode try: from Bcfg2.Server.Encryption import ssl_encrypt, bruteforce_decrypt, \ EVPError From 35bbbaa53b54cdaa83bc90048d3a63bd0c404a4b Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:47:05 +0100 Subject: [PATCH 09/15] Compat: Do not use "all" from Compat on the Server. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 44046b352a..73e3ff85a6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -18,9 +18,7 @@ import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache -# pylint: disable=W0622 -from Bcfg2.Compat import all, wraps -# pylint: enable=W0622 +from Bcfg2.Compat import wraps from Bcfg2.version import Bcfg2VersionInfo try: From 3d72071a4349900e4e506299dcddc9fdb223e4c0 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:52:18 +0100 Subject: [PATCH 10/15] Compat: Do not use "walk_packages" from Compat on the Server. --- src/lib/Bcfg2/Server/Lint/__init__.py | 4 +--- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 873e5f1496..668b66e613 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -11,14 +11,12 @@ import termios import textwrap import time - import lxml.etree - +from pkgutil import walk_packages import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins -from Bcfg2.Compat import walk_packages from Bcfg2.Options import _debug diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 13ce50b97f..1194e2ce3f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -6,11 +6,13 @@ import errno import operator import lxml.etree +from pkgutil import walk_packages + import Bcfg2.Options import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 -from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages +from Bcfg2.Compat import u_str, unicode, b64encode # pylint: enable=W0622 try: From 4b1283cd004182e4fc8b1ba2ff18d2284396522a Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 17:57:45 +0100 Subject: [PATCH 11/15] Compat: Do not use "wraps" from Compat on the Server. --- src/lib/Bcfg2/Server/Core.py | 3 ++- src/lib/Bcfg2/Server/MultiprocessingCore.py | 3 ++- src/lib/Bcfg2/Server/Plugin/helpers.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py | 2 +- src/lib/Bcfg2/Server/Statistics.py | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 445bc17b5b..ac4a23e5c0 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -18,9 +18,10 @@ import Bcfg2.DBSettings import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor +from functools import wraps from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Statistics import track_statistics diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 6049d59378..97a4e1bbe9 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -18,8 +18,9 @@ import Bcfg2.Options import Bcfg2.Server.Cache import Bcfg2.Server.Plugin +from functools import wraps from itertools import cycle -from Bcfg2.Compat import Queue, Empty, wraps +from Bcfg2.Compat import Queue, Empty from Bcfg2.Server.Core import Core, exposed from Bcfg2.Server.BuiltinCore import BuiltinCore from multiprocessing.connection import Listener, Client diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index ed524600ad..55489516bf 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -10,12 +10,13 @@ import operator import lxml.etree from collections import MutableMapping +from functools import wraps import Bcfg2.Server import Bcfg2.Options import Bcfg2.Server.FileMonitor from Bcfg2.Logger import Debuggable -from Bcfg2.Compat import CmpMixin, wraps +from Bcfg2.Compat import CmpMixin from Bcfg2.Server.Plugin.base import Plugin from Bcfg2.Server.Plugin.interfaces import Generator, TemplateDataProvider from Bcfg2.Server.Plugin.exceptions import SpecificityError, \ diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 73e3ff85a6..0c3166800f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -11,6 +11,7 @@ import logging import lxml.etree from collections import MutableMapping +from functools import wraps import Bcfg2.Server import Bcfg2.Options @@ -18,7 +19,6 @@ import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import wraps from Bcfg2.version import Bcfg2VersionInfo try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py index 89cc230903..d823033fd5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/YumHelper.py @@ -10,7 +10,7 @@ import logging import Bcfg2.Options import Bcfg2.Logger -from Bcfg2.Compat import wraps +from functools import wraps from lockfile import FileLock, LockTimeout try: import json diff --git a/src/lib/Bcfg2/Server/Statistics.py b/src/lib/Bcfg2/Server/Statistics.py index e34135d4ba..e16ca0628f 100644 --- a/src/lib/Bcfg2/Server/Statistics.py +++ b/src/lib/Bcfg2/Server/Statistics.py @@ -3,7 +3,7 @@ :func:`Bcfg2.Server.Core.BaseCore.get_statistics`.""" import time -from Bcfg2.Compat import wraps +from functools import wraps class Statistic(object): From af72350973914cbe5ab163c1974bead705e6927f Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Nov 2016 18:38:15 +0100 Subject: [PATCH 12/15] doc: Update release notes. --- doc/releases/1.4.0pre2.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt index 103af1a0aa..43408f8ef9 100644 --- a/doc/releases/1.4.0pre2.txt +++ b/doc/releases/1.4.0pre2.txt @@ -46,6 +46,8 @@ backwards-incompatible user-facing changes your queries default to the class names and the Ldap plugin expires the metadata caches if the config file changes. +* Droped support for Python 2.5 or below on the bcfg2 server. + Thanks ------ From 20d3a53ec293304440c93871e0b161f2772741e8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 10 Nov 2017 18:52:11 +0100 Subject: [PATCH 13/15] testsuite: Exclude server tools from legacy checks The server does not support Python < 2.6 anymore, so we do not need to (and cannot) check the server tools with the older python versions. --- testsuite/Testsrc/test_code_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 44c4633a77..2ff3459564 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -83,6 +83,7 @@ if sys.version_info < (2, 6): # Server requires python 2.6 no_checks['lib/Bcfg2'] = ['Server'] + no_checks['sbin'] = ['bcfg2-*'] try: any From 22ca084ac9ec0a56e971b7cb62fb4d8ffb1a9150 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 4 Dec 2017 16:51:51 +0100 Subject: [PATCH 14/15] testsuite: Fix "error only" tests of sbin/ scripts Before the "error only" tests of the scripts in the sbin/ directory ignored the list of files that should not checked at all. --- testsuite/Testsrc/test_code_checks.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index 2ff3459564..78a20cf57a 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -247,16 +247,19 @@ def test_sbin(self): @skipUnless(self.has_exec(), "%s not found, skipping" % self.command[0]) def inner(): - all_sbin = [os.path.join(srcpath, "sbin", f) - for f in glob.glob(os.path.join(srcpath, "sbin", "*"))] - full_list = blacklist_filter([f for f in all_sbin - if not os.path.islink(f)], - self.full_blacklist) + all_sbin = [ + os.path.join(srcpath, "sbin", f) + for f in glob.glob(os.path.join(srcpath, "sbin", "*")) + if not os.path.islink(os.path.join(srcpath, "sbin", f)) + ] + + full_list = blacklist_filter(all_sbin, self.full_blacklist) self._test_full(full_list, extra_args=self.sbin_args) - errors_list = blacklist_filter([f for f in all_sbin - if not os.path.islink(f)], - self.contingent_blacklist) + whitelist = expand_path_dict(error_checks) + errors_list = blacklist_filter( + whitelist_filter(all_sbin, whitelist), + self.contingent_blacklist) self._test_errors(errors_list, extra_args=self.sbin_args) inner() From 2dd9defd7eaf86be57524b80706e7905c6474ffc Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 4 Dec 2017 17:01:48 +0100 Subject: [PATCH 15/15] testsuite: Do not use django-2.0 on Python 2.7 --- testsuite/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/install.sh b/testsuite/install.sh index 1185f89818..a423a7e456 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -42,7 +42,7 @@ else pip_wheel m2crypto guppy fi - pip_wheel django mercurial cheetah3 + pip_wheel 'django<2' mercurial cheetah3 fi fi fi