From e9b8d4dd611aa0fb7154a7fc4be5b3e42f471c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E6=AD=A3?= Date: Sun, 26 Nov 2017 20:09:17 +0800 Subject: [PATCH] Compatible python2.x --- pierky/p2es/es.py | 43 ++++++++++++++++++--------------- pierky/p2es/readers.py | 4 +-- pierky/p2es/transformations.py | 42 ++++++++++++++++---------------- pierky/p2es/writers.py | 2 +- scripts/pmacct-to-elasticsearch | 32 ++++++++++++------------ 5 files changed, 64 insertions(+), 59 deletions(-) diff --git a/pierky/p2es/es.py b/pierky/p2es/es.py index dc9e3f8..b6f97c2 100644 --- a/pierky/p2es/es.py +++ b/pierky/p2es/es.py @@ -2,6 +2,11 @@ # See full license in LICENSE file. import json +try: + import signal + signal.signal(signal.SIGCHLD, signal.SIG_DFL) +except: + pass import requests from requests.auth import HTTPDigestAuth, HTTPBasicAuth @@ -16,7 +21,7 @@ def http(CONFIG, url, method="GET", data=None): auth = HTTPDigestAuth(CONFIG['ES_UserName'], CONFIG['ES_Password']) else: raise P2ESError( - 'Unexpected authentication type: {}'.format(CONFIG['ES_AuthType']) + 'Unexpected authentication type: {0}'.format(CONFIG['ES_AuthType']) ) headers = {'Content-Type': 'application/x-ndjson'} @@ -30,14 +35,14 @@ def http(CONFIG, url, method="GET", data=None): elif method == "HEAD": return requests.head(url, auth=auth, headers=headers) else: - raise Exception("Method unknown: {}".format(method)) + raise Exception("Method unknown: {0}".format(method)) # Sends data to ES. # Raises exceptions: yes. def send_to_es(CONFIG, index_name, data): # HTTP bulk insert toward ES - url = '{}/{}/{}/_bulk'.format( + url = '{0}/{1}/{2}/_bulk'.format( CONFIG['ES_URL'], index_name, CONFIG['ES_Type'] @@ -47,7 +52,7 @@ def send_to_es(CONFIG, index_name, data): http_res = http(CONFIG, url, method="POST", data=data) except Exception as e: raise P2ESError( - 'Error while executing HTTP bulk insert on {} - {}'.format( + 'Error while executing HTTP bulk insert on {0} - {1}'.format( index_name, str(e) ) ) @@ -55,9 +60,9 @@ def send_to_es(CONFIG, index_name, data): # Interpreting HTTP bulk insert response if http_res.status_code != 200: raise P2ESError( - 'Bulk insert on {} failed - ' - 'HTTP status code = {} - ' - 'Response {}'.format( + 'Bulk insert on {0} failed - ' + 'HTTP status code = {1} - ' + 'Response {2}'.format( index_name, http_res.status_code, http_res.text ) ) @@ -67,8 +72,8 @@ def send_to_es(CONFIG, index_name, data): except Exception as e: raise P2ESError( 'Error while decoding JSON HTTP response - ' - '{} - ' - 'first 100 characters: {}'.format( + '{0} - ' + 'first 100 characters: {1}'.format( str(e), http_res.text[:100], ) @@ -76,7 +81,7 @@ def send_to_es(CONFIG, index_name, data): if json_res['errors']: raise P2ESError( - 'Bulk insert on {} failed to process ' + 'Bulk insert on {0} failed to process ' 'one or more documents'.format(index_name) ) @@ -84,7 +89,7 @@ def send_to_es(CONFIG, index_name, data): # Returns: True | False. # Raises exceptions: yes. def does_index_exist(index_name, CONFIG): - url = '{}/{}'.format(CONFIG['ES_URL'], index_name) + url = '{0}/{1}'.format(CONFIG['ES_URL'], index_name) try: status_code = http(CONFIG, url, method="HEAD").status_code @@ -92,10 +97,10 @@ def does_index_exist(index_name, CONFIG): return True if status_code == 404: return False - raise Exception("Unexpected status code: {}".format(status_code)) + raise Exception("Unexpected status code: {0}".format(status_code)) except Exception as err: raise P2ESError( - 'Error while checking if {} index exists: {}'.format( + 'Error while checking if {0} index exists: {1}'.format( index_name, str(err) ) ) @@ -109,33 +114,33 @@ def create_index(index_name, CONFIG): return # index does not exist, creating it - tpl_path = '{}/{}'.format(CONFIG['CONF_DIR'], CONFIG['ES_IndexTemplateFileName']) + tpl_path = '{0}/{1}'.format(CONFIG['CONF_DIR'], CONFIG['ES_IndexTemplateFileName']) try: with open(tpl_path, "r") as f: tpl = f.read() except Exception as e: raise P2ESError( - 'Error while reading index template from file {}: {}'.format( + 'Error while reading index template from file {0}: {1}'.format( tpl_path, str(e) ) ) - url = '{}/{}'.format(CONFIG['ES_URL'], index_name) + url = '{0}/{1}'.format(CONFIG['ES_URL'], index_name) last_err = None try: # using PUT http_res = http(CONFIG, url, method="PUT", data=tpl) except Exception as e1: - last_err = "Error using PUT method: {}".format(str(e1)) + last_err = "Error using PUT method: {0}".format(str(e1)) # trying the old way try: http_res = http(CONFIG, url, method="POST", data=tpl) except Exception as e2: # something went wrong: does index exist anyway? last_err += " - " - last_err += "Error using old way: {}".format(str(e2)) + last_err += "Error using old way: {0}".format(str(e2)) pass try: @@ -144,7 +149,7 @@ def create_index(index_name, CONFIG): except: pass - err = "An error occurred while creating index {} from template {}: " + err = "An error occurred while creating index {0} from template {1}: " if last_err: err += last_err else: diff --git a/pierky/p2es/readers.py b/pierky/p2es/readers.py index e186b22..0300ecf 100644 --- a/pierky/p2es/readers.py +++ b/pierky/p2es/readers.py @@ -38,7 +38,7 @@ def expand_data_macros(s, dic): if "$" in s: res = s for k in dic: - res = res.replace("${}".format(k), str(dic[k])) + res = res.replace("${0}".format(k), str(dic[k])) return res return s @@ -118,7 +118,7 @@ def __init__(self, CONFIG, input_file, errors_queue, writer_queue): for thread_idx in range(CONFIG['ReaderThreads']): self.readers.append( self.READER_THREAD_CLASS( - "reader{}".format(thread_idx), + "reader{0}".format(thread_idx), CONFIG, errors_queue, writer_queue diff --git a/pierky/p2es/transformations.py b/pierky/p2es/transformations.py index b96a799..253704d 100644 --- a/pierky/p2es/transformations.py +++ b/pierky/p2es/transformations.py @@ -37,7 +37,7 @@ def parse_conditions_list(c, d): else: raise P2ESError( 'Logical groups must begin with "AND" or "OR" ' - '("{}" found)'.format(c[0]) + '("{0}" found)'.format(c[0]) ) else: # default to "AND" if not specified @@ -60,7 +60,7 @@ def parse_conditions_dict(c, d, opfield): op = c[k] if not op in ('=', '>', '>=', '<', '<=', '!=', 'in', 'notin'): - raise P2ESError('Unexpected operator: "{}"'.format(op)) + raise P2ESError('Unexpected operator: "{0}"'.format(op)) else: if n is None: n = k @@ -69,7 +69,7 @@ def parse_conditions_dict(c, d, opfield): raise P2ESError('Only one name/value pair allowed') if op in ('in', 'notin') and not isinstance(v, list): - raise P2ESError('The "{}" operator requires a list'.format(op)) + raise P2ESError('The "{0}" operator requires a list'.format(op)) if n is None: raise P2ESError('Name/value pair expected') @@ -94,7 +94,7 @@ def parse_conditions_dict(c, d, opfield): elif op == 'notin': return not d[n] in v else: - raise P2ESError('Operator not implemented: "{}"'.format(op)) + raise P2ESError('Operator not implemented: "{0}"'.format(op)) # Parse conditions c against data d. # Return: True | False (conditions matched / did not match). @@ -105,7 +105,7 @@ def parse_conditions(c, d, opfield='__op__'): elif isinstance(c, dict): return parse_conditions_dict(c, d, opfield) else: - raise P2ESError('Unexpected object type {} from {}'.format( + raise P2ESError('Unexpected object type {0} from {1}'.format( type(c), str(c) )) @@ -116,37 +116,37 @@ def test_transformation(tr): ret = True try: - tr_det = 'Transformations matrix ({})'.format(transformation) + tr_det = 'Transformations matrix ({0})'.format(transformation) except: tr_det = 'Transformations matrix' if 'Conditions' not in tr: - raise P2ESError('{}, "Conditions" is missing'.format(tr_det)) + raise P2ESError('{0}, "Conditions" is missing'.format(tr_det)) if 'Actions' not in tr: - raise P2ESError('{}, "Actions" is missing'.format(tr_det)) + raise P2ESError('{0}, "Actions" is missing'.format(tr_det)) try: - parse_conditions(tr['Conditions'], {}) + parse_conditions(tr['Conditions'], {0}) except P2ESError as e: - raise P2ESError('{}, invalid "Conditions": {}'.format(tr_det, str(e))) + raise P2ESError('{0}, invalid "Conditions": {1}'.format(tr_det, str(e))) for action in tr['Actions']: if 'Type' not in action: - raise P2ESError('{}, "Type" is missing'.format(tr_det)) + raise P2ESError('{0}, "Type" is missing'.format(tr_det)) - tr_det += ', action type = {}'.format(action['Type']) + tr_det += ', action type = {0}'.format(action['Type']) if action['Type'] not in ('AddField', 'AddFieldLookup', 'DelField'): - raise P2ESError('{}, "Type" unknown'.format(tr_det)) + raise P2ESError('{0}, "Type" unknown'.format(tr_det)) if 'Name' not in action: - raise P2ESError('{}, "Name" is missing'.format(tr_det)) + raise P2ESError('{0}, "Name" is missing'.format(tr_det)) if action['Type'] == 'AddField': if 'Value' not in action: raise P2ESError( - '{}, "Value" is missing for new field "{}"'.format( + '{0}, "Value" is missing for new field "{1}"'.format( tr_det, action['Name'] ) ) @@ -154,18 +154,18 @@ def test_transformation(tr): if action['Type'] == 'AddFieldLookup': if 'LookupFieldName' not in action: raise P2ESError( - '{}, "LookupFieldName" is missing for ' - 'new field "{}"'.format(tr_det, action['Name']) + '{0}, "LookupFieldName" is missing for ' + 'new field "{1}"'.format(tr_det, action['Name']) ) if 'LookupTable' in action and 'LookupTableFile' in action: raise P2ESError( - '{}, only one from "LookupTable" and ' + '{0}, only one from "LookupTable" and ' '"LookupTableFile" allowed'.format(tr_det) ) if 'LookupTable' not in action and 'LookupTableFile' not in action: raise P2ESError( - '{}, "LookupTable" and "LookupTableFile" missing ' - 'for new field "{}"'.format(tr_det, action['Name']) + '{0}, "LookupTable" and "LookupTableFile" missing ' + 'for new field "{1}"'.format(tr_det, action['Name']) ) if 'LookupTableFile' in action: try: @@ -173,7 +173,7 @@ def test_transformation(tr): action['LookupTable'] = json.load(f) except Exception as e: raise P2ESError( - '{}, error loading lookup table from {}: {}'.format( + '{0}, error loading lookup table from {1}: {2}'.format( tr_det, action['LookupTableFile'], str(e) ) ) diff --git a/pierky/p2es/writers.py b/pierky/p2es/writers.py index 1e6c9f2..ae4d649 100644 --- a/pierky/p2es/writers.py +++ b/pierky/p2es/writers.py @@ -74,7 +74,7 @@ def __init__(self, *args, **kwargs): create_index(self.index_name, self.CONFIG) except P2ESError as e: raise P2ESError( - "Error while creating index {}: {}".format( + "Error while creating index {0}: {1}".format( self.index_name, str(e) ) ) diff --git a/scripts/pmacct-to-elasticsearch b/scripts/pmacct-to-elasticsearch index a992b61..bb87c29 100755 --- a/scripts/pmacct-to-elasticsearch +++ b/scripts/pmacct-to-elasticsearch @@ -25,14 +25,14 @@ from pierky.p2es.version import __version__ from pierky.p2es.writers import PrintOnlyWriterThread, ESWriterThread APP_NAME = 'pmacct-to-elasticsearch' -CURRENT_RELEASE = 'v{}'.format(__version__) +CURRENT_RELEASE = 'v{0}'.format(__version__) CONF_DIR = '/etc/p2es' DEF_CONFIG = { 'CONF_DIR': CONF_DIR, - 'LogFile': '/var/log/{}-$PluginName.log'.format(APP_NAME), + 'LogFile': '/var/log/{0}-$PluginName.log'.format(APP_NAME), 'ES_URL': 'http://localhost:9200', 'ES_IndexName': '', @@ -83,13 +83,13 @@ def check_config(): if not 'ES_IndexTemplateFileName' in CONFIG: CONFIG['ES_IndexTemplateFileName'] = 'new-index-template.json' else: - index_tpl_path = '{}/{}'.format( + index_tpl_path = '{0}/{1}'.format( CONF_DIR, CONFIG['ES_IndexTemplateFileName'] ) if not os.path.isfile(index_tpl_path): log(logging.ERROR, - 'Can\'t find index template file {}'.format(index_tpl_path)) + 'Can\'t find index template file {0}'.format(index_tpl_path)) return False else: with open(index_tpl_path, "r") as f: @@ -97,8 +97,8 @@ def check_config(): index_tpl = json.load(f) except Exception as e: log(logging.ERROR, - 'Index template from {} is not ' - 'in valid JSON format: {}'.format( + 'Index template from {0} is not ' + 'in valid JSON format: {1}'.format( index_tpl_path, str(e) ), exc_info=True) @@ -162,7 +162,7 @@ def check_config(): valid_input_formats = ('json', 'csv') if CONFIG['InputFormat'] not in valid_input_formats: log(logging.ERROR, - 'Unknown input format "{}": must be one of {}'.format( + 'Unknown input format "{0}": must be one of {1}'.format( CONFIG['InputFormat'], ", ".join(valid_input_formats) ) ) @@ -174,7 +174,7 @@ def check_config(): test_transformation(tr) except P2ESError as e: log(logging.ERROR, - 'Invalid transformation: {}'.format(str(e))) + 'Invalid transformation: {0}'.format(str(e))) return False return True @@ -211,7 +211,7 @@ def setup_logging(baselogfile=None): logger.addHandler(hdlr) except: log(logging.ERROR, - "Can't setup logging to file {}. " + "Can't setup logging to file {0}. " "Ensure it has write permissions for the current user.".format( logfilepath ) @@ -236,10 +236,10 @@ def main(): global CONF_DIR parser = argparse.ArgumentParser( - description="pmacct-to-elasticsearch {}: a tool to read " + description="pmacct-to-elasticsearch {0}: a tool to read " "pmacct's output and to store it into " "ElasticSearch.".format(CURRENT_RELEASE), - epilog="Copyright (c) 2014-{} - Pier Carlo Chiodi - " + epilog="Copyright (c) 2014-{0} - Pier Carlo Chiodi - " "https://pierky.com".format(2017) ) parser.add_argument( @@ -377,7 +377,7 @@ def main(): with open(args.test_condition_data, "r") as f: d = json.load(f) print( - "Tested condition evaluated to {}".format( + "Tested condition evaluated to {0}".format( parse_conditions(c, d) ) ) @@ -392,13 +392,13 @@ def main(): CONFIG["PluginName"] = args.pluginname # Loading configuration - new_cfg_file_name = "{}.conf".format(CONFIG["PluginName"]) + new_cfg_file_name = "{0}.conf".format(CONFIG["PluginName"]) try: - with open('{}/{}'.format(CONF_DIR, new_cfg_file_name), "r") as f: + with open('{0}/{1}'.format(CONF_DIR, new_cfg_file_name), "r") as f: new_cfg = json.load(f) except: log(logging.ERROR, - 'Error loading configuration from {}/{}'.format( + 'Error loading configuration from {0}/{1}'.format( CONF_DIR, new_cfg_file_name), exc_info=True) return False @@ -486,7 +486,7 @@ def main(): reader.process_input() except Exception as e: log(logging.ERROR, - "Error while processing input: {}".format(str(e))) + "Error while processing input: {0}".format(str(e))) ret_code = False finally: reader.finalize()