From dc08fddac3a619181cd663d944c19a645b1356ed Mon Sep 17 00:00:00 2001 From: Christian Harendt Date: Fri, 13 May 2022 14:33:30 +0200 Subject: [PATCH] initial import --- .env.example | 19 ++ .gitignore | 10 + Dockerfile | 17 + LICENSE.txt | 177 +++++++++++ NOTICE | 1 + README.md | 184 +++++++++++ docker-compose.override.example.yml | 7 + docker-compose.yml | 6 + docker-entrypoint.sh | 3 + pyproject.toml | 3 + requirements.txt | 6 + ripeupdater/__init__.py | 1 + ripeupdater/backup_manager.py | 78 +++++ ripeupdater/configuration.py | 153 +++++++++ ripeupdater/exceptions.py | 54 ++++ ripeupdater/functions.py | 170 ++++++++++ ripeupdater/log_manager.py | 46 +++ ripeupdater/main.py | 123 ++++++++ ripeupdater/netbox.py | 198 ++++++++++++ ripeupdater/ripe.py | 393 ++++++++++++++++++++++++ ripeupdater/templates/backups.html | 16 + setup.cfg | 5 + templates/base_mycompany.example.json | 9 + templates/base_mycustomer1.example.json | 10 + templates/lir_org.example.json | 8 + templates/templates.example.json | 25 ++ test-requirements.txt | 4 + tests/__init__.py | 0 tests/base_mycompany.json | 9 + tests/example.json | 25 ++ tests/lir_org.json | 8 + tests/test_functions.py | 66 ++++ tests/test_netbox.py | 45 +++ tests/test_ripe.py | 95 ++++++ tox.ini | 11 + 35 files changed, 1985 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE.txt create mode 100644 NOTICE create mode 100644 README.md create mode 100644 docker-compose.override.example.yml create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 ripeupdater/__init__.py create mode 100644 ripeupdater/backup_manager.py create mode 100644 ripeupdater/configuration.py create mode 100644 ripeupdater/exceptions.py create mode 100644 ripeupdater/functions.py create mode 100755 ripeupdater/log_manager.py create mode 100755 ripeupdater/main.py create mode 100644 ripeupdater/netbox.py create mode 100644 ripeupdater/ripe.py create mode 100644 ripeupdater/templates/backups.html create mode 100644 setup.cfg create mode 100644 templates/base_mycompany.example.json create mode 100644 templates/base_mycustomer1.example.json create mode 100644 templates/lir_org.example.json create mode 100644 templates/templates.example.json create mode 100644 test-requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/base_mycompany.json create mode 100644 tests/example.json create mode 100644 tests/lir_org.json create mode 100644 tests/test_functions.py create mode 100644 tests/test_netbox.py create mode 100644 tests/test_ripe.py create mode 100644 tox.ini diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e7fb39d --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +S3_BACKUP=no +S3_ENDPOINT_URL=https://s3.example.com +S3_BUCKET=example-bucket +S3_ACCESS_KEY=123456abcdef +S3_SECRET_ACCESS_KEY=123456abcdef +SMALLEST_PREFIX_V4=31 +SMALLEST_PREFIX_V6=127 +MAIL_REPORT=no +SMTP=smtp.example.com:587 +SENDER_MAIL=ripe-updater@example.com +RECIPIENT_MAIL=noc@example.com +NETBOX_URL=https://netbox.local +NETBOX_TOKEN=123456abcdef +DEFAULT_COUNTRY=DE +RIPE_MNT_PASSWORD=emptypassword +RIPE_DB=TEST +UPDATE_TOKEN=Token 123456abcdef +DEBUG=no +TEMPLATE_DIR=./templates diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8893c70 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.pyc +*.swp +/venv/ +*.egg-info +docker-compose.override.yml +.DS_Store +.env +.vscode +.tox +.coverage diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..67d5335 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10-alpine + +RUN apk add --no-cache gcc +RUN addgroup -S ripeupdater && adduser -S ripeupdater -G ripeupdater + +USER ripeupdater + +WORKDIR /opt/ripeupdater/ + +COPY requirements.txt ./ +RUN pip install -Ur requirements.txt + +COPY ripeupdater ./ripeupdater/ + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] +CMD python -m gunicorn -b :80 -w 2 ripeupdater.main:app diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..4947287 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..56a14f6 --- /dev/null +++ b/NOTICE @@ -0,0 +1 @@ +Copyrighted and licensed under Apache License 2.0 by Inter.link GmbH. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..dc18bff --- /dev/null +++ b/README.md @@ -0,0 +1,184 @@ +# ripe-updater + +ripe-updater is an API wrapper tool between [NetBox](https://github.com/netbox-community/netbox/) and [RIPE-DB](https://apps.db.ripe.net/), to keep INETNUM and INET6NUM objects updated. Initial work has started at [SysEleven](https://syseleven.de) and development continued at [Inter.link](https://inter.link). + +ripe-updater is a [Flask](https://flask.palletsprojects.com/) based Python app. The code is available on [GitHub](https://github.com/interdotlink/ripe-updater/) + +## Features +* Using NetBox Webhooks on Prefix updates +* Templates for RIPE-DB attributes +* Backups of overwritten/deleted objects (stored in S3) +* Email reporting +* handling of overlapping INET(6)NUM objects + +## Deployment +### Requirements +* NetBox 2.4 or later +* Python 3.8 or later + +### Getting started +These steps are mandatory to get ripe-updater up and running. +1. deploy ripe-updater +1. configure ripe-updater +1. configure NetBox +1. setup templates + +### Containerized (recommended) +Copy and edit `.env` +``` +cp .env.example .env +vi .env +docker run \ + -p 8000:80 \ + -v "/home/user/ripe-updater/templates:/opt/ripeupdater/templates:ro" \ + --env-file .env \ + interdotlink/ripe-updater +``` + +#### docker-compose +Copy and edit `docker-compose.override.yml` +``` +cp docker-compose.override.example.yml docker-compose.override.yml +docker-compose up -d +``` + +### Installation on Linux +Edit `ripeupdater/configuration.py`. +``` +pip install -r requirements.txt +python -m gunicorn -b :80 -w 2 ripeupdater.main:app +``` + +### Note for production deployments + +For production use it is recommended, to setup a reverse proxy e.g. Nginx in front of the ripe-updater and add an SSL certificate, e.g. letsencrypt. + +## Configuration +Configuration is set via environment variables, but you can also edit `ripeupdater/configuration.py`. + +| parameter | values | default | description | +| --- | --- | --- | --- | +| DEBUG | yes/no | no | enables verbose logging | +| MAIL_REPORT | yes/no | no | enables email-reporting | +| SMTP | url | 127.0.0.1 | url or ip of smtp server | +| SMTP_STARTTLS | yes/no | no | use STARTTLS when connecting to smtp server | +| SENDER_MAIL | email | - | sender mail of email-reports | +| RECIPIENT_MAIL | email | - | receiver of email-reports | +| UPDATE_TOKEN | string | - | if set, each netbox webhook must contain this tokes as Authorisation header | +| NETBOX_URL | url | - | url of your netbox instance | +| NETBOX_TOKEN | string | - | netbox token, which can read prefixes, aggregates, regions and sites | +| DEFAULT_COUNTRY | ISO3166-II country | - | default country if none could be determined, e.g. DE or NL | +| TEMPLATES_DIR | path | /opt/ripeupdater/templates | location of templates | +| RIPE_MNT_PASSWORD | string | - | ripe maintainer password with write permissions to your INET(6)NUM objects | +| RIPE_DB | RIPE/TEST | TEST | which ripe-db to use | +| RIPE_TEST_MNT | string | TEST-DBM-MNT | which maintainer to use in the TEST database, as your maintainer may not be present | +| RIPE_TEST_ORG | string | ORG-EIPB1-TEST | which organisation to use in the TEST database, as your organisation may not be present | +| RIPE_TEST_PERSON | string | AA1-TEST | which person to use in the TEST database, as your person may not be present | +| RIPE_TEST_STATUS_V4 | string | ALLOCATED PA | which status to use in the TEST database, as your status may not be able to be set. Your parent INETNUM object, with your MNT-LOWER attribute set to your maintainer may be missing. | +| RIPE_TEST_STATUS_V6 | string | ALLOCATED PA | which status to use in the TEST database, as your status may not be able to be set. Your parent INET6NUM object, with your MNT-LOWER attribute set to your maintainer may be missing. | +| SMALLEST_PREFIX_V4 | 0-32 | 31 | prefix length bigger than this limit will not be handled | +| SMALLEST_PREFIX_V6 | 0-128 | 127 | prefix length bigger than this limit will not be handled | +| S3_BACKUP | yes/no | no | enable or disable S3 backups | +| S3_ENDPOINT_URL | url | - | specify url of your s3 endpoint | +| S3_ACCESS_KEY | string | - | access key to your s3 storage | +| S3_SECRET_ACCESS_KEY | string | - | secret access key to your s3 storage | +| S3_BUCKET | string | - | bucket to store backups in | + +### NetBox configuration +You'll need to add three custom fields to NetBox and data needs to be structured in a specific way. + +#### custom field - lir +* Name: `lir` +* Label: LIR +* Assigned Models: ipam -> aggregates +* Type: Selection +* Required: yes +* Choices: ***all LIRs you are responsible for*** +* Description: RIPE Local Internet Registry + +#### custom field - ripe_report +* Name: `ripe_report` +* Label: RIPE Report +* Assigned Models: ipam -> prefixes +* Type: Boolean +* Required: no +* Default: false +* Description: should this prefix be in RIPE-DB + +#### custom field - ripe_template +* Name: `ripe_template` +* Label: RIPE Template +* Assigned Models: ipam -> prefixes +* Type: Selection +* Required: no +* Choices: ***all templates you have created*** + +#### region - country +Your sites need to have a country as a parent region found in [iso3166.countries_by_name](https://github.com/deactivated/python-iso3166) + +#### Webhook +add a webhook to NetBox: +* Name: `ripe-updater` +* Enabled: yes +* Events: Create, Update, Delete +* HTTP Request + * HTTP Method: POST + * Payload URL: http(s)://your-ripe-updater-host/update + * HTTP Content Type: application/json +* Assigned Models: ipam | prefix +* Additional Headers - ***if you have set a token in ripe-updater config, set it here*** + * `Authorisation: Token YOURTOKEN` +* SSL - enable if you have a valid SSL Certificate for your ripe-updater + +## Templates +Templates are devided into three components. +1. `lir_org.json` - a list of LIRs you are responsible for, each mapped to a organisation object. +1. `base_something.json` - a base template with INET(6)NUM attributes. E.g. you have one for yourself and one for each customer which needs to have different attributes (e.g. abuse-c) in RIPE-DB. +1. `templates.json` - a list of templates. These must be also added to NetBox custom field choices of ripe_template. Each mapped to a base template. + +> With the provided example .env file you should be able to test your templates in the TEST database. + +### setup list of LIRs +* copy and edit lir_org.json `cp templates/lir_org.example.json templates/lir_org.json` +* Add each LIR you are responsible for to an organisation object like `"de.examplelir1": "ORG-EIPB1-TEST",` + +### setup your templates +* You should create a template for each case, where you want to document different attributes to your INET(6)NUM objects. E.g. like a different `abuse-c` +* You can take `templates/base_mycompany.example.json` as a starting point. +* You must include an **empty** statement: `{"org": ""},` to autofill organisation attributes from your lir_org list. + +### setup list of templates +* Copy and edit templates.json `cp templates/templates.example.json templates/templates.json` +* Add your templates you are planning to use like + ``` + "CLOUD-POOL": {"attributes": [ + {"descr": "MyCompany Cloud Pool"} + ], + "inherit": "base_mycompany.json" + }, + ``` + +## Backups +If you have enabled and configured a S3 backup storage, you can browse the json representation of deleted or overwritten objects at `http(s)://your-ripe-updater-host/backups`. +To restore a backup manually, you can post the json file to the RIPE database: +``` +curl -X POST -H 'Content-Type: application/json' --data @prefix.json 'https://rest.db.ripe.net/ripe/inetnum?password=RIPE_MNT_PASSWORD' +``` + +## Development +To run the unit tests, run + +``` +pip install tox +tox +``` + +## Known limitations +* Having Ripe-Report set for parent and it's child-prefixes will fail, as you can only have one level of prefixes below your aggregates in RIPE-DB. + * ***Workaround***: Disable Ripe-Reporting of the parent or child prefixes. +* Extending a prefix in NetBox (e.g. /27 to /26) will fail, as there is not deterministic way of detecting this. + * ***Workaround***: Disable Ripe-Reporting of this prefix, extend prefix size, reenable Ripe-Reporting + +## Initial Authors +* Mohamad Mouselli (https://github.com/mmouselli) +* Christian Harendt (christian at inter.link) \ No newline at end of file diff --git a/docker-compose.override.example.yml b/docker-compose.override.example.yml new file mode 100644 index 0000000..3d867ab --- /dev/null +++ b/docker-compose.override.example.yml @@ -0,0 +1,7 @@ +version: "3.9" +services: + ripe-updater: + ports: + - 8000:80 + volumes: + - "./templates:/opt/ripeupdater/templates:ro" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4e8d95b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,6 @@ +version: "3.9" +services: + ripe-updater: + image: interdotlink/ripe-updater:latest + env_file: + - .env diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..4ba1611 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -e +exec "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1b68d94 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..75802c4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +requests==2.27.1 +flask==2.1.1 +iso3166==2.0.2 +gunicorn==20.1.0 +boto3==1.21.44 +pynetbox==6.6.2 \ No newline at end of file diff --git a/ripeupdater/__init__.py b/ripeupdater/__init__.py new file mode 100644 index 0000000..4802e90 --- /dev/null +++ b/ripeupdater/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0" diff --git a/ripeupdater/backup_manager.py b/ripeupdater/backup_manager.py new file mode 100644 index 0000000..aa2bd58 --- /dev/null +++ b/ripeupdater/backup_manager.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +import os +import boto3 + +from botocore.exceptions import ClientError + +from .log_manager import LogManager +from .configuration import * + + +class BackupManager: + """ + Handles storage of backups for ripe objects + """ + def __init__(self): + """ + connect to s3 and ensures presence of the bucket + """ + self.logger = LogManager().logger + + if S3_BACKUP == 'yes': + self.logger.info(f"connect to s3 {S3_ENDPOINT_URL}") + self.s3 = boto3.client( + service_name='s3', + endpoint_url=S3_ENDPOINT_URL, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_ACCESS_KEY + ) + + try: + self.logger.info(f"creating bucket {S3_BUCKET}") + self.s3.create_bucket(Bucket=S3_BUCKET) + except ClientError as error: + if error.response['Error']['Code'] == 'BucketAlreadyExists': + self.logger.info("bucket already exists") + else: + raise error + else: + self.logger.info("S3-Backup disabled") + + def put(self, filename, content): + """ + upload an object to s3 + """ + if S3_BACKUP == 'yes': + return self.s3.put_object( + Bucket=S3_BUCKET, + Key=filename, + Body=content + ) + + return None + + def get(self, filename): + """ + return the content of an object + """ + if S3_BACKUP == 'yes': + return self.s3.get_object( + Bucket=S3_BUCKET, + Key=filename + )['Body'].read() + + return "" + + def list(self): + """ + list all objects in this bucket + """ + if S3_BACKUP == 'yes': + files = self.s3.list_objects(Bucket=S3_BUCKET) + self.logger.debug(f'{files=}') + if files.get('Contents'): + return [o['Key'] for o in files['Contents']] + + return [] + diff --git a/ripeupdater/configuration.py b/ripeupdater/configuration.py new file mode 100644 index 0000000..3183166 --- /dev/null +++ b/ripeupdater/configuration.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +from os import getenv + +# DEBUG +# enables verbose logging +# values: yes/no +# default: no +DEBUG = getenv('DEBUG', 'no') + +# MAIL_REPORT +# enables email-reporting +# values: yes/no +# default: no +MAIL_REPORT = getenv('MAIL_REPORT', 'no') + +# SMTP +# url or ip of smtp server +# values: url +# default: 127.0.0.1 +SMTP = getenv('SMTP', '127.0.0.1') + +# SMTP_STARTTLS +# use STARTTLS when connecting to smtp server +# values: yes/no +# default: no +SMTP_STARTTLS = getenv('SMTP_STARTTLS', 'no') + +# SENDER_MAIL +# sender mail of email-reports +# values: email +# default: - +SENDER_MAIL = getenv('SENDER_MAIL') + +# RECIPIENT_MAIL +# receiver of email-reports +# values: email +# default: - +RECIPIENT_MAIL = getenv('RECIPIENT_MAIL') + +# UPDATE_TOKEN +# if set, each netbox webhook must contain this tokes as Authorisation header +# values: string +# default: - +UPDATE_TOKEN = getenv('UPDATE_TOKEN') + +# NETBOX_URL +# url of your netbox instance +# values: url +# default: - +NETBOX_URL = getenv('NETBOX_URL') + +# NETBOX_TOKEN +# netbox token, which can read prefixes, aggregates, regions and sites +# values: string +# default: - +NETBOX_TOKEN = getenv('NETBOX_TOKEN') + +# DEFAULT_COUNTRY +# default country if none could be determined +# values: ISO3166-II country +# default: - +DEFAULT_COUNTRY = getenv('DEFAULT_COUNTRY') + +# TEMPLATES_DIR +# location of templates +# values: path +# default: /opt/ripeupdater/templates +TEMPLATES_DIR = getenv('TEMPLATES_DIR', '/opt/ripeupdater/templates') + +# RIPE_MNT_PASSWORD +# ripe maintainer password with write permissions to your INET(6)NUM objects +# values: string +# default: - +RIPE_MNT_PASSWORD = getenv('RIPE_MNT_PASSWORD') + +# RIPE_DB +# which ripe-db to use +# values: RIPE/TEST +# default: TEST +RIPE_DB = getenv('RIPE_DB', 'TEST') + +# RIPE_TEST_MNT +# which maintainer to use in the TEST database, as your maintainer may not be present +# values: string +# default: TEST-DBM-MNT +RIPE_TEST_MNT = getenv('RIPE_TEST_MNT', 'TEST-DBM-MNT') + +# RIPE_TEST_ORG +# which organisation to use in the TEST database, as your organisation may not be present +# values: string +# default: ORG-EIPB1-TEST +RIPE_TEST_ORG = getenv('RIPE_TEST_ORG', 'ORG-EIPB1-TEST') + +# RIPE_TEST_PERSON +# which person to use in the TEST database, as your person may not be present +# values: string +# default: AA1-TEST +RIPE_TEST_PERSON = getenv('RIPE_TEST_PERSON', 'AA1-TEST') + +# RIPE_TEST_STATUS_V4 +# which status to use in the TEST database, as your status may not be able to be set. Your parent INETNUM object, with your MNT-LOWER attribute set to your maintainer may be missing. +# values: string +# default: ALLOCATED PA +RIPE_TEST_STATUS_V4 = getenv('RIPE_TEST_STATUS_V4', 'ALLOCATED PA') + +# RIPE_TEST_STATUS_v6 +# which status to use in the TEST database, as your status may not be able to be set. Your parent INET6NUM object, with your MNT-LOWER attribute set to your maintainer may be missing. +# values: string +# default: ALLOCATED PA +RIPE_TEST_STATUS_V6 = getenv('RIPE_TEST_STATUS_V6', 'ALLOCATED PA') + +# SMALLEST_PREFIX_V4 +# prefix length bigger than this limit will not be handled +# values: 0-32 +# default: 31 +SMALLEST_PREFIX_V4 = getenv('SMALLEST_PREFIX_V4', '31') + +# SMALLEST_PREFIX_V6 +# prefix length bigger than this limit will not be handled +# values: 0-128 +# default: 127 +SMALLEST_PREFIX_V6 = getenv('SMALLEST_PREFIX_V6', '127') + +# S3_BACKUP +# enable or disable S3 backups +# values: yes/no +# default: no +S3_BACKUP = getenv('S3_BACKUP', 'no') + +# S3_ENDPOINTURL +# specify url of your s3 endpoint +# values: url +# default: - +S3_ENDPOINT_URL = getenv('S3_ENDPOINT_URL') + +# S3_ACCESS_KEY +# access key to your s3 storage +# values: string +# default: - +S3_ACCESS_KEY = getenv('S3_ACCESS_KEY') + +# S3_SECRET_ACCESS_KEY +# secret access key to your s3 storage +# values: string +# default: - +S3_SECRET_ACCESS_KEY = getenv('S3_SECRET_ACCESS_KEY') + +# S3_BUCKET +# bucket to store backups in +# values: string +# default: - +S3_BUCKET = getenv('S3_BUCKET') diff --git a/ripeupdater/exceptions.py b/ripeupdater/exceptions.py new file mode 100644 index 0000000..2a277eb --- /dev/null +++ b/ripeupdater/exceptions.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +""" +custom exception +""" + + +class RipeUpdaterException(Exception): + """ + exception base class + """ + pass + + +class ErrorSmallPrefix(RipeUpdaterException): + """ + raised if prefix is too small to be handled + """ + pass + + +class MissingDataFromNetbox(RipeUpdaterException): + """ + raised if data cannot be pulled from netbox + """ + pass + + +class NotRoutedNetwork(RipeUpdaterException): + """ + raised if prefix is not meant to be in RIPE DB, e.g. RFC1918 + """ + pass + + +class BadRequest(RipeUpdaterException): + """ + raised if invalid request + """ + pass + + +class ConfigError(RipeUpdaterException): + """ + raised if config data is missing + """ + pass + + +class RipeDBError(RipeUpdaterException): + """ + raised if data could not be querried from RIPE DB + """ + pass diff --git a/ripeupdater/functions.py b/ripeupdater/functions.py new file mode 100644 index 0000000..1489abb --- /dev/null +++ b/ripeupdater/functions.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- + +import json +import os +import smtplib +import socket +from email.message import EmailMessage + +from ipaddress import ip_network +from .exceptions import ErrorSmallPrefix, NotRoutedNetwork +from .log_manager import LogManager +from .configuration import * + +# Dictionary RIPE Documentaion of response codes for each action +RIPE_DOCU_URLS = {'POST': 'https://github.com/RIPE-NCC/whois/wiki/WHOIS-REST-API-Create', + 'PUT': 'https://github.com/RIPE-NCC/whois/wiki/WHOIS-REST-API-Update', + 'DELETE': 'https://github.com/RIPE-NCC/whois/wiki/WHOIS-REST-API-Delete'} + +logger = LogManager().logger + + +def read_json_file(template): + """ + Reading JSON template file and return dict + """ + if not os.path.exists(template): + msg = f'No template file {template}' + logger.critical(msg) + raise RuntimeError(msg) + + with open(template, 'r') as f: + dict = json.load(f) + return dict + + +def flatten_ripe_attributes(obj): + """ + flattens ripe attributes + """ + ripe_attributes = find('attributes.attribute', obj) + return {attr.get('name'): attr.get('value') for attr in ripe_attributes} + + +def format_ripe_object(obj, prefix=''): + """ + expects a ripe_object dict and return a flat string representation + """ + string = '' + if obj: + for key, value in flatten_ripe_attributes(obj).items(): + string += f'{prefix}{key}:\t\t{value}\n' + + return string + + +def is_v6(prefix): + return ip_network(prefix).version == 6 + + +def validate_prefix(prefix): + """ + validate if prefix is valid to be pushed to RIPE DB + """ + logger.debug('Processing prefix to formating it to valid RIPE format') + network = ip_network(prefix) + + if is_v6(prefix): + + # Check if prefix big enough; bigger than defined + if network.prefixlen > int(SMALLEST_PREFIX_V6): + raise ErrorSmallPrefix(f'This prefix is too small update only bigger than {SMALLEST_PREFIX_V6}') + return False + + # Check if private network; no need to continue + if network.is_loopback or \ + network.is_reserved or \ + network.is_private or \ + network.is_multicast or \ + network.is_link_local or \ + not network.is_global: + raise NotRoutedNetwork('This is not routed prefix, it will be ignored') + return False + else: + # Check if prefix big enough; bigger than defined + if network.prefixlen > int(SMALLEST_PREFIX_V4): + raise ErrorSmallPrefix(f'This prefix is too small update only bigger than {SMALLEST_PREFIX_V4}') + + # Check if private network; no need to continue + if network.is_loopback or \ + network.is_reserved or \ + network.is_private or \ + network.is_multicast: + raise NotRoutedNetwork('This is not routed prefix, it will be ignored') + return False + + return True + + +def format_cidr(prefix): + """ + change format of prefix to legacy CIDR notation + """ + network = ip_network(prefix) + + return f'{network[0]} - {network[-1]}' + + +def notify(ripe_object, action, prefix, username, response_code, ripe_errors): + """ + This function uses smtplib and sendmail to send mails to the local MTA + MTA forward it to your recipient. Added to support alarming, + when something is not working. + """ + # Read hostname and IP Address to send out within mail + hostname = socket.gethostname() + try: + ipaddr = socket.gethostbyname(hostname) + except socket.gaierror as err: + ipaddr = 'unknown' + # Building mail content + msg = EmailMessage() + ripe_errors = '\n'.join(ripe_errors) + status = 'succeeded' if response_code == 200 else 'failed' + text = f"""{action} inetnum {prefix} has {status}: +{ripe_errors} +---------------- +{ripe_object} +Result: {status} +Action: {action} +Response code: {str(response_code)} +Response codes doc: {RIPE_DOCU_URLS[action]} +Triggered by: {username} +FQDN: {hostname} +RIPE-Service source IP: {ipaddr} +---------------- +\nFor more informations check logs +\nYour awesome RIPE-Service!""" + + msg.set_content(text) + msg['Subject'] = f'{action} {prefix} has {status}' + msg['From'] = SENDER_MAIL + msg['To'] = RECIPIENT_MAIL + + logger.debug(msg) + + if MAIL_REPORT == 'yes': + try: + logger.debug(f'opening SMTP connection to {SMTP}') + with smtplib.SMTP(SMTP) as server: + if SMTP_STARTTLS == 'yes': + server.starttls() + server.send_message(msg) + except (ConnectionRefusedError, socket.timeout, OSError, smtplib.SMTPServerDisconnected) as err: + msg = f'unable to connect to SMTP server: {SMTP} - {err}' + logger.critical(msg) + raise RuntimeError(msg) + + +def find(path, obj): + """ + find an element in a dictionary using a path + path: 'elem1.elem2.elem3' + obj: {'elem1': {'elem2': {'elem3': 'foo'}}} + """ + for elem in path.split('.'): + obj = obj.get(elem) + if obj is None: + break + + return obj diff --git a/ripeupdater/log_manager.py b/ripeupdater/log_manager.py new file mode 100755 index 0000000..8bddc02 --- /dev/null +++ b/ripeupdater/log_manager.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import os +import logging +from flask import has_request_context, request +from .configuration import * + +loggers = {} + + +class RequestFormatter(logging.Formatter): + def format(self, record): + if has_request_context(): + record.url = request.url + record.remote_addr = request.remote_addr + else: + record.url = None + record.remote_addr = None + + return super().format(record) + + +class LogManager: + """ + setup loggig + """ + def __init__(self): + """ + setup logging + """ + global loggers + + loglevel = logging.DEBUG if DEBUG == 'yes' else logging.INFO + + if loggers.get('logger'): + self.logger = loggers.get('logger') + else: + self.logger = logging.getLogger('logger') + formatter = RequestFormatter( + '[%(asctime)s] [%(process)d] %(remote_addr)s requested %(url)s %(levelname)s in %(module)s: %(message)s' + ) + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + self.logger.setLevel(loglevel) + self.logger.addHandler(console_handler) + loggers['logger'] = self.logger diff --git a/ripeupdater/main.py b/ripeupdater/main.py new file mode 100755 index 0000000..cb7e18d --- /dev/null +++ b/ripeupdater/main.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +# -*- coding: utf-8 -*- + +""" +This module contains HTTP-Service based on Flask, which update RIPE Inetnum & +Inet6num automatically. +It catches webhooks from NetBox and initializes HTTP queries also to NetBox, to build at the end +a valid RIPE Object. +""" +import os + +from flask import Flask, abort, request, render_template +from flask.logging import default_handler + +from .backup_manager import BackupManager +from .log_manager import LogManager +from .netbox import ObjectBuilder +from .ripe import RipeObjectManager +from .exceptions import (RipeUpdaterException, NotRoutedNetwork, ErrorSmallPrefix) +from .configuration import * + +logmgr = LogManager() +logger = logmgr.logger + +# Initialize Flask and giving the application a name 'app' +logger.info('Initialize App') + +app = Flask(__name__) +app.logger.removeHandler(default_handler) +app.logger.addHandler(logger) +backup = BackupManager() + + +@app.route('/health') +def check_health(): + logger.debug('calling /health') + return 'Ok' + + +@app.route('/backups') +def list_backups(): + logger.info('list backups') + return render_template('backups.html', backups=backup.list()) + + +@app.route('/backup/') +def get_backup(name): + logger.info('get backup') + return backup.get(name) + + +@app.route('/update', methods=['POST']) +def update(): + """ + /update is a route which accepts JSON HTTP requests and returns 200 + if the incoming webhook is prefix. + """ + if request.headers.get('Authorisation') != UPDATE_TOKEN: + logger.error('token missmatch') + abort(401) + + logger.info('Update route is runnning and waiting to catch prefixes...') + + # Content-Type: application/json + webhook = request.json + if webhook is None: + msg = 'request payload must be application/json' + logger.error(msg) + return msg, 400 + + # ensure valid netbox request + try: + if webhook['model'] != 'prefix': + msg = 'only prefixes are supported' + logger.error(msg) + return msg, 400 + except KeyError as e: + msg = f'not a valid netbox request. Key not found: {e}' + logger.error(msg) + return msg, 400 + + # ensure presence of custom fields + try: + data = webhook['data'] + custom_fields = data['custom_fields'] + ripe_report = custom_fields['ripe_report'] + except (KeyError, TypeError) as e: + msg = f'missing custom fields. {type(e)}: {e}' + logger.error(msg) + return msg, 400 + + try: + # If ripe_report not selected or false then delete object from RIPE-DB + if ripe_report is not True: + logger.info(f"ripe_report is false, deleting prefix {webhook['data']['prefix']}") + netbox_object = ObjectBuilder(webhook) + ripe = RipeObjectManager(netbox_object, backup) + ripe.delete_object() + + else: + # If the incoming webhook updated or created, (not deleted) then push webhook to + # RIPE-DB + if webhook['event'] != 'deleted': + logger.info(f"updating prefix {webhook['data']['prefix']}") + netbox_object = ObjectBuilder(webhook) + ripe = RipeObjectManager(netbox_object, backup) + ripe.push_object() + else: + # If the incoming webhook is selected as deleted then also delete if from + # RIPE-DB + logger.info(f"prefix deleted in NetBox, deleting prefix {webhook['data']['prefix']} in RIPE DB") + netbox_object = ObjectBuilder(webhook) + ripe = RipeObjectManager(netbox_object, backup) + ripe.delete_object() + except NotRoutedNetwork: + return 'NotRoutedNetwork, skipping request', 200 + except ErrorSmallPrefix: + return 'ErrorSmallPrefix, skipping request', 200 + except RipeUpdaterException as err: + return f'{err=}', 500 + + return '', 204 diff --git a/ripeupdater/netbox.py b/ripeupdater/netbox.py new file mode 100644 index 0000000..f96fe41 --- /dev/null +++ b/ripeupdater/netbox.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- + +import os + +import pynetbox + +from iso3166 import countries_by_alpha2, countries_by_name +from .exceptions import MissingDataFromNetbox +from .functions import read_json_file +from .log_manager import LogManager +from .configuration import * + +# Name of Lir Org mapping template +LIR_ORG = 'lir_org.json' + + +class FetchData: + def __init__(self): + self.logger = LogManager().logger + self.nb = pynetbox.api(NETBOX_URL, token=NETBOX_TOKEN) + + def authorize_delete_overlapped_candidate(self, overlapped_candidate): + """ + if overlapped_candidated is not a prefix nor an aggregate in netbox + return True to indicate this candidate should be deleted from RIPE DB + """ + is_prefix = bool(self.nb.ipam.prefixes.get(prefix=str(overlapped_candidate))) + is_aggregate = bool(self.nb.ipam.aggregates.get(prefix=str(overlapped_candidate))) + self.logger.debug(f'Searched inside netbox for {overlapped_candidate=}, result: \ + {is_prefix=} {is_aggregate=}') + + # Return Ture is the overlapped_object is neither aggregate nor prefix + if is_aggregate or is_prefix: + self.logger.warning(f'Overlapped_candidate {overlapped_candidate} found in NetBox \ + {is_prefix=} {is_aggregate=}') + return False + else: + self.logger.info('Overlapped object is neither aggregate nor prefix, \ + authorized to delete it from RIPE-DB') + return True + + def org(self, prefix): + """ + lookup lir in parent aggregate for prefix and return matching RIPE org + """ + template = f'{TEMPLATES_DIR}/{LIR_ORG}' + dict_template = read_json_file(template) + dict_template = dict_template['templates']['lir_org'].items() + + aggregate = self.nb.ipam.aggregates.get(q=prefix) + netbox_lir = aggregate.custom_fields['lir'] + # be compatible with older netbox api + if type(netbox_lir) is dict: + netbox_lir = netbox_lir['label'] + netbox_lir = netbox_lir.lower() + + self.logger.info('Defining the suitable RIPE Org attribute') + for lir, org in dict_template: + if netbox_lir == lir: + return org + + return None + + def country(self, site_slug): + """ + This methode get for a prefix's country in ISO3166-II format + ISO3166-II is expected from RIPE database + """ + site = self.nb.dcim.sites.get(slug=site_slug) + region = self.nb.dcim.regions.get(slug=site.region.slug) + + self.logger.info('Finding the suitable ISO country name, which RIPE accepts') + while region: + country = region.slug.upper() + self.logger.debug(f'testing region {country}') + if country in countries_by_name: + country_alpha2 = countries_by_name[country].alpha2 + return country_alpha2 + + region = region.parent + + return None + + +class ObjectBuilder: + """ + This class describs methodes to return catchable data from Netbox webhook + """ + def __init__(self, webhook): + self.logger = LogManager().logger + self.webhook = webhook + self.logger.info('Parsing incoming prefix from Netbox') + fetch_data = FetchData() + self.country_netbox = fetch_data.country + self.org_netbox = fetch_data.org + + def prefix(self): + """ + returns prefix as string + """ + data = self.webhook + + try: + prefix = data['data']['prefix'] + + except TypeError: + msg = 'No selected prefix in Netbox' + self.logger.error(msg) + raise MissingDataFromNetbox(msg) + + self.logger.info(f'Prefix: {str(prefix)}') + return prefix + + def username(self): + """ + returns webhook trigger (username) as string + """ + data = self.webhook + + try: + username = data['username'] + + except TypeError: + msg = 'No given user in Netbox webhook, RIPE_Service expects a username' + self.logger.warning(msg) + username = 'None' + + self.logger.info(f'User: {str(username)}') + return username + + def netbox_template(self): + """ + Returns netbox_template as string + """ + data = self.webhook + + custom_fields = data['data']['custom_fields'] + ripe_report = self.ripe_report() + ripe_template = custom_fields.get('ripe_template') + # be compatible with older netbox api + if type(ripe_template) is dict: + ripe_template = ripe_template['label'] + + if not ripe_report: + return None + + try: + netbox_template = ripe_template.upper() + except TypeError: + msg = 'No selected ripe_template in Netbox' + self.logger.error(msg) + raise MissingDataFromNetbox(msg) + + netbox_template = str(netbox_template) + self.logger.info('netbox_template: ' + netbox_template) + return netbox_template + + def ripe_report(self): + """ + Returns ripe_report as bool + """ + data = self.webhook + + custom_fields = data['data']['custom_fields'] + ripe_report = custom_fields.get('ripe_report', False) + self.logger.info(f'Report to RIPE is set: {ripe_report=}') + if ripe_report is True: + return ripe_report + else: + return False + + def country(self): + """ + Returns country as string + """ + data = self.webhook + + try: + site_slug = data['data']['site']['slug'] + country = self.country_netbox(site_slug) + except TypeError: + default_country = DEFAULT_COUNTRY.upper() + if countries_by_alpha2[default_country]: + country = default_country + else: + self.logger.error('Default country must be in iso alpha2 format') + + self.logger.info(f'Country: {str(country)}') + return country + + def org(self): + """ + returns RIPE org as string + """ + prefix = self.prefix() + org = self.org_netbox(prefix) + self.logger.info(f'Org: {str(org)}') + return org diff --git a/ripeupdater/ripe.py b/ripeupdater/ripe.py new file mode 100644 index 0000000..8e162da --- /dev/null +++ b/ripeupdater/ripe.py @@ -0,0 +1,393 @@ +# -*- coding: utf-8 -*- + +import os + +import requests +import json + +from difflib import ndiff +from ipaddress import (ip_network, ip_address, summarize_address_range) +from .exceptions import (BadRequest, ConfigError, RipeDBError) +from .functions import (validate_prefix, is_v6, notify, read_json_file, format_ripe_object, find, + format_cidr) +from .log_manager import LogManager +from .netbox import FetchData +from .configuration import * + +# Inetnum defines how Inetnum (IPv4) object look likes in the RIPE-DB +INETNUM = 'inetnum' +# Status of each object (IPv4), which has to be setten by the outgoing object query +STATUS_INETNUM = 'ASSIGNED PA' +# Inet6num defines how Inet6num (IPv6) object look likes in the RIPE-DB +INET6NUM = 'inet6num' +# Status of each object (IPv6), which has to be setten by the outgoing object query +STATUS_INET6NUM = 'ASSIGNED' +# Which headers must be used by each query to RIPE +RIPE_HEADERS = {'Content-Type': 'application/json', + 'Accept': 'application/json; charset=utf-8'} +RIPE_PARAMS = {'password': RIPE_MNT_PASSWORD} + +# The main templates file +TEMPLATES = 'templates.json' + + +class RipeObjectManager(): + def __init__(self, netbox_object, backup): + logmgr = LogManager() + self.backup = backup + self.logger = logmgr.logger + self.prefix = netbox_object.prefix() + + validate_prefix(self.prefix) + + if is_v6(self.prefix): + self.objecttype = INET6NUM + self.status = STATUS_INET6NUM + else: + self.objecttype = INETNUM + self.status = STATUS_INETNUM + + databases = { + 'RIPE': 'https://rest.db.ripe.net/ripe', + 'TEST': 'https://rest-test.db.ripe.net/test', + } + searchurls = { + 'RIPE': 'https://rest.db.ripe.net/search', + 'TEST': 'https://rest-test.db.ripe.net/search', + } + + self.baseurl = databases.get(RIPE_DB) + + if not self.baseurl: + raise ConfigError('Please set RIPE_DB to RIPE or TEST') + + self.url = f'{self.baseurl}/{self.objecttype}' + self.searchurl = searchurls.get(RIPE_DB) + self.username = netbox_object.username() + self.org = netbox_object.org() + self.netbox_template = netbox_object.netbox_template() + self.country = netbox_object.country() + + # always create a backup + self.backup_ripe_object() + + def get_old_object(self): + """ + get old object from RIPE DB and returns it as json + """ + self.logger.info(f'Getting old ripe object {self.prefix}') + response = requests.get(f'{self.url}/{self.prefix}?unfiltered', headers=RIPE_HEADERS) + + # return object if found + if response.ok: + self.logger.info(f'Getting old object has succeeded Return Code: {response.status_code}') + return response.json() + + # return None if object is not found + elif response.status_code == 404: + self.logger.info(f'Object is not existing in RIPE-DB Return Code: {response.status_code}') + return None + + else: + self.logger.error(f'Could not query old object, something went wrong! Return Code {response.status_code}') + # This raise is important to prevent the application from going further + raise BadRequest('Bad request, something went wrong!') + + def backup_ripe_object(self): + """ + save json string of an ripe object + """ + filename = f"prefix_{self.prefix.replace('/', '_')}.json" + + ripe_object = self.get_old_object() + if ripe_object: + self.logger.info(f'saving ripe object {filename}') + self.backup.put(filename, json.dumps(ripe_object)) + + def read_local_template(self): + netbox_template = self.netbox_template + file = f'{TEMPLATES_DIR}/{TEMPLATES}' + self.logger.info(f'Reading {netbox_template} in templates file: {file}') + templates = read_json_file(file) + selected_template = templates['templates'][netbox_template] + return selected_template + + def read_master_template(self): + inherit = self.read_local_template()['inherit'] + file = f'{TEMPLATES_DIR}/{inherit}' + self.logger.info(f'Reading template file: {file}') + master_attributes = read_json_file(file) + master_attributes = master_attributes['attributes'] + return master_attributes + + def overlapped_with(self): + """ + Checks if there is overlapping and return a candidate, it there is not it returns False + the overlapped object must be network prefix + """ + params = { + 'source': RIPE_DB, + 'type-filter': self.objecttype, + 'flags': 'no-referenced', + 'query-string': self.prefix + } + request = requests.get(self.searchurl, params=params, headers=RIPE_HEADERS) + + # found matching entry in RIPE DB, this could be the prefix itself or an overlapping prefix + if request.status_code == 200: + overlap = request.json()['objects']['object'][0]['primary-key']['attribute'][0]['value'] + if is_v6(self.prefix): + prefix = ip_network(overlap) + else: + cidr = overlap.split(' - ') + prefix = next(summarize_address_range(ip_address(cidr[0]), ip_address(cidr[1]))) + + if prefix != ip_network(self.prefix): + self.logger.info(f'May overlapped with: {prefix}') + return prefix + + return False + + # if no prefix is found there is no overlapping prefix + if request.status_code == 404: + self.logger.info(f'No overlapping prefix for {self.prefix} found') + return False + + # something went wrong + raise RipeDBError(f'Could not query RIPE DB for {self.prefix}: {request}') + + def generate_object(self): + """ + generates the new object for RIPE DB based on selected template + """ + # Defining list to gather attributes to prioritize the master_fields + templates_fields = [] + # Defining list to gather attributes to prioritize the dynamic generated attributes + master_fields = [] + # Defining list to gather all attributes together + all_fields = [] + + # List of attributes in template + template_attributes = self.read_local_template()['attributes'] + # List of attributes in master template + master_attributes = self.read_master_template() + + # Parsing template attributes to check which ones have a value + for t_attribute in template_attributes: + for t_name, t_value in t_attribute.items(): + if t_value: + if t_name != 'org': + templates_fields.append({t_name: t_value}) + else: + self.org = t_value + for m_attribute in master_attributes: + for m_name, m_value in m_attribute.items(): + if m_value: + master_fields.append({m_name: m_value}) + if m_name in t_attribute.keys() and m_name != 'descr': + if m_name in t_attribute.keys() and m_name != 'country': + master_fields.remove({m_name: m_value}) + if m_name == 'org': + self.org = m_value + master_fields.remove({m_name: m_value}) + + # List of dynamic generated attributes from prefix, This list is to guarantee the sequence + dynamic_attributes = [{self.objecttype: self.prefix if is_v6(self.prefix) else format_cidr(self.prefix)}, + {'netname': self.netbox_template}, + {'org': self.org}, + {'country': self.country}] + + # Gathering all templates in one list all_fields + all_fields.extend(dynamic_attributes) + all_fields.extend(templates_fields) + all_fields.extend(master_fields) + + # List for sorted fields, will be used to insert fields in specific sequence inside it + sorted_fields = [] + i_descr = 0 # Helps to sort one and many descr fields + i_country = 0 # Helps to sort one and many country fields + + for item in all_fields: + for key in item.keys(): + if RIPE_DB == 'TEST': + # patch attributes, that don't exist in TEST DB + if key == 'org': + item[key] = RIPE_TEST_ORG + if key in ['mnt-by', 'mnt-ref', 'mnt-lower', 'mnt-domains', 'mnt-routes', 'mnt-irt']: + item[key] = RIPE_TEST_MNT + if key in ['admin-c', 'tech-c', 'abuse-c']: + item[key] = RIPE_TEST_PERSON + if key == 'source': + item[key] = RIPE_DB + if key == 'status': + # override status, as parent objects with mnt-lower may not be present in TEST-DB + item[key] = RIPE_TEST_STATUS_V6 if is_v6(self.prefix) else RIPE_TEST_STATUS_V4 + self.status = RIPE_TEST_STATUS_V6 if is_v6(self.prefix) else RIPE_TEST_STATUS_V4 + + if key == 'descr': + # Sort descr fields up second place. Counting from 0 + sorted_fields.insert(i_descr + 2, item) + i_descr += 1 + elif key == 'country': + # Sort country fields up fourth place. Counting from 0 + if i_descr != 0: + sorted_fields.insert(i_country + i_descr + 4, item) + i_country += 1 + else: + sorted_fields.insert(i_country + i_descr + 4, item) + i_country += 1 + else: + sorted_fields.append(item) + + # This condition make it possible to overwrite status field + if key == 'status': + status_overwritted = sorted_fields.pop(all_fields.index(item)) + else: + status_overwritted = False + + if status_overwritted: + sorted_fields.insert(len(all_fields)-1, item) + else: + sorted_fields.insert(len(all_fields)-1, {'status': self.status}) + + obj = { + 'objects': { + 'object': [{ + 'source': {'id': RIPE_DB}, + 'attributes': { + 'attribute': [{'name': k, 'value': v} for a in sorted_fields for k, v in a.items() if v] + } + }] + } + } + + self.logger.debug(f'{obj=}') + + return obj + + def post_object(self, new_object): + # Create object + self.logger.info(f'CREATE {self.url}') + request = requests.post(self.url, json=new_object, headers=RIPE_HEADERS, params=RIPE_PARAMS) + + ripe_object, ripe_errors = self.handle_request(request) + + if request.ok: + notify(format_ripe_object(ripe_object, '+ '), request.request.method, self.prefix, self.username, + request.status_code, ripe_errors) + + return + elif request.status_code == 400: + overlapped = self.overlapped_with() + if overlapped: + netbox = FetchData() + authorize = netbox.authorize_delete_overlapped_candidate(overlapped) + if authorize: + # Saving old prefix to push after delete + cache_prefix = self.prefix + + self.prefix = overlapped + self.delete_object() + + self.prefix = cache_prefix + post = requests.post(self.url, json=new_object, headers=RIPE_HEADERS, params=RIPE_PARAMS) + ripe_object, ripe_errors = self.handle_request(post) + + if post.ok: + msg = f'I had to delete overlapped: {overlapped}' + ripe_errors = [msg] + self.logger.info(msg) + notify(format_ripe_object(ripe_object, '+ '), post.request.method, self.prefix, self.username, + post.status_code, ripe_errors) + + return + else: + ripe_errors.append(f'Overlap found for {self.prefix}: {overlapped}') + + notify(format_ripe_object(ripe_object, '+ '), request.request.method, self.prefix, self.username, + request.status_code, ripe_errors) + + msg = f'Could not create prefix {self.prefix}' + self.logger.error(msg) + raise BadRequest(msg) + + def put_object(self, old_object, new_object): + # Update object + self.logger.info(f'CREATE {self.url}') + request = requests.put(f'{self.url}/{self.prefix if is_v6(self.prefix) else format_cidr(self.prefix)}', + json=new_object, headers=RIPE_HEADERS, params=RIPE_PARAMS) + + ripe_object, ripe_errors = self.handle_request(request) + + diff = ndiff(format_ripe_object(old_object['objects']['object'][0]).splitlines(keepends=True), + format_ripe_object(ripe_object).splitlines(keepends=True)) + + if not request.ok: + msg = f'UPDATE for {self.prefix} failed: {request=} {ripe_errors=}' + self.logger.error(msg) + raise BadRequest(msg) + + notify(''.join(diff), request.request.method, self.prefix, self.username, + request.status_code, ripe_errors) + + def push_object(self): + """ + entry point if report_ripe is set to true + determines if post (create) or put (update) should be executed + """ + old_object = self.get_old_object() + new_object = self.generate_object() + self.logger.debug(f'{old_object=}') + self.logger.debug(f'{new_object=}') + + # if old object exists run update, otherwise create + if old_object: + self.put_object(old_object, new_object) + else: + self.post_object(new_object) + + def delete_object(self): + """ + delete object from RIPE DB + """ + self.logger.info(f'DELETE {self.url}') + request = requests.delete(f'{self.url}/{self.prefix}', headers=RIPE_HEADERS, params=RIPE_PARAMS) + + ripe_object, ripe_errors = self.handle_request(request) + + if not request.ok: + msg = f'DELETE for {self.prefix} failed: {request=} {ripe_errors=}' + self.logger.error(msg) + + # if object is already delete, ok else raise exception + if request.status_code != 404: + raise BadRequest(msg) + + notify(format_ripe_object(ripe_object, '-'), request.request.method, self.prefix, self.username, + request.status_code, ripe_errors) + + def handle_request(self, request): + self.logger.debug(request) + response = request.json() + self.logger.debug(response) + + ripe_objects = find('objects.object', response) + self.logger.debug(f'{ripe_objects=}') + ripe_errormessages = find('errormessages.errormessage', response) + self.logger.debug(f'{ripe_errormessages=}') + + ripe_object = {} + ripe_errors = [] + + if ripe_objects: + ripe_object = ripe_objects[0] + + if ripe_errormessages: + ripe_errors = [msg.get('text') for msg in ripe_errormessages] + + if request.ok: + self.logger.info(f'{request.request.method} {self.prefix} succeeded') + else: + self.logger.error(f'{request.request.method} {self.prefix} failed') + + return ripe_object, ripe_errors diff --git a/ripeupdater/templates/backups.html b/ripeupdater/templates/backups.html new file mode 100644 index 0000000..b919f5f --- /dev/null +++ b/ripeupdater/templates/backups.html @@ -0,0 +1,16 @@ + + + + ripe-updater backups + + +

Backups

+ + + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e47b739 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +name = ripe-updater + +[options] +packages = find: \ No newline at end of file diff --git a/templates/base_mycompany.example.json b/templates/base_mycompany.example.json new file mode 100644 index 0000000..6e7ce1c --- /dev/null +++ b/templates/base_mycompany.example.json @@ -0,0 +1,9 @@ +{ "attributes": [ + {"org": ""}, + {"remarks": "Managed by ripe-updater"}, + {"admin-c": "AA1-TEST"}, + {"tech-c": "AA1-TEST"}, + {"notify": "noc@example.com"}, + {"mnt-by": "TEST-DBM-MNT"}, + {"source": "TEST"} +]} \ No newline at end of file diff --git a/templates/base_mycustomer1.example.json b/templates/base_mycustomer1.example.json new file mode 100644 index 0000000..a7b338a --- /dev/null +++ b/templates/base_mycustomer1.example.json @@ -0,0 +1,10 @@ +{ "attributes": [ + {"remarks": "Managed by ripe-updater"}, + {"org": "ORG-EIPB1-TEST"}, + {"admin-c": "AA2-TEST"}, + {"tech-c": "AA2-TEST"}, + {"abuse-c": "AA1-TEST"}, + {"notify": "noc@example.com"}, + {"mnt-by": "TEST-NCC-HM-MNT"}, + {"source": "TEST"} +]} \ No newline at end of file diff --git a/templates/lir_org.example.json b/templates/lir_org.example.json new file mode 100644 index 0000000..43a3f3c --- /dev/null +++ b/templates/lir_org.example.json @@ -0,0 +1,8 @@ +{ + "templates": { + "lir_org": { + "de.examplelir1": "ORG-EIPB1-TEST", + "nl.examplelir2": "ORG-TT1-TEST" + } + } +} diff --git a/templates/templates.example.json b/templates/templates.example.json new file mode 100644 index 0000000..bcfc51b --- /dev/null +++ b/templates/templates.example.json @@ -0,0 +1,25 @@ +{ + "templates": + { + "CLOUD-POOL": {"attributes": [ + {"descr": "MyCompany Cloud Pool"} + ], + "inherit": "base_mycompany.example.json" + }, + "INFRA-TRANSFER-NET": {"attributes": [ + {"descr": "MyCompany Infrastructure Transfer Network"} + ], + "inherit": "base_mycompany.example.json" + }, + "CUST-ACCESS-NET": {"attributes": [ + {"descr": "MyCompany Customer Access"} + ], + "inherit": "base_mycompany.example.json" + }, + "CUST-ACCESS-NET-MYCUSTOMER1": {"attributes": [ + {"descr": "MyCustomer 1 Ltd., Example Street 33"} + ], + "inherit": "base_mycustomer1.example.json" + } + } +} diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..61d8079 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,4 @@ +pytest +tox +pytest-cov +requests-mock diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/base_mycompany.json b/tests/base_mycompany.json new file mode 100644 index 0000000..5bcd951 --- /dev/null +++ b/tests/base_mycompany.json @@ -0,0 +1,9 @@ +{ "attributes": [ + {"org": ""}, + {"remarks": "Managed by ripeupdater"}, + {"admin-c": "AA1-TEST"}, + {"tech-c": "AA1-TEST"}, + {"notify": "noc@example.com"}, + {"mnt-by": "TEST-DBM-MNT"}, + {"source": "TEST"} +]} \ No newline at end of file diff --git a/tests/example.json b/tests/example.json new file mode 100644 index 0000000..1d58328 --- /dev/null +++ b/tests/example.json @@ -0,0 +1,25 @@ +{ + "templates": + { + "CLOUD-POOL": {"attributes": [ + {"descr": "MyCompany Cloud Pool"} + ], + "inherit": "base_mycompany.json" + }, + "INFRA-TRANSFER-NET": {"attributes": [ + {"descr": "MyCompany Infrastructure Transfer Network"} + ], + "inherit": "base_mycompany.example.json" + }, + "CUST-ACCESS-NET": {"attributes": [ + {"descr": "MyCompany Customer Access"} + ], + "inherit": "base_mycompany.example.json" + }, + "CUST-ACCESS-NET-MYCUSTOMER1": {"attributes": [ + {"descr": "MyCustomer 1 Ltd., Example Street 33"} + ], + "inherit": "base_mycustomer1.example.json" + } + } +} diff --git a/tests/lir_org.json b/tests/lir_org.json new file mode 100644 index 0000000..9201bda --- /dev/null +++ b/tests/lir_org.json @@ -0,0 +1,8 @@ +{ + "templates": { + "lir_org": { + "de.examplelir1": "ORG-EIPB1-TEST", + "nl.examplelir2": "ORG-TT1-TEST" + } + } +} diff --git a/tests/test_functions.py b/tests/test_functions.py new file mode 100644 index 0000000..e8b8ecf --- /dev/null +++ b/tests/test_functions.py @@ -0,0 +1,66 @@ +from pytest import raises + +from ripeupdater.functions import * +from ripeupdater.exceptions import * + + +def test_read_json_file(): + d = read_json_file("tests/example.json") + assert type(d) is dict + + +def test_flatten_ripe_object(): + obj = {"type": "inetnum","link": {"type": "locator"},"source": {"id": "ripe"},"attributes": {"attribute": [{"name": "country","value": "DE"},{"name": "remarks","value": "Managed by ripeupdater"},{"name": "status","value": "ASSIGNED PA"}]}} + attr = flatten_ripe_attributes(obj) + + assert attr["country"] == "DE" + + +def test_format_ripe_object(): + obj = {"type": "inetnum","link": {"type": "locator"},"source": {"id": "ripe"},"attributes": {"attribute": [{"name": "country","value": "DE"},{"name": "remarks","value": "Managed by ripeupdater"},{"name": "status","value": "ASSIGNED PA"}]}} + string = format_ripe_object(obj, "+") + + assert string.find("+status\t\tASSIGNED PA") + + +def test_is_v6(): + assert is_v6("2001:db8:f::/48") + assert not is_v6("198.51.100.0/24") + + +def test_validate_prefix(): + with raises(ErrorSmallPrefix) as execinfo: + validate_prefix("2001:db8::/128") + assert "too small" in str(execinfo.value) + + with raises(NotRoutedNetwork) as execinfo: + validate_prefix("fe80::/64") + assert "not routed" in str(execinfo.value) + + with raises(ErrorSmallPrefix) as execinfo: + validate_prefix("127.0.0.0/32") + assert "too small" in str(execinfo.value) + + with raises(NotRoutedNetwork) as execinfo: + validate_prefix("127.0.0.0/8") + assert "not routed" in str(execinfo.value) + + with raises(NotRoutedNetwork) as execinfo: + validate_prefix("172.16.0.0/12") + assert "not routed" in str(execinfo.value) + + assert validate_prefix("2001:1234:4567::/64") + assert validate_prefix("1.0.0.0/24") + + +def test_format_cidr(): + assert "198.51.100.0 - 198.51.100.255" == format_cidr("198.51.100.0/24") + + +def test_notify(): + obj = {"type": "inetnum","link": {"type": "locator"},"source": {"id": "ripe"},"attributes": {"attribute": [{"name": "country","value": "DE"},{"name": "remarks","value": "Managed by ripeupdater"},{"name": "status","value": "ASSIGNED PA"}]}} + notify(obj, "POST", "198.51.100.0/24", "testuser", 200, []) + + +def test_find(): + assert find("elem1.elem2", {"elem1": {"elem2": "foo"}}) == "foo" \ No newline at end of file diff --git a/tests/test_netbox.py b/tests/test_netbox.py new file mode 100644 index 0000000..0daa0a6 --- /dev/null +++ b/tests/test_netbox.py @@ -0,0 +1,45 @@ +import os + +from unittest.mock import patch, Mock + +from ripeupdater.netbox import ObjectBuilder + +_dir_path = os.path.dirname(os.path.realpath(__file__)) + +@patch("pynetbox.api") +def test_prefix(netbox_api): + webhook = { + "data": { + "prefix": "2001:1234:4567::/64" + } + } + netbox_object = ObjectBuilder(webhook) + assert netbox_object.prefix() == "2001:1234:4567::/64" + + +@patch("pynetbox.api") +@patch("ripeupdater.netbox.TEMPLATES_DIR", f"{_dir_path}/") +def test_org(netbox_api): + webhook = { + "data": { + "prefix": "2001:1234:4567::/64" + } + } + netbox_api.return_value.ipam.aggregates.get.return_value = Mock(custom_fields={"lir": "de.examplelir1"}) + netbox_object = ObjectBuilder(webhook) + assert netbox_object.org() == "ORG-EIPB1-TEST"\ + + +@patch("pynetbox.api") +@patch("ripeupdater.netbox.TEMPLATES_DIR", f"{_dir_path}/") +def test_country(netbox_api): + webhook = { + "data": { + "site": { + "slug": "myslug" + } + } + } + netbox_api.return_value.dcim.regions.get.return_value = Mock(slug="germany") + netbox_object = ObjectBuilder(webhook) + assert netbox_object.country() == "DE" \ No newline at end of file diff --git a/tests/test_ripe.py b/tests/test_ripe.py new file mode 100644 index 0000000..dc700d4 --- /dev/null +++ b/tests/test_ripe.py @@ -0,0 +1,95 @@ +import os +from unittest.mock import patch, Mock +import requests_mock + +from ripeupdater.backup_manager import BackupManager +from ripeupdater.netbox import ObjectBuilder +from ripeupdater.ripe import RipeObjectManager + +_dir_path = os.path.dirname(os.path.realpath(__file__)) + + +@patch("pynetbox.api") +@patch("ripeupdater.netbox.TEMPLATES_DIR", f"{_dir_path}/") +@patch("ripeupdater.ripe.TEMPLATES_DIR", f"{_dir_path}/") +@patch("ripeupdater.ripe.TEMPLATES", f"example.json") +def test_ripe(netbox_api): + # putting it all together. + webhook = { + "data": { + "prefix": "2001:1234:4567::/64", + "site": { + "slug": "myslug" + }, + "custom_fields": { + "ripe_report": True, + "ripe_template": "CLOUD-POOL", + } + }, + "username": "username", + } + netbox_api.return_value.ipam.aggregates.get.return_value = Mock(custom_fields={"lir": "de.examplelir1"}) + netbox_api.return_value.dcim.regions.get.return_value = Mock(slug="germany") + netbox_object = ObjectBuilder(webhook) + assert netbox_object.org() == "ORG-EIPB1-TEST" + assert netbox_object.country() == "DE" + + with requests_mock.Mocker() as m: + old_object = { + "objects": { + "object": [ + { + "attributes": { + "attribute": { + + } + } + } + ] + } + } + m.get("https://rest-test.db.ripe.net/test/inet6num/2001:1234:4567::/64?unfiltered", json=old_object) + ripe = RipeObjectManager(netbox_object, BackupManager()) + o = ripe.get_old_object() + assert o == old_object + + put_out = { + "objects": { + "object": [ + { + "attributes": { + "attribute": { + } + } + } + ] + } + } + m.put("https://rest-test.db.ripe.net/test/inet6num/2001:1234:4567::/64", json=put_out) + ripe.push_object() + assert m.last_request.json() == { + 'objects': { + 'object': [ + { + 'source': {'id': 'TEST'}, + 'attributes': { + 'attribute': [ + {'name': 'inet6num', 'value': '2001:1234:4567::/64'}, + {'name': 'netname', 'value': 'CLOUD-POOL'}, + {'name': 'descr', 'value': 'MyCompany Cloud Pool'}, + {'name': 'org', 'value': 'ORG-EIPB1-TEST'}, + {'name': 'country', 'value': 'DE'}, + {'name': 'remarks', 'value': 'Managed by ripeupdater'}, + {'name': 'admin-c', 'value': 'AA1-TEST'}, + {'name': 'tech-c', 'value': 'AA1-TEST'}, + {'name': 'notify', 'value': 'noc@example.com'}, + {'name': 'mnt-by', 'value': 'TEST-DBM-MNT'}, + {'name': 'status', 'value': 'ALLOCATED PA'}, + {'name': 'source', 'value': 'TEST'} + ] + } + } + ] + } + } + diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..4d79e17 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py39, +isolated_build = True + +[base] + +[testenv] +deps = + -rrequirements.txt + -rtest-requirements.txt +commands = pytest --cov=ripeupdater