diff --git a/README.md b/README.md index 05e37e0..7bbee16 100644 --- a/README.md +++ b/README.md @@ -64,69 +64,99 @@ module][nginx limit req]. Running server (without installing as an debian package) -------------- ``` +# install MongoDB and Pymongo client +sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 +echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.6.list sudo apt-get update -sudo apt-get install git python-pip python-flask python-sqlalchemy sqlite3 python-yaml +sudo apt-get install -y mongodb-org +python -m pip install pymongo + +sudo apt-get install git python-pip python-flask pymongo python-yaml git clone https://github.com/ipop-project/ipop-stats.git -cd ipop-stats/ipopstat-0.12/DEBIAN +cd ipop-stats/ipopusagereport-0.1/DEBIAN sudo bash preinst -sudo passwd ipop +sudo passwd ipopreport Enter new UNIX password: ipop Retype new UNIX password: ipop passwd: password updated successfully -su ipop +sudo service mongod start +su ipopreport Password:ipop -cd ../usr/share/ipop-stat +cd ../usr/share/ipop-usagereport ./run.py ``` now you can access this server through webbrowser. -http://ip_address:8080/api +http://ip_address:8081/api controller reports status info -http://ip_address:8080/api/submit +http://ip_address:8081/api/submit -sqlite database file is located /var/lib/ipop-stat/ipop-stat.db -Simple database query snippets below. +database location: ``` -$sqlite3 /var/lib/ipop-stat/ipop-stat.db -sqlite>select * from user; -sqlite>select * from ping; +/var/lib/mongodb/ ``` -Building Debian Packages ------------------------- - - -Locate at the parent directory of ipop-stat then run below. +you can query database using Mongo shell as: +``` +$ mongo +> use usage_report +> db.user.find() # to get the data in database +> db.user.count() # to get number of documents in collection +``` +Reset Database: ``` -dpkg-deb --build ipopstat-0.14 +$ mongo +> show dbs +> use usage_report +> db.dropDatabase() ``` +Building Debian Package +------------------------------------------------ +locate at the parent directory of ipopusagereport-0.1 +``` +$ cd ipopusagereport-0.1 +$ dpkg-deb --build ipopusagereport-0.1 +``` +Installation and Runnnig ipopusagereport debian package +------------------------------------------------ +``` +sudo dpkg -i ipopusagereport-0.1.deb +sudo apt-get update +sudo apt-get -f install +``` -Install and Running ipopstat from Debian Package +Start, Stop, Restart ipopusagereport service ------------------------------------------------ +Start service: (make sure mongodb server is running before starting ipopusagereport) +``` +$ service mongod start +$ service ipop-usagereport start ``` -sudo dpkg -i ipopstat-0.14.deb -sudo apt-get update -sudo apt-get -f install + +Stop service: +``` +$ service ipop-usagereport stop ``` +Restart Service: +``` +$ service ipop-usagereport restart +``` +Check status: +``` +$ service ipop-usagereport status +``` -Note for next time ------------------------------------------------- -I need to stop the ipop-stat before update. -Add the command in preinst or something like "service ipop-stat stop" -Issues ------------------------------------------------- -For some reason, "service ipop-stat start/stop" does not work. diff --git a/client.py b/client.py index 444d332..581d492 100755 --- a/client.py +++ b/client.py @@ -1,16 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env import datetime import json import hashlib -import urllib2 +import urllib.request as urllib2 +import urllib.parse -data = json.dumps({ - "xmpp_host" : hashlib.sha1("127.0.0.1").hexdigest(), - "uid": hashlib.sha1("abcd").hexdigest(), - "xmpp_username":hashlib.sha1("username").hexdigest(), +report_data = { + "xmpp_host": hashlib.sha1("127.0.0.1".encode("utf-8")).hexdigest(), + "xmpp_username": hashlib.sha1("xmpp_username".encode("utf-8")).hexdigest(), "time": str(datetime.datetime.now()), "controller": "test_client", - "version": 3}) + "version": 3} + +data = json.dumps(report_data).encode('utf8') try: url="http://" + "127.0.0.1" + ":" +str(8080) + "/api/submit" @@ -22,6 +24,6 @@ res.read())) if res.getcode() != 200: raise -except: - print("Status report failed.") +except Exception as error: + print("Status report failed. Error is {0}".format(error)) diff --git a/ipopstat-0.15/DEBIAN/control b/ipopstat-0.15/DEBIAN/control new file mode 100644 index 0000000..eac5eec --- /dev/null +++ b/ipopstat-0.15/DEBIAN/control @@ -0,0 +1,9 @@ +Package: ipop-stat +Source: linux +Version: 0.14 +Architecture: amd64 +Maintainer: Kyuho Jeong +Installed-Size: 41150 +Pre-Depends: dpkg (>= 1.10.24) +Depends: python-pip, python-flask, python-sqlalchemy, sqlite3, python-yaml, python-tornado +Description: Humble stat server diff --git a/ipopstat-0.15/DEBIAN/postinst b/ipopstat-0.15/DEBIAN/postinst new file mode 100755 index 0000000..ab74539 --- /dev/null +++ b/ipopstat-0.15/DEBIAN/postinst @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +service ipop-stat start +update-rc.d ipop-stat defaults diff --git a/ipopstat-0.15/DEBIAN/preinst b/ipopstat-0.15/DEBIAN/preinst new file mode 100755 index 0000000..ea9b334 --- /dev/null +++ b/ipopstat-0.15/DEBIAN/preinst @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +mkdir -p /var/lib/ipop-stat +touch /var/lib/ipop-stat/ipop-stat.db +touch /var/log/ipop-stat.log +useradd -d /var/lib/ipop-stat ipop +chown --quiet ipop:ipop /var/lib/ipop-stat +chown --quiet ipop:ipop /var/lib/ipop-stat/ipop-stat.db +chown --quiet ipop:ipop /var/log/ipop-stat.log diff --git a/ipopstat-0.15/etc/init.d/ipop-stat b/ipopstat-0.15/etc/init.d/ipop-stat new file mode 100755 index 0000000..70b9afd --- /dev/null +++ b/ipopstat-0.15/etc/init.d/ipop-stat @@ -0,0 +1,134 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: ipop-stat service +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: ipop stat +# Description: ipop stat +### END INIT INFO + +# Author: Kyuho Jeong + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="ipop stat" +NAME=ipop-stat +DAEMON=/usr/share/$NAME/run.py +DAEMON_ARGS="-c /usr/share/$NAME/config/debug.yml" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +. /lib/lsb/init-functions + +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile --chuid ipop $PIDFILE --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --chuid ipop --make-pidfile --pidfile $PIDFILE --background \ + --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS > /dev/null" \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Wait for children to finish too if this is a daemon that forks + # and if the daemon is only ever run from this initscript. + # If the above conditions are not satisfied then add some other code + # that waits for the process to drop all resources that could be + # needed by services started subsequently. A last resort is to + # sleep for some time. + start-stop-daemon --stop --quiet --chuid ipop --oknodo --retry=0/30/KILL/5 --exec $DAEMON + [ "$?" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --chuid ipop --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/ipopstat-0.15/usr/share/ipop-stat/MANIFEST.in b/ipopstat-0.15/usr/share/ipop-stat/MANIFEST.in new file mode 100644 index 0000000..0d8e19d --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include ipop_stats/templates * +recursive-include ipop_stats/static * diff --git a/ipopstat-0.15/usr/share/ipop-stat/config/config.yml b/ipopstat-0.15/usr/share/ipop-stat/config/config.yml new file mode 100644 index 0000000..f32ffed --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/config/config.yml @@ -0,0 +1,32 @@ +# http://flask.pocoo.org/docs/config/ +database_name: "usage_report" +database_uri: "mongodb://localhost:27017" +new_database: False +flask: + debug: true + testing: true +# http://docs.python.org/2/library/logging.config.html#logging-config-dictschema +logging: + version: 1 + root: + level: DEBUG + handlers: + console: + class: logging.RotatingFileHandler + stream: ext://sys.stdout + formatter: default + file: + class : logging.handlers.RotatingFileHandler + formatter: default + filename: /var/log/ipop-stat.log + maxBytes: 99999999 + backupCount: 9 + formatters: + default: + format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' +ratelimit: + burst: 10 # allow a burst "pool" + uuid: 60 # seconds + ipv4: 60 # seconds + ipv6: 60 # seconds diff --git a/ipopstat-0.15/usr/share/ipop-stat/config/debug.yml b/ipopstat-0.15/usr/share/ipop-stat/config/debug.yml new file mode 100644 index 0000000..7677d6c --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/config/debug.yml @@ -0,0 +1,36 @@ +# http://flask.pocoo.org/docs/config/ +database_name: "usage_report" +database_uri: "mongodb://localhost:27017" +new_database: False +flask: + debug: true + testing: true +# http://docs.python.org/2/library/logging.config.html#logging-config-dictschema +logging: + version: 1 + root: + level: DEBUG + handlers: + console: + class: logging.StreamHandler + stream: ext://sys.stdout + formatter: default + file: + class : logging.handlers.RotatingFileHandler + formatter: default + filename: /var/log/ipop-stat.log + maxBytes: 9999999 + backupCount: 9 + formatters: + default: + format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + loggers: + all: + level: DEBUG + handlers: [console, file] +ratelimit: + burst: 10 # allow a burst "pool" + uuid: 60 # seconds + ipv4: 60 # seconds + ipv6: 60 # seconds diff --git a/ipopstat-0.15/usr/share/ipop-stat/flask_run.py b/ipopstat-0.15/usr/share/ipop-stat/flask_run.py new file mode 100755 index 0000000..6e3b9c3 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/flask_run.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from ipop_stats.app import create as create_app +from flask import Flask +import argparse +import os, os.path + +parser = argparse.ArgumentParser(description="Standalone test server for " + "ipop-stats") + +try: + default_config_path = os.path.abspath(os.environ["IPOP_STATS_SETTINGS"]) +except: + default_config_path = "config/debug.yml" + +parser.add_argument( + "-c", "--config", + default=default_config_path, + type=os.path.abspath, + help="the flask config file for the server" +) + +args = parser.parse_args() +create_app(**vars(args)).run(host="0.0.0.0", port=8080) diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/__init__.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/__init__.py @@ -0,0 +1 @@ +pass diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/app.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/app.py new file mode 100644 index 0000000..d43096f --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/app.py @@ -0,0 +1,37 @@ +from flask import Flask +import yaml +import os.path +import logging.config +from database import Database + +def create(config): + """http://flask.pocoo.org/docs/patterns/appfactories/""" + + app = Flask(__name__.split('.')[0]) + + # Load YAML config into app.config + if not isinstance(config, dict): + with open(os.path.join(app.instance_path, config)) as f: + config = yaml.load(f) + config.update({k.upper(): v for k, v in config["flask"].items()}) + app.config.update(config) + del config + + # Configure logging + logging.config.dictConfig(app.config["logging"]) + logger = logging.getLogger('all') + logger.info("BUILT LOGGER. Logging started") + + # Initialize database + app.database = Database(app) + + # Load blueprints + from .views.submit import submit + app.register_blueprint(submit) + + if False and app.debug: + app.logger.warning("Debug mode is on. Do not use this in production.") + from .views.debug import debug + app.register_blueprint(debug) + + return app diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/database.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/database.py new file mode 100644 index 0000000..0a8d636 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/database.py @@ -0,0 +1,18 @@ + +import logging +import uuid +import functools +from contextlib import contextmanager + +from pymongo import MongoClient + +class Database(object): + def __init__(self, app): + self.app = app + + self.stat_mongo = MongoClient(self.app.config["database_uri"]) + logger = logging.getLogger('all') + logger.info("MongoDB Server is running at :{0}".format(self.app.config["database_uri"])) + self._db = self.stat_mongo[self.app.config["database_name"]] + self.user_collection = self._db["user"] + diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/errors.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/errors.py new file mode 100644 index 0000000..54ed3e8 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/errors.py @@ -0,0 +1,33 @@ +from flask import jsonify +from werkzeug.exceptions import HTTPException + +# We follow the JSend spec for API errors: http://labs.omniti.com/labs/jsend + +class ApiFail(HTTPException): + """There was a problem with the data submitted, or some pre-condition of the + API call wasn't satisfied""" + def __init__(message=None, data=None, code=400, **kwargs): + data = data or kwargs + if instanceof(message, str): + data, message = message, None + if message: + data["message"] = message + + response = jsonify(status="fail", data=data) + response.status_code = status_code + HTTPException.__init__(self, message, response) + self.code = code + +class ApiError(HTTPException): + "An error occurred in processing the request, i.e. an exception was thrown" + def __init__(message, code=500, data=None, **kwargs): + data = data or kwargs + response_dict = {"status": "error", "message": message} + if code != 500: + response_dict["code"] = code + if data: + response_dict["data"] = data + response = jsonify(response_dict) + response.status_code = status_code + HTTPException.__init__(self, message, response) + self.code = code diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/__init__.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/__init__.py @@ -0,0 +1 @@ +pass diff --git a/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/submit.py b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/submit.py new file mode 100644 index 0000000..d7e3e6e --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/ipop_stats/views/submit.py @@ -0,0 +1,35 @@ +from ..errors import * +from flask import Blueprint, request, jsonify +from flask import current_app as app +import datetime +import socket +import uuid +import logging + +from pymongo import MongoClient + + +submit = Blueprint("submit", __name__, url_prefix="/api") + +@submit.route("/submit", methods=["POST"]) +def update(): + logger = logging.getLogger('all') + logger.info("Report Message Received") + data = request.json + report = {"Report": data} + obj_id = app.database.user_collection.insert_one(report) + response = jsonify(result=uuid.uuid4().hex, status="success") # TODO: what is result? + return response + +@submit.route("/") +def hello(): + return "IPOP-Usage Stat Server!" + +@submit.route("/generate_uuid") +def generate_uuid(): + """Convenience uuid generator, for clients who don't have a convenient local + generator.""" + response = jsonify(result=uuid.uuid4().hex, status="success") + response.headers["Expires"] = "-1" + response.headers["Cache-Control"] = "no-cache, no-store" + return response diff --git a/ipopstat-0.15/usr/share/ipop-stat/run.py b/ipopstat-0.15/usr/share/ipop-stat/run.py new file mode 100755 index 0000000..695e10c --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/run.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +from flask import Flask +from tornado.wsgi import WSGIContainer +from tornado.httpserver import HTTPServer +from tornado.ioloop import IOLoop +from ipop_stats.app import create as create_app +import argparse +import os, os.path + + +parser = argparse.ArgumentParser(description="Standalone test server for " + "ipop-stats") + +try: + default_config_path = os.path.abspath(os.environ["IPOP_STATS_SETTINGS"]) +except: + default_config_path = "config/debug.yml" + +parser.add_argument( + "-c", "--config", + default=default_config_path, + type=os.path.abspath, + help="the flask config file for the server" +) + +# Create Flask APP +args = parser.parse_args() +app = create_app(**vars(args)) #run(host="0.0.0.0", port=8080) + +# Run with Tonado +http_server = HTTPServer(WSGIContainer(app)) +http_server.listen(8080) +IOLoop.instance().start() + + diff --git a/ipopstat-0.15/usr/share/ipop-stat/setup.py b/ipopstat-0.15/usr/share/ipop-stat/setup.py new file mode 100644 index 0000000..de3b226 --- /dev/null +++ b/ipopstat-0.15/usr/share/ipop-stat/setup.py @@ -0,0 +1,14 @@ +from setuptools import * + +setup( + name="ipop-stats", + version="0.0.1", + install_requires=[ + "flask>=0.10.1,<0.11", + "pymongo>=3.6,<4", + "PyYAML>=3.10,<4.0", + ], + description="Gathers anonymous usage statistics of IPOP users", + packages=find_packages(), + scripts=["bin/ipop-stats"], +) diff --git a/ipopusagereport-0.1/DEBIAN/control b/ipopusagereport-0.1/DEBIAN/control new file mode 100644 index 0000000..ae1f468 --- /dev/null +++ b/ipopusagereport-0.1/DEBIAN/control @@ -0,0 +1,9 @@ +Package: ipop-usagereport +Source: linux +Version: 0.1 +Architecture: amd64 +Maintainer: Gaurav Yeole +Installed-Size: 41150 +Pre-Depends: dpkg (>= 1.10.24) +Depends: python-pip, python-flask, python-yaml, python-tornado +Description: IPOP Usage Report Server diff --git a/ipopusagereport-0.1/DEBIAN/postinst b/ipopusagereport-0.1/DEBIAN/postinst new file mode 100755 index 0000000..2ce1ea2 --- /dev/null +++ b/ipopusagereport-0.1/DEBIAN/postinst @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +update-rc.d ipop-usagereport defaults +service ipop-usagereport start + diff --git a/ipopusagereport-0.1/DEBIAN/preinst b/ipopusagereport-0.1/DEBIAN/preinst new file mode 100755 index 0000000..3c58c3d --- /dev/null +++ b/ipopusagereport-0.1/DEBIAN/preinst @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +touch /var/log/ipop-usagereport.log +useradd -d /var/lib/ipop-usagereport ipopreport +chown --quiet ipopreport:ipopreport /var/log/ipop-usagereport.log diff --git a/ipopusagereport-0.1/etc/init.d/ipop-usagereport b/ipopusagereport-0.1/etc/init.d/ipop-usagereport new file mode 100755 index 0000000..0cf97a7 --- /dev/null +++ b/ipopusagereport-0.1/etc/init.d/ipop-usagereport @@ -0,0 +1,121 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: ipop-usagereport +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: ipop usage report +# Description: ipop usage report +### END INIT INFO + +# Author: Gaurav Yeole + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="ipop usage report" +NAME=ipop-usagereport +DAEMON=/usr/share/$NAME/run.py +DAEMON_ARGS="-c /usr/share/$NAME/config/debug.yml" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +. /lib/lsb/init-functions + +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --pidfile --chuid ipopreport $PIDFILE --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --chuid ipopreport --make-pidfile --pidfile $PIDFILE --background \ + --startas /bin/bash -- -c "exec $DAEMON $DAEMON_ARGS > /dev/null" \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --chuid ipopreport --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/MANIFEST.in b/ipopusagereport-0.1/usr/share/ipop-usagereport/MANIFEST.in new file mode 100644 index 0000000..0d8e19d --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include ipop_stats/templates * +recursive-include ipop_stats/static * diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/config/config.yml b/ipopusagereport-0.1/usr/share/ipop-usagereport/config/config.yml new file mode 100644 index 0000000..d0a18db --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/config/config.yml @@ -0,0 +1,32 @@ +# http://flask.pocoo.org/docs/config/ +database_name: "usage_report" +database_uri: "mongodb://localhost:27017" +new_database: False +flask: + debug: true + testing: true +# http://docs.python.org/2/library/logging.config.html#logging-config-dictschema +logging: + version: 1 + root: + level: DEBUG + handlers: + console: + class: logging.RotatingFileHandler + stream: ext://sys.stdout + formatter: default + file: + class : logging.handlers.RotatingFileHandler + formatter: default + filename: /var/log/ipop-usagereport.log + maxBytes: 99999999 + backupCount: 9 + formatters: + default: + format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' +ratelimit: + burst: 10 # allow a burst "pool" + uuid: 60 # seconds + ipv4: 60 # seconds + ipv6: 60 # seconds diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/config/debug.yml b/ipopusagereport-0.1/usr/share/ipop-usagereport/config/debug.yml new file mode 100644 index 0000000..ce025ba --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/config/debug.yml @@ -0,0 +1,36 @@ +# http://flask.pocoo.org/docs/config/ +database_name: "usage_report" +database_uri: "mongodb://localhost:27017" +new_database: False +flask: + debug: true + testing: true +# http://docs.python.org/2/library/logging.config.html#logging-config-dictschema +logging: + version: 1 + root: + level: DEBUG + handlers: + console: + class: logging.StreamHandler + stream: ext://sys.stdout + formatter: default + file: + class : logging.handlers.RotatingFileHandler + formatter: default + filename: /var/log/ipop-usagereport.log + maxBytes: 9999999 + backupCount: 9 + formatters: + default: + format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + loggers: + all: + level: DEBUG + handlers: [console, file] +ratelimit: + burst: 10 # allow a burst "pool" + uuid: 60 # seconds + ipv4: 60 # seconds + ipv6: 60 # seconds diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/flask_run.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/flask_run.py new file mode 100755 index 0000000..6e3b9c3 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/flask_run.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from ipop_stats.app import create as create_app +from flask import Flask +import argparse +import os, os.path + +parser = argparse.ArgumentParser(description="Standalone test server for " + "ipop-stats") + +try: + default_config_path = os.path.abspath(os.environ["IPOP_STATS_SETTINGS"]) +except: + default_config_path = "config/debug.yml" + +parser.add_argument( + "-c", "--config", + default=default_config_path, + type=os.path.abspath, + help="the flask config file for the server" +) + +args = parser.parse_args() +create_app(**vars(args)).run(host="0.0.0.0", port=8080) diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/__init__.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/__init__.py @@ -0,0 +1 @@ +pass diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/app.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/app.py new file mode 100644 index 0000000..d43096f --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/app.py @@ -0,0 +1,37 @@ +from flask import Flask +import yaml +import os.path +import logging.config +from database import Database + +def create(config): + """http://flask.pocoo.org/docs/patterns/appfactories/""" + + app = Flask(__name__.split('.')[0]) + + # Load YAML config into app.config + if not isinstance(config, dict): + with open(os.path.join(app.instance_path, config)) as f: + config = yaml.load(f) + config.update({k.upper(): v for k, v in config["flask"].items()}) + app.config.update(config) + del config + + # Configure logging + logging.config.dictConfig(app.config["logging"]) + logger = logging.getLogger('all') + logger.info("BUILT LOGGER. Logging started") + + # Initialize database + app.database = Database(app) + + # Load blueprints + from .views.submit import submit + app.register_blueprint(submit) + + if False and app.debug: + app.logger.warning("Debug mode is on. Do not use this in production.") + from .views.debug import debug + app.register_blueprint(debug) + + return app diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/database.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/database.py new file mode 100644 index 0000000..0a8d636 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/database.py @@ -0,0 +1,18 @@ + +import logging +import uuid +import functools +from contextlib import contextmanager + +from pymongo import MongoClient + +class Database(object): + def __init__(self, app): + self.app = app + + self.stat_mongo = MongoClient(self.app.config["database_uri"]) + logger = logging.getLogger('all') + logger.info("MongoDB Server is running at :{0}".format(self.app.config["database_uri"])) + self._db = self.stat_mongo[self.app.config["database_name"]] + self.user_collection = self._db["user"] + diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/errors.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/errors.py new file mode 100644 index 0000000..54ed3e8 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/errors.py @@ -0,0 +1,33 @@ +from flask import jsonify +from werkzeug.exceptions import HTTPException + +# We follow the JSend spec for API errors: http://labs.omniti.com/labs/jsend + +class ApiFail(HTTPException): + """There was a problem with the data submitted, or some pre-condition of the + API call wasn't satisfied""" + def __init__(message=None, data=None, code=400, **kwargs): + data = data or kwargs + if instanceof(message, str): + data, message = message, None + if message: + data["message"] = message + + response = jsonify(status="fail", data=data) + response.status_code = status_code + HTTPException.__init__(self, message, response) + self.code = code + +class ApiError(HTTPException): + "An error occurred in processing the request, i.e. an exception was thrown" + def __init__(message, code=500, data=None, **kwargs): + data = data or kwargs + response_dict = {"status": "error", "message": message} + if code != 500: + response_dict["code"] = code + if data: + response_dict["data"] = data + response = jsonify(response_dict) + response.status_code = status_code + HTTPException.__init__(self, message, response) + self.code = code diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/__init__.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/__init__.py new file mode 100644 index 0000000..2ae2839 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/__init__.py @@ -0,0 +1 @@ +pass diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/submit.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/submit.py new file mode 100644 index 0000000..d7e3e6e --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/ipop_stats/views/submit.py @@ -0,0 +1,35 @@ +from ..errors import * +from flask import Blueprint, request, jsonify +from flask import current_app as app +import datetime +import socket +import uuid +import logging + +from pymongo import MongoClient + + +submit = Blueprint("submit", __name__, url_prefix="/api") + +@submit.route("/submit", methods=["POST"]) +def update(): + logger = logging.getLogger('all') + logger.info("Report Message Received") + data = request.json + report = {"Report": data} + obj_id = app.database.user_collection.insert_one(report) + response = jsonify(result=uuid.uuid4().hex, status="success") # TODO: what is result? + return response + +@submit.route("/") +def hello(): + return "IPOP-Usage Stat Server!" + +@submit.route("/generate_uuid") +def generate_uuid(): + """Convenience uuid generator, for clients who don't have a convenient local + generator.""" + response = jsonify(result=uuid.uuid4().hex, status="success") + response.headers["Expires"] = "-1" + response.headers["Cache-Control"] = "no-cache, no-store" + return response diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/run.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/run.py new file mode 100755 index 0000000..4d75b47 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/run.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +from flask import Flask +from tornado.wsgi import WSGIContainer +from tornado.httpserver import HTTPServer +from tornado.ioloop import IOLoop +from ipop_stats.app import create as create_app +import argparse +import os, os.path + + +parser = argparse.ArgumentParser(description="Standalone test server for " + "ipop-stats") + +try: + default_config_path = os.path.abspath(os.environ["IPOP_STATS_SETTINGS"]) +except: + default_config_path = "config/debug.yml" + +parser.add_argument( + "-c", "--config", + default=default_config_path, + type=os.path.abspath, + help="the flask config file for the server" +) + +# Create Flask APP +args = parser.parse_args() +app = create_app(**vars(args)) #run(host="0.0.0.0", port=8080) + +# Run with Tonado +http_server = HTTPServer(WSGIContainer(app)) +http_server.listen(8081) +IOLoop.instance().start() \ No newline at end of file diff --git a/ipopusagereport-0.1/usr/share/ipop-usagereport/setup.py b/ipopusagereport-0.1/usr/share/ipop-usagereport/setup.py new file mode 100644 index 0000000..de3b226 --- /dev/null +++ b/ipopusagereport-0.1/usr/share/ipop-usagereport/setup.py @@ -0,0 +1,14 @@ +from setuptools import * + +setup( + name="ipop-stats", + version="0.0.1", + install_requires=[ + "flask>=0.10.1,<0.11", + "pymongo>=3.6,<4", + "PyYAML>=3.10,<4.0", + ], + description="Gathers anonymous usage statistics of IPOP users", + packages=find_packages(), + scripts=["bin/ipop-stats"], +)