Parsing Glasnost Data.ipynb

{
 "metadata": {
  "name": "Parsing Glasnost Data"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "<h1>Parsing Glasnost Data</h1>\n",
      "\n",
      "Glasnost data comes in a very specific format. It is a long logfile with plenty of lines defining having both client and server tests.\n",
      "\n",
      "According to: http://code.google.com/p/glasnost/wiki/GlasnostDataFormat the data in version 1 starts with: Received: replay XXXX [client|server] it runs the tests several times on different ports to detect shaping. let's parse the format\n",
      "\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import itertools,re #we are going to need them\n",
      "\n",
      "f=open(\"testdata.log\")\n",
      "clientip=re.compile(\"^([0-9]+) Client .* ([0-9.]+) connected\")\n",
      "replay=re.compile(\"^([0-9]+) Received: replay ([a-zA-Z]+) as ([a-z]+)\")\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 57
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "ok, let's make this functions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def extract_re(lines,regex):\n",
      "    regex=re.compile(regex)\n",
      "    ln=itertools.dropwhile(lambda x: not regex.match(x), lines)\n",
      "    return regex.match(ln.next()).groups()\n",
      "\n",
      "def extract_test(lines):\n",
      "    return dict(zip(('time','test','mode','port'),extract_re(lines,\"^([0-9]+) Received: replay ([a-zA-Z]+) as ([a-z]+) on port ([0-9]+)\")))\n",
      "\n",
      "def extract_client(lines):\n",
      "    return dict(zip(('time','ip'),extract_re(lines,\"^([0-9]+) Client .* ([0-9.]+) connected\")))\n",
      "\n",
      "def extract_server_results(lines):\n",
      "    return dict(zip(('time','transferred','received','seconds','txrate','rxrate'),\n",
      "    extract_re(lines,\"^([0-9]+) Transmitted ([0-9]+) bytes and received ([0-9]+) bytes in ([0-9.]+) seconds: ([0-9.]+) ([0-9.]+) bps\")))\n",
      "\n",
      "def extract_client_results(lines):\n",
      "    return dict(zip(('time','transferred','received','seconds','txrate','rxrate'),\n",
      "    extract_re(lines,\"^([0-9]+) Client: Transferred ([0-9]+) bytes and received ([0-9]+) bytes in ([0-9.]+) seconds: ([0-9.]+) ([0-9.]+) bps\")))\n",
      "\n",
      "def extract_test_results(lines):\n",
      "    while lines:\n",
      "        yield {\"test\": extract_test(lines), \"server\": extract_server_results(lines),\n",
      "            \"client\": extract_client_results(lines)}\n",
      "\n",
      "results=[r for r in extract_test_results(f)]\n",
      "print results"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[{'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338742929'}, 'client': {'received': '17', 'seconds': '19.874', 'txrate': '5031704.941129114', 'transferred': '12500013', 'rxrate': '6.843111603099527', 'time': '1345338764950'}, 'server': {'received': '1813320', 'seconds': '20.873', 'txrate': '7', 'transferred': '17', 'rxrate': '695006', 'time': '1345338764950'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338764950'}, 'client': {'received': '17', 'seconds': '19.944', 'txrate': '5014044.5246690735', 'transferred': '12500013', 'rxrate': '6.81909346169274', 'time': '1345338787813'}, 'server': {'received': '2445500', 'seconds': '20.945', 'txrate': '6', 'transferred': '17', 'rxrate': '934055', 'time': '1345338787813'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338788383'}, 'client': {'received': '17', 'seconds': '19.921', 'txrate': '5019833.542492847', 'transferred': '12500013', 'rxrate': '6.8269665177450936', 'time': '1345338811385'}, 'server': {'received': '2420680', 'seconds': '20.921', 'txrate': '7', 'transferred': '17', 'rxrate': '925630', 'time': '1345338811385'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338812425'}, 'client': {'received': '17', 'seconds': '19.93', 'txrate': '5017566.683391872', 'transferred': '12500013', 'rxrate': '6.823883592574009', 'time': '1345338834993'}, 'server': {'received': '2427980', 'seconds': '20.920', 'txrate': '7', 'transferred': '17', 'rxrate': '928492', 'time': '1345338834993'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338834993'}, 'client': {'received': '17', 'seconds': '19.924', 'txrate': '5019077.69524192', 'transferred': '12500013', 'rxrate': '6.825938566552901', 'time': '1345338857870'}, 'server': {'received': '1765140', 'seconds': '20.921', 'txrate': '7', 'transferred': '17', 'rxrate': '674989', 'time': '1345338857870'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'client', 'time': '1345338858789'}, 'client': {'received': '17', 'seconds': '19.905', 'txrate': '5023868.57573474', 'transferred': '12500013', 'rxrate': '6.832454157246922', 'time': '1345338880830'}, 'server': {'received': '1987060', 'seconds': '20.903', 'txrate': '7', 'transferred': '17', 'rxrate': '760495', 'time': '1345338880830'}}, {'test': {'test': 'BitTorrent', 'port': '0', 'mode': 'client', 'time': '1345338880830'}, 'client': {'received': '17', 'seconds': '19.939', 'txrate': '5015301.870705652', 'transferred': '12500013', 'rxrate': '6.820803450524099', 'time': '1345338903714'}, 'server': {'received': '2236720', 'seconds': '20.936', 'txrate': '6', 'transferred': '17', 'rxrate': '854699', 'time': '1345338903714'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'client', 'time': '1345338903714'}, 'client': {'received': '17', 'seconds': '19.907', 'txrate': '5023363.841864671', 'transferred': '12500013', 'rxrate': '6.831767719897523', 'time': '1345338926598'}, 'server': {'received': '1994360', 'seconds': '20.917', 'txrate': '7', 'transferred': '17', 'rxrate': '762786', 'time': '1345338926598'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'client', 'time': '1345338926598'}, 'client': {'received': '17', 'seconds': '19.919', 'txrate': '5020337.567146945', 'transferred': '12500013', 'rxrate': '6.827651990561775', 'time': '1345338949489'}, 'server': {'received': '2439660', 'seconds': '20.917', 'txrate': '7', 'transferred': '17', 'rxrate': '933079', 'time': '1345338949489'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'client', 'time': '1345338950573'}, 'client': {'received': '17', 'seconds': '19.94', 'txrate': '5015050.351053159', 'transferred': '12500013', 'rxrate': '6.820461384152457', 'time': '1345338973057'}, 'server': {'received': '2387100', 'seconds': '20.940', 'txrate': '6', 'transferred': '17', 'rxrate': '911989', 'time': '1345338973057'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'client', 'time': '1345338974255'}, 'client': {'received': '17', 'seconds': '19.933', 'txrate': '5016811.518587267', 'transferred': '12500013', 'rxrate': '6.822856569507851', 'time': '1345338996654'}, 'server': {'received': '2439660', 'seconds': '20.949', 'txrate': '6', 'transferred': '17', 'rxrate': '931667', 'time': '1345338996654'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'client', 'time': '1345338997737'}, 'client': {'received': '17', 'seconds': '19.923', 'txrate': '5019329.619033279', 'transferred': '12500013', 'rxrate': '6.826281182552829', 'time': '1345339020210'}, 'server': {'received': '2213360', 'seconds': '20.927', 'txrate': '6', 'transferred': '17', 'rxrate': '846125', 'time': '1345339020210'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339020210'}, 'client': {'received': '11643500', 'seconds': '20.901', 'txrate': '6.506865700205732', 'transferred': '17', 'rxrate': '4456628.869432084', 'time': '1345339042335'}, 'server': {'received': '17', 'seconds': '20.099', 'txrate': '4634572', 'transferred': '11643500', 'rxrate': '7', 'time': '1345339042335'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339042659'}, 'client': {'received': '11774900', 'seconds': '20.901', 'txrate': '6.506865700205732', 'transferred': '17', 'rxrate': '4506923.113726616', 'time': '1345339064175'}, 'server': {'received': '17', 'seconds': '20.403', 'txrate': '4617040', 'transferred': '11774900', 'rxrate': '7', 'time': '1345339064175'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339064198'}, 'client': {'received': '12500013', 'seconds': '20.904', 'txrate': '13.011863758132415', 'transferred': '34', 'rxrate': '4783778.415614236', 'time': '1345339085867'}, 'server': {'received': '17', 'seconds': '20.555', 'txrate': '4865037', 'transferred': '12500013', 'rxrate': '7', 'time': '1345339085867'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339085867'}, 'client': {'received': '10065240', 'seconds': '20.872', 'txrate': '6.515906477577616', 'transferred': '17', 'rxrate': '3857891.9126101956', 'time': '1345339107053'}, 'server': {'received': '17', 'seconds': '20.041', 'txrate': '4017896', 'transferred': '10065240', 'rxrate': '7', 'time': '1345339107053'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339107406'}, 'client': {'received': '11196740', 'seconds': '20.914', 'txrate': '6.502821076790666', 'transferred': '17', 'rxrate': '4282964.521373242', 'time': '1345339128525'}, 'server': {'received': '17', 'seconds': '20.015', 'txrate': '4475361', 'transferred': '11196740', 'rxrate': '7', 'time': '1345339128525'}}, {'test': {'test': 'BitTorrent', 'port': '6881', 'mode': 'server', 'time': '1345339128932'}, 'client': {'received': '11485660', 'seconds': '20.908', 'txrate': '6.50468720107136', 'transferred': '17', 'rxrate': '4394742.682226898', 'time': '1345339150775'}, 'server': {'received': '17', 'seconds': '20.730', 'txrate': '4446115', 'transferred': '11520860', 'rxrate': '7', 'time': '1345339150775'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339150775'}, 'client': {'received': '10549960', 'seconds': '20.912', 'txrate': '6.503442999234889', 'transferred': '17', 'rxrate': '4035944.912012242', 'time': '1345339172834'}, 'server': {'received': '17', 'seconds': '20.526', 'txrate': '4111872', 'transferred': '10549960', 'rxrate': '7', 'time': '1345339172834'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339172834'}, 'client': {'received': '12746753', 'seconds': '20.927', 'txrate': '12.997562956945574', 'transferred': '34', 'rxrate': '4872844.841592202', 'time': '1345339194439'}, 'server': {'received': '34', 'seconds': '20.511', 'txrate': '4971710', 'transferred': '12746753', 'rxrate': '13', 'time': '1345339194439'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339194439'}, 'client': {'received': '8411060', 'seconds': '20.921', 'txrate': '6.500645284642226', 'transferred': '17', 'rxrate': '3216312.7957554613', 'time': '1345339216335'}, 'server': {'received': '17', 'seconds': '20.793', 'txrate': '3246258', 'transferred': '8437340', 'rxrate': '7', 'time': '1345339216335'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339216335'}, 'client': {'received': '9951360', 'seconds': '20.926', 'txrate': '6.499092038612253', 'transferred': '17', 'rxrate': '3804400.2676096726', 'time': '1345339237643'}, 'server': {'received': '17', 'seconds': '20.221', 'txrate': '3936979', 'transferred': '9951360', 'rxrate': '7', 'time': '1345339237643'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339237856'}, 'client': {'received': '9873980', 'seconds': '20.929', 'txrate': '6.498160447226337', 'transferred': '17', 'rxrate': '3774276.840747289', 'time': '1345339259030'}, 'server': {'received': '17', 'seconds': '20.082', 'txrate': '3933466', 'transferred': '9873980', 'rxrate': '7', 'time': '1345339259030'}}, {'test': {'test': 'BitTorrent', 'port': '50806', 'mode': 'server', 'time': '1345339259383'}, 'client': {'received': '7432860', 'seconds': '20.828', 'txrate': '6.529671595928558', 'transferred': '17', 'rxrate': '2854949.106971385', 'time': '1345339280469'}, 'server': {'received': '17', 'seconds': '19.859', 'txrate': '2994289', 'transferred': '7432860', 'rxrate': '7', 'time': '1345339280461'}}]\n"
       ]
      }
     ],
     "prompt_number": 58
    }
   ],
   "metadata": {}
  }
 ]
}