Merge pull request #60 from airavata-courses/assignment_1_fixes

Assignment 1 fixes

APIgateway/routing/eventHandler.js

@@ -15,7 +15,6 @@ return new Promise((resolve,reject)=>{
             console.log(error)
             reject(error)
         } else {  
-        console.log('produced')
 
         resolve(data)
         }

APIgateway/server.js

@@ -54,9 +54,9 @@ consumer.on('message', (message)=>{
   let data=JSON.parse(message.value)
 
   let uid=String(data['uid'])
-  console.log(data)
+
   if(resID[uid] && resID[uid]!=null){
-    console.log('sending response')
+
     ob=resID[uid]
     resID[uid]=null
     delete resID["uid"];

dataModelling/app.py

@@ -1,20 +1,22 @@
 from flask import Flask
 from services.produce import produce
-from kafka import KafkaConsumer
+from pykafka import KafkaClient
 import nexradaws
 import json 
 
 conn = nexradaws.NexradAwsInterface()
 app = Flask(__name__)
 
-print('done')
-consumer = KafkaConsumer('dataModellingConsumerF')
+
+client = KafkaClient()
+consumer =  client.topics['dataModellingConsumerF'].get_simple_consumer(consumer_group="dataModellingConsumerF",
+                                     auto_commit_enable=True)
 
 for message in consumer:
     if message is not None:
         dataMsg=json.loads(message.value)
-        print(dataMsg,message.topic)
-        produce(dataMsg,conn)
+        produce(dataMsg,conn,client)
+        consumer.commit_offsets()
 
 if __name__ == "__main__":
  app.run(host='127.0.0.0', port=8080)

dataModelling/requirements.txt

@@ -1,4 +1,5 @@
-arm-pyart==1.11.0
+appdirs==1.4.3
+
 boto3==1.11.13
 botocore==1.14.13
 certifi==2019.11.28
@@ -7,37 +8,53 @@ chardet==3.0.4
 Click==7.0
 cycler==0.10.0
 DateTime==4.3
+decorator==4.4.1
 dnspython==1.16.0
 docutils==0.15.2
 Flask==1.1.1
+flask-kafka==0.0.2
+flask-mongoengine==0.9.5
+Flask-PyMongo==2.3.0
+Flask-WTF==0.14.3
 idna==2.8
-imageio==2.6.1
+
+ipython-genutils==0.2.0
 itsdangerous==1.1.0
 Jinja2==2.11.1
 jmespath==0.9.4
-kafka==1.3.5
 kafka-python==1.4.7
 kazoo==2.5.0
 kiwisolver==1.1.0
 MarkupSafe==1.1.1
 matplotlib==3.1.3
+MetPy==0.12.0
+mongoengine==0.19.1
 netCDF4==1.5.3
 nexradaws==1.1
 numpy==1.17.3
+packaging==20.1
 pandas==1.0.1
 Pillow==6.2.1
+Pint==0.10.1
+pooch==1.0.0
 pyimgur==0.6.0
+PyJWT==1.7.1
 pykafka==2.8.0
 pymongo==3.10.1
 pyparsing==2.4.6
 python-dateutil==2.8.1
+python-dotenv==0.11.0
 pytz==2019.3
 requests==2.22.0
 s3transfer==0.3.3
 scipy==1.3.1
 six==1.14.0
 tabulate==0.8.6
+traitlets==4.3.3
+twilio==6.35.4
 urllib3==1.25.8
 virtualenv==16.7.9
 Werkzeug==1.0.0
+WTForms==2.2.1
+xarray==0.15.0
 zope.interface==4.7.1

dataModelling/services/produce.py

@@ -1,64 +1,92 @@
 
 import matplotlib.pyplot as plt
 import matplotlib
-import pyart
-import numpy as np
-matplotlib.use('Agg')
-from PIL import Image
-import base64
 import matplotlib.image as mpimg
 import json
-import pandas as pd
+import numpy as np
 import pyimgur
-from kafka import KafkaProducer
-producer = KafkaProducer(bootstrap_servers='localhost:9092')
-
+from pykafka import KafkaClient
+from metpy.cbook import get_test_data
+from metpy.io import Level2File
+from metpy.plots import add_metpy_logo, add_timestamp
 CLIENT_ID = "78adc2281c1c838"
 
 im = pyimgur.Imgur(CLIENT_ID)
 
 
 
-def produce(data,conn):
+def produce(data,conn,client):
         Year=data['inputData']['Year']
         Month=data['inputData']['Month']
         Day=data['inputData']['Day']
         Radar=data['inputData']['Radar']
-        scans = conn.get_avail_scans(Year,Month,Day,Radar) # year, month and day
-        results = conn.download(scans[0], 'templocation')
+        uid=data['uid']
+        inputData=data['inputData']
+        userID=data["userID"]
 
-        fig = plt.figure(figsize=(16,12))
+
+        numberOfPlots=1
+        scans = conn.get_avail_scans(Year,Month,Day,Radar) # year, month and day
+        results = conn.download(scans[numberOfPlots-1], 'templocation')
+
+        # fig = plt.figure(figsize=(16,12))
         for i,scan in enumerate(results.iter_success(),start=1):
-                ax = fig.add_subplot(2,2,i)
-                radar = scan.open_pyart()
+        #         ax = fig.add_subplot(1,1,i)
+        #         radar = scan.open_pyart()
+
+        #         display = pyart.graph.RadarDisplay(radar)
+        #         display.plot('reflectivity',0,ax=ax,title="{} {}".format(scan.radar_id,scan.scan_time))
+        #         display.set_limits((-150, 150), (-150, 150), ax=ax)
+
+                sweep = 0
+                name = scan.open()
+                f = Level2File(name)
+                # First item in ray is header, which has azimuth angle
+                az = np.array([ray[0].az_angle for ray in f.sweeps[sweep]])
+
+                # 5th item is a dict mapping a var name (byte string) to a tuple
+                # of (header, data array)
+                ref_hdr = f.sweeps[sweep][0][4][b'REF'][0]
+                ref_range = np.arange(ref_hdr.num_gates) * ref_hdr.gate_width + ref_hdr.first_gate
+                ref = np.array([ray[4][b'REF'][1] for ray in f.sweeps[sweep]])
+
+                rho_hdr = f.sweeps[sweep][0][4][b'RHO'][0]
+                rho_range = (np.arange(rho_hdr.num_gates + 1) - 0.5) * rho_hdr.gate_width + rho_hdr.first_gate
+                rho = np.array([ray[4][b'RHO'][1] for ray in f.sweeps[sweep]])
+
+                fig, axes = plt.subplots(1, 2, figsize=(15, 8))
+                add_metpy_logo(fig, 190, 85, size='large')
+                for var_data, var_range, ax in zip((ref, rho), (ref_range, rho_range), axes):
+                # Turn into an array, then mask
+                        data = np.ma.array(var_data)
+                        data[np.isnan(data)] = np.ma.masked
+
+                        # Convert az,range to x,y
+                        xlocs = var_range * np.sin(np.deg2rad(az[:, np.newaxis]))
+                        ylocs = var_range * np.cos(np.deg2rad(az[:, np.newaxis]))
+
+                        # Plot the data
+                        ax.pcolormesh(xlocs, ylocs, data, cmap='viridis')
+                        ax.set_aspect('equal', 'datalim')
+                        ax.set_xlim(-40, 20)
+                        ax.set_ylim(-30, 30)
+                        add_timestamp(ax, f.dt, y=0.02, high_contrast=True)
+
+
+
+                pltName='images/'+str(uid+str(i)+'.png')
+                plt.savefig(pltName)
+                plt.close(fig)
+                uploadImage = im.upload_image(pltName, title="Uploaded with PyImgur")
+                link=str(uploadImage.link)
+
+                body={  "inputData":inputData,
+                        "outputData":link,
+                        "uid":uid,
+                        "userID":userID
+                }
+
+                with (client.topics['dataAnalysisConsumerF']).get_sync_producer() as producer:
+                                producer.produce(bytes(json.dumps(body),'utf-8'))
 
-                display = pyart.graph.RadarDisplay(radar)
-                display.plot('reflectivity',0,ax=ax,title="{} {}".format(scan.radar_id,scan.scan_time))
-                display.set_limits((-150, 150), (-150, 150), ax=ax)
-        plotName=data["uid"]+'.png'
-        plt.savefig(plotName)
-
-        plt.close(fig)
-
-        uploadImage = im.upload_image(plotName, title="Uploaded with PyImgur")
-        link=str(uploadImage.link)
-        # fig.canvas.draw ( )
-        # w,h = fig.canvas.get_width_height()
-        # buf = np.frombuffer ( fig.canvas.tostring_argb(), dtype=np.uint8 )
-        # buf.shape = ( w, h,4 )
-        # buf = np.roll ( buf, 3, axis = 2 )
-        # w, h, d = buf.shape
-        # res=Image.frombytes( "RGBA", ( w ,h ), buf.tostring( ) )
-        # res=res.resize((960,720),Image.ANTIALIAS)
-        body={  "inputData":data["inputData"],
-                "outputData":link,
-                "uid":data["uid"],
-                "userID":data["userID"]
-            }
-        # body='{"resultPlot":"0","uid":"8","userID":"2"}'
-
-        print(bytes(json.dumps(body),'utf-8'))
-        producer.send('dataAnalysisConsumerF', bytes(json.dumps(body),'utf-8'))
-
-
 

dataRetrieval/app.py

@@ -8,14 +8,16 @@
 
 
 client = KafkaClient()
-consumer =  client.topics['dataRetrievalConsumerF'].get_simple_consumer()
+consumer =  client.topics['dataRetrievalConsumerF'].get_simple_consumer(consumer_group="dataModellingConsumerF",
+                                     auto_commit_enable=True)
+
 for message in consumer:
     if message is not None:
-        print(message.value)
         data=json.loads((message.value))
-        print((json.dumps(data)))
         produce(data,client)
-
+
+
+
 
 
 

dataRetrieval/controllers.py

@@ -13,23 +13,11 @@ def produce(data,client):
     userID=data['userID']
     uid=data['uid']
     res=return_regions(Month,Day,Year,Radar)
-    print(res)
     if(res=='success'):
         body = {"inputData":{"Month":Month,"Day":Day,"Year":Year,"Radar":Radar},"userID":userID,"uid":uid}
-        # print(bytes(json.dumps(body),'utf-8'))
-        # 
-
-
-
-
+
         with (client.topics['dataModellingConsumerF']).get_sync_producer() as producer:
             producer.produce((bytes(json.dumps(body),'utf-8')))
 
-
-    # else:
-    #     with (client.topics['apiGatewayConsumer']).get_sync_producer() as producer:
-    #         producer.produce(msg)
-
-    else:
-        print('error')
+
 

dataRetrieval/requirements.txt

@@ -4,7 +4,6 @@ certifi==2019.11.28
 chardet==3.0.4
 Click==7.0
 dnspython==1.16.0
-docutils==0.15.2
 Flask==1.1.1
 flask-kafka==0.0.2
 flask-mongoengine==0.9.5

frontend/package.json

@@ -16,13 +16,14 @@
     "react-datepicker": "^2.12.0",
     "react-dom": "^16.12.0",
     "react-loading": "^2.0.3",
+    "react-modal-image": "^2.5.0",
     "react-router-dom": "^5.1.2",
     "react-scripts": "3.3.0",
     "react-select": "^3.0.8",
     "reactstrap": "^8.4.1"
   },
   "scripts": {
-    "start": "PORT=4000 react-scripts start",
+    "start": "react-scripts start",
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject"

frontend/src/components/dashboard/sessionViewer.jsx

@@ -22,7 +22,6 @@ export default class SessionView extends Component {
       dashService.sessions(data).then(res => {
         if (res.status === 200) {
           this.setState({ sessions: res.data.sessions });
-          console.log(this.state.sessions);
         }
       });
     } else {
@@ -65,7 +64,7 @@ class SessionCard extends Component {
 
        <Card style={{ width: '18rem' }}>
 
-       {this.props.session.outputData && <Card.Img className='sessionCardImg' variant="top" src={this.props.session.outputData} href={this.props.session.outputData}/>}
+       {this.props.session.outputData && <Card.Img className='sessionCardImg m-auto' variant="top" src={this.props.session.outputData} href={this.props.session.outputData}/>}
        {!this.props.session.outputData && <Card.Img className='sessionCardImg' variant="top" src={notFound} href={this.props.session.outputData}/>}
   <Card.Body className='text-center text-primary'>
     <Card.Title>{'Station Name - '+this.props.session.inputData.Radar}</Card.Title>