diff --git a/Makefile b/Makefile index 370f7e07e..df4fc6fc3 100644 --- a/Makefile +++ b/Makefile @@ -19,15 +19,6 @@ export LD_LIBRARY_PATH=/usr/local/lib local: cd tools && sudo make && cd ../lucida && make -start_all: - cd tools && ./start_all_tmux.sh - -start_all_secure: - cd tools && ./start_all_tmux.sh secure - -start_test_all: - cd tools && ./start_all_tmux.sh test - all_service: cd lucida && make all diff --git a/lucida/calendar/CalendarClient/CalendarClient.java b/lucida/calendar/CalendarClient/CalendarClient.java index fbe4ac5dd..1960fd792 100755 --- a/lucida/calendar/CalendarClient/CalendarClient.java +++ b/lucida/calendar/CalendarClient/CalendarClient.java @@ -1,9 +1,6 @@ //Java packages -import java.io.FileInputStream; -import java.io.InputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Properties; //Thrift java libraries import org.apache.thrift.TException; @@ -22,17 +19,11 @@ public class CalendarClient { public static void main(String [] args) throws IOException { - // Collect the port number. - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("CA_PORT"); - Integer port = Integer.valueOf(port_str); - if (args.length == 1) { - port = Integer.parseInt(args[0]); - } else { - System.out.println("Using default port for Calendar Client: " + port); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); } + Integer port = Integer.valueOf(args[0]); // Query. String LUCID = "Clinc"; diff --git a/lucida/calendar/CalendarClient/start-Calendar-client.sh b/lucida/calendar/CalendarClient/start-Calendar-client.sh index d79380b29..6a8a2460d 100755 --- a/lucida/calendar/CalendarClient/start-Calendar-client.sh +++ b/lucida/calendar/CalendarClient/start-Calendar-client.sh @@ -1,8 +1,9 @@ #!/bin/bash + # Add Thrift classes to class path export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:thrift # Add other jar files to class path export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar # run the server -java -cp "$JAVA_CLASS_PATH" CalendarClient "$@" +java -cp "$JAVA_CLASS_PATH" CalendarClient "$1" diff --git a/lucida/calendar/Makefile b/lucida/calendar/Makefile index 472761d01..b58c76f25 100644 --- a/lucida/calendar/Makefile +++ b/lucida/calendar/Makefile @@ -24,10 +24,15 @@ corenlp: fi start_server: - ./gradlew run + @if [ "$(port)" != "" ]; then \ + ./gradlew run -Pargs=$(port); \ + fi start_test: - cd CalendarClient; ./start-Calendar-client.sh + @if [ "$(port)" != "" ]; then \ + cd CalendarClient; \ + ./start-Calendar-client.sh $(port); \ + fi clean: rm -rf src/main/java/thrift && rm -rf CalendarClient/thrift && rm -rf CalendarClient/*.class && rm -rf build && rm -rf lib diff --git a/lucida/calendar/build.gradle b/lucida/calendar/build.gradle index defc0fb1d..66a892c43 100644 --- a/lucida/calendar/build.gradle +++ b/lucida/calendar/build.gradle @@ -39,5 +39,4 @@ dependencies { compile name: 'stanford-corenlp-3.7.0' compile name: 'xom-1.2.10-src' compile name: 'xom' - } diff --git a/lucida/calendar/src/main/java/calendar/CalendarDaemon.java b/lucida/calendar/src/main/java/calendar/CalendarDaemon.java index f16b8d892..1e3272c04 100644 --- a/lucida/calendar/src/main/java/calendar/CalendarDaemon.java +++ b/lucida/calendar/src/main/java/calendar/CalendarDaemon.java @@ -1,13 +1,10 @@ package calendar; -import java.io.FileInputStream; -import java.io.InputStream; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.Properties; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.server.TNonblockingServer; @@ -34,11 +31,12 @@ public class CalendarDaemon { public static void main(String [] args) throws TTransportException, IOException, InterruptedException { - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("CA_PORT"); - Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + TProcessor proc = new LucidaService.AsyncProcessor( new CAServiceHandler.AsyncCAServiceHandler()); TNonblockingServerTransport transport = new TNonblockingServerSocket(port); @@ -47,7 +45,7 @@ public static void main(String [] args) .protocolFactory(new TBinaryProtocol.Factory()) .transportFactory(new TFramedTransport.Factory()); final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); - System.out.println("CA at port " + port_str); + System.out.println("CA at port " + port); server.serve(); } } diff --git a/lucida/commandcenter/api/Database.py b/lucida/commandcenter/api/Database.py new file mode 100644 index 000000000..2164f249b --- /dev/null +++ b/lucida/commandcenter/api/Database.py @@ -0,0 +1,441 @@ +import os +import sys +from pymongo import * +from bson.objectid import ObjectId + +class MongoDB(object): + def __init__(self): + mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR') + if mongodb_addr: + self.db = MongoClient(mongodb_addr, 27017).lucida + else: + self.db = MongoClient().lucida + + def get_services(self): + dictReturn = [] + + service_list = self.db["service_info"].find() + count_service = service_list.count() + for i in range(count_service): + document = service_list[i] + document['_id'] = str(document['_id']) + dictReturn.append(document) + + return dictReturn + + def add_service(self, name, acronym, input_type, learn_type): + """ + return code: + 0: success + 1: name has already exists + 2: acronym has already used + """ + + num = 0 + instance = [] + collection = self.db.service_info + + # check if current service name is used + count = collection.count({'name': name}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service name already used.') + return 1, '' + + # check if current service acronym is used + count = collection.count({'acronym': acronym}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service acronym already used.') + return 2, '' + + # list the attributes for the interface + post = { + "name": name, # name of service + "acronym": acronym, # acronym of service + "num": num, # biggest id of instance + "instance": instance, # host/port pair of instances + "input": input_type, # input type + "learn": learn_type # learn type + # "location": location # location of service in local + } + + # insert the service information into MongoDB + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def add_empty_service(self): + """ + return code: + 0: success + """ + + collection = self.db.service_info + + name = '' + acronym = '' + input_type = 'text' + learn_type = 'none' + num = 0 + instance = [] + + post = { + "name": name, # name of service + "acronym": acronym, # acronym of service + "num": num, # number of instance + "instance": instance, # host/port pair of instances + "input": input_type, # input type + "learn": learn_type # learn type + # "location": location # location of service in local + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + + def update_service(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: service name not found + 2: updated name already used + 3: updated acronym already used + """ + + collection = self.db.service_info + + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] service not exists in MongoDB.') + return 1 + + # check if update no difference return success + result = collection.find({'_id': ObjectId(_id)})[0] + if result[op] == value: + return 0 + + if op == 'name': + count = collection.count({'name': value}) + if count != 0: + print('[python error] Updated name already used') + return 2 + + if op == 'acronym': + count = collection.count({'acronym': value}) + if count != 0: + print('[python error] Updated acronym already used') + return 3 + + collection.update({'_id': ObjectId(_id)}, {'$set': {op: value }}) + return 0 + + def delete_service(self, _id): + """ + return code: + 0: success + 1: service not exist + """ + + collection = self.db.service_info + + # check if current service is in MongoDB + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] service not exists in MongoDB.') + return 1 + + collection.remove({'_id': ObjectId(_id)}) + return 0 + + def get_workflows(self): + dictReturn = [] + + workflow_list = self.db["workflow_info"].find() + count_workflow = workflow_list.count() + for i in range(count_workflow): + document = workflow_list[i] + document['_id'] = str(document['_id']) + dictReturn.append(document) + + return dictReturn + + def add_workflow(self, name, input_type, classifier_path, class_code, stategraph): + """ + return code: + 0: success + 1: workflow name already exists + """ + + collection = self.db.workflow_info + + # check if current workflow is in MongoDB + count = collection.count({'name': name}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] workflow name already used.') + return 1, '' + + # list the attributes for the interface + post = { + "name": name, # name of workflow + "input": input_type, # allowed input type + "classifier": classifier_path, # classifier data path + "code": class_code, # code for implementation of the workflow class + "stategraph": stategraph # metadata for state graph + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def add_empty_workflow(self): + """ + return code: + 0: success + """ + + collection = self.db.workflow_info + + name = '' + input_type = [] + classifier_path = '' + class_code = '' + stategraph = '' + + post = { + "name": name, # name of workflow + "input": input_type, # allowed input type + "classifier": classifier_path, # classifier data path + "code": class_code, # code for implementation of the workflow class + "stategraph": stategraph # metadata for state graph + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def update_workflow(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: workflow name not found + 2: updated name already used + """ + + collection = self.db.workflow_info + + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] workflow not exists') + return 1 + + # check if update no difference return success + result = collection.find({'_id': ObjectId(_id)})[0] + if result[op] == value: + return 0 + + if op == 'name': + count = collection.count({'name': value}) + if count != 0: + print('[python error] Updated name already used') + return 2 + + collection.update({'_id': ObjectId(_id)}, {'$set': {op: value }}) + return 0 + + def delete_workflow(self, _id): + """ + return code: + 0: success + 1: workflow not exists + """ + + collection = self.db.workflow_info + + # check if current workflow is in MongoDB + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] workflow not exists') + return 1 + + collection.remove({'_id': ObjectId(_id)}) + return 0 + + def add_instance(self, _id, name, host, port): + """ + return code: + 0: success + 1: host/port not valid + 2: service not exist + 3: host/port already used + """ + + collection = self.db.service_info + if host == 'localhost': + host = '127.0.0.1' + + if not validate_ip_port(host, port): + print('[python error] Host/port pair is not valid.') + return 1, '' + + # check if current service is in MongoDB + object_id = ObjectId(_id) + count = collection.count({'_id': object_id}) + if count != 1: + print('[python error] service not exists in MongoDB.') + return 2, '' + + """ + # check if name is used + result = collection.find({'name': service_name, 'instance.name': name}) + if result.count() != 0: + print('[python error] Instance name has already been used.') + return 3 + """ + + # check if host and port is used + result = collection.find({'instance' : { '$elemMatch': { 'host': host, 'port': port}}}) + if result.count() != 0: + print('[python error] Host/port has already been used.') + return 3, '' + + result = collection.find({'_id': object_id}) + instance_id = str(result[0]['num']) + collection.update_one({'_id': object_id}, {'$inc': {'num': 1}}) + collection.update_one({'_id': object_id}, {'$push': {'instance': { + 'name': name, + 'host': host, + 'port': port, + 'id': instance_id + }}}) + return 0, instance_id + + def add_empty_instance(self, _id): + """ + return code: + 0: success + 1: service not exists + """ + + collection = self.db.service_info + + name = '' + host = '127.0.0.1' + port = 0 + object_id = ObjectId(_id) + count = collection.count({'_id': object_id}) + if count != 1: + print('[python error] service not exists in MongoDB.') + return 1, '' + + result = collection.find({'_id': object_id}) + instance_id = str(result[0]['num']) + collection.update_one({'_id': object_id}, {'$inc': {'num': 1}}) + collection.update_one({'_id': object_id}, {'$push': {'instance': { + 'name': name, + 'host': host, + 'port': port, + 'id': instance_id + }}}) + return 0, instance_id + + def update_instance(self, _id, instance_id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: instance not found + 2: host/port pair not valid + 3: host/port pair already used + """ + + collection = self.db.service_info + if op == 'host': + if value == 'localhost': + value = '127.0.0.1' + + # check if current service is in MongoDB + object_id = ObjectId(_id) + result = collection.find({'_id': object_id, 'instance.id': instance_id}) + if result.count() != 1: + print('[python error] Instance name not exists.') + return 1 + + # check update nothing + cur_instance = {} + instance_result = result[0]['instance'] + for instance in instance_result: + if instance['id'] == instance_id: + cur_instance = instance + if instance[op] == value: + return 0 + + if op == 'host': + old_port = cur_instance['port'] + + if not validate_ip_port(value, old_port): + print('[python error] Host/port pair is not valid.') + return 2 + + result = collection.find({'instance': {'$elemMatch': {'host': value, 'port': old_port}}}) + if result.count() != 0: + print('[python error] Updated host/port has already been used') + return 3 + + if op == 'port': + old_host = cur_instance['host'] + + if not validate_ip_port(old_host, value): + print('[python error] Host/port pair is not valid.') + return 2 + + result = collection.find({'instance': {'$elemMatch': {'host': old_host, 'port': value}}}) + if result.count() != 0: + print('[python error] Updated host/port has already been used') + return 3 + + op = 'instance.$.'+op + collection.update({'_id': object_id, 'instance.id': instance_id}, {'$set': {op: value}}) + return 0 + + def delete_instance(self, _id, instance_id): + """ + return code: + 0: success + 1: instance not exist + """ + collection = self.db.service_info + + # check if current service is in MongoDB + object_id = ObjectId(_id) + result = collection.find({'_id': object_id, 'instance.id': instance_id}) + if result.count() != 1: + print('[python error] Instance name not exists.') + return 1 + + collection.update({'_id':object_id}, {'$pull': {'instance': {'id': instance_id}}}) + return 0 + +def validate_ip_port(s, p): + """ + Check if ip/port is valid with ipv4 + """ + if s == 'localhost': + s = '127.0.0.1' + a = s.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + if p < 0 or p > 65535: + return False + return True + +db = MongoDB() \ No newline at end of file diff --git a/lucida/commandcenter/api/Instance_api.py b/lucida/commandcenter/api/Instance_api.py new file mode 100644 index 000000000..e11010290 --- /dev/null +++ b/lucida/commandcenter/api/Instance_api.py @@ -0,0 +1,96 @@ +from flask import * +from Database import db + +instance_api = Blueprint('instance_api', __name__, template_folder='templates') + +@instance_api.route('/api/v1/instance', methods = ['GET', 'POST']) +def instance_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + pass + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if '_id' not in requestFields or 'name' not in requestFields or \ + 'host' not in requestFields or 'port' not in requestFields: + error = {'error': 'Field missing for adding instance'} + return jsonify(error), 422 + + ret, instance_id = db.add_instance(requestFields['_id'], requestFields['name'], + requestFields['host'], requestFields['port']) + + if ret == 1: + error = {'error': 'Host/port pair not valid'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Host/port pair already used'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Instance successfully added!', 'instance_id': instance_id} + return jsonify(result), 200 + + elif option == 'add_empty': + if '_id' not in requestFields: + error = {'error': 'Field missing for adding instance'} + return jsonify(error), 422 + + ret, instance_id = db.add_empty_instance(requestFields['_id']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Instance successfully added!', 'instance_id': instance_id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'instance_id' not in requestFields or \ + 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating instance'} + return jsonify(error), 422 + + ret = db.update_instance(requestFields['_id'], requestFields['instance_id'], + requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Instance not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Host/port pair is not valid'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Updated host/port has already been used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Instance successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields or 'instance_id' not in requestFields: + error = {'error': 'Field missing for deleting instance'} + return jsonify(error), 422 + + ret = db.delete_instance(requestFields['_id'], requestFields['instance_id']) + + if ret == 1: + error = {'error': 'Instance not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Instance successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/Service_api.py b/lucida/commandcenter/api/Service_api.py new file mode 100644 index 000000000..03504c34e --- /dev/null +++ b/lucida/commandcenter/api/Service_api.py @@ -0,0 +1,86 @@ +from flask import * +from Database import db + +service_api = Blueprint('service_api', __name__, template_folder='templates') + +@service_api.route('/api/v1/service', methods = ['GET', 'POST']) +def service_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + result = db.get_services() + JSON_obj = {'service_list': result} + return jsonify(JSON_obj), 200 + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if 'name' not in requestFields or 'acronym' not in requestFields or \ + 'input' not in requestFields or 'learn' not in requestFields: + error = {'error': 'Field missing for adding service'} + return jsonify(error), 422 + + ret, _id = db.add_service(requestFields['name'], requestFields['acronym'], + requestFields['input'], requestFields['learn']) + + if ret == 1: + error = {'error': 'Service name has already existed'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Service acronym has already used'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Service successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'add_empty': + ret, _id = db.add_empty_service() + + if ret == 0: + result = {'success': 'Service successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating service'} + return jsonify(error), 422 + + ret = db.update_service(requestFields['_id'], requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Updated name already used'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Updated acronym already used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Service successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields: + error = {'error': 'Field missing for deleting service'} + return jsonify(error), 422 + + ret = db.delete_service(requestFields['_id']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Service successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/Workflow_api.py b/lucida/commandcenter/api/Workflow_api.py new file mode 100644 index 000000000..8eada57bb --- /dev/null +++ b/lucida/commandcenter/api/Workflow_api.py @@ -0,0 +1,80 @@ +from flask import * +from Database import db + +workflow_api = Blueprint('workflow_api', __name__, template_folder='templates') + +@workflow_api.route('/api/v1/workflow', methods = ['GET', 'POST']) +def workflow_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + result = db.get_workflows() + JSON_obj = {'workflow_list': result} + return jsonify(JSON_obj), 200 + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if 'name' not in requestFields or 'input' not in requestFields or \ + 'classifier' not in requestFields or 'code' not in requestFields: + error = {'error': 'Field missing for adding workflow'} + return jsonify(error), 422 + + ret, _id = db.add_workflow(requestFields['name'], requestFields['input'], + requestFields['classifier'], requestFields['code']) + + if ret == 1: + error = {'error': 'Workflow name has already existed'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Workflow successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'add_empty': + ret, _id = db.add_empty_workflow() + + if ret == 0: + result = {'success': 'Workflow successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating workflow'} + return jsonify(error), 422 + + ret = db.update_workflow(requestFields['_id'], requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Workflow not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Updated name already used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Workflow successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields: + error = {'error': 'Field missing for deleting workflow'} + return jsonify(error), 422 + + ret = db.delete_workflow(requestFields['_id']) + + if ret == 1: + error = {'error': 'Workflow not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Workflow successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/__init__.py b/lucida/commandcenter/api/__init__.py new file mode 100644 index 000000000..52fb6e71a --- /dev/null +++ b/lucida/commandcenter/api/__init__.py @@ -0,0 +1 @@ +__all__ = ['Database', 'Service_api', 'Workflow_api', 'Instance_api'] \ No newline at end of file diff --git a/lucida/commandcenter/app.py b/lucida/commandcenter/app.py index 92d8f91df..2be2a6b7c 100644 --- a/lucida/commandcenter/app.py +++ b/lucida/commandcenter/app.py @@ -7,7 +7,7 @@ '/../../tools/thrift-0.9.3/lib/py/build/lib*')[0]) from controllers import * -from controllers.Parser import cmd_port +from api import * from flask import * from threading import Thread import logging @@ -25,6 +25,10 @@ app.register_blueprint(Create.create) app.register_blueprint(Learn.learn) app.register_blueprint(Infer.infer) +app.register_blueprint(ServiceRegistry.serviceregistry) +app.register_blueprint(Service_api.service_api) +app.register_blueprint(Workflow_api.workflow_api) +app.register_blueprint(Instance_api.instance_api) # Session. app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' @@ -43,7 +47,7 @@ def flask_listener(): threaded=True) def web_socket_listener(): - print 'Start web socket at ' + str(cmd_port) + print 'Start web socket at 8081' logging.basicConfig(level=logging.DEBUG, format="%(levelname)8s %(asctime)s %(message)s ") logging.debug('Starting up server') @@ -52,13 +56,13 @@ def web_socket_listener(): # For wss (with ASR capability) if os.environ.get('SECURE_HOST'): print 'Starting secure web socket' - WebSocket.Application().listen(cmd_port, ssl_options={ + WebSocket.Application().listen(8081, ssl_options={ "certfile":"certs/server.crt", "keyfile":"certs/server.key"}) # For ws (without ASR capability) else: print 'Starting non-secure web socket' - WebSocket.Application().listen(cmd_port) + WebSocket.Application().listen(8081) WebSocket.tornado.ioloop.IOLoop.instance().start() diff --git a/lucida/commandcenter/controllers/Config.py b/lucida/commandcenter/controllers/Config.py index a843337f4..962049f58 100644 --- a/lucida/commandcenter/controllers/Config.py +++ b/lucida/commandcenter/controllers/Config.py @@ -1,176 +1,176 @@ +#!/usr/bin/env python2 + from Service import Service from Graph import Graph, Node -from Parser import port_dic -from dcm import* - -# The maximum number of texts or images for each user. -# This is to prevent the server from over-loading. -MAX_DOC_NUM_PER_USER = 30 # non-negative inetegr +from pymongo import MongoClient +from Database import database +import os, sys, re +from dcm import * -# Train or load the query classifier. -# If you set it to 'load', it is assumed that -# models are already saved in `../models`. TRAIN_OR_LOAD = 'train' # either 'train' or 'load' +""" +Train or load the query classifier. +If you set it to 'load', it is assumed that +models are already saved in `../models`. +""" + +SERVICES = {} +""" +Service dict that store all available services dynamically +Host IP addresses are resolved dynamically: +either set by Kubernetes or localhost. +""" + +WFList = {} +""" +Workflow dict that store all available workflows dynamically +""" +CLASSIFIER_DESCRIPTIONS = { + 'text': {}, + 'image': {}, + 'text_image': {} +} +""" +Classifier stored for each workflow +""" +CLASSIFIER_PATH = {} +""" +Classifier data path for each workflow +""" -####################### How does a workflow work? Reference firstWorkFlow as a walkthrough example. +SESSION = {} +""" +Structure used to save the state/context across requests in a session +example: +SESSION = { : + 'graph': , + 'data': +} +""" + +LEARNERS = { 'audio' : [], 'image' : [], 'text' : [] } +""" +Store all service supporting learn +""" + +def appendServiceRequest(data,arg1,arg2,arg3): + print(data,arg1,arg2,arg3) + data.append(serviceRequestData(arg1,arg2,[unicode(arg3)])) + return data -#Contains serviceName and data to pass. Needed for batch (and thereby parallel) processing. class serviceRequestData(object): +#Contains serviceName and data to pass. Needed for batch (and thereby parallel) processing. - def __init__(self,nameOfService,argData): + def __init__(self,batchedDataName,nameOfService,argData): self.argumentData = argData self.serviceName = nameOfService - - -class workFlow(object): - def __init__(self): - self.currentState = 0; # What state on the state graph - self.isEnd = False; - self.batchedData = [] - - - - - + self.batchedDataName = batchedDataName + +class workFlow(object): + + def __init__(self): + self.currentState = 0 # What state on the state graph + self.isEnd = False + self.batchedData = [] + class firstWorkflow(workFlow): - - - def processCurrentState(self,inputModifierText,inputModifierImage): - print "Executing state logic"; - - if(self.currentState==0): - print "State 0"; - self.currentState = 1; # This decides what state to go to next - # batchedData contains a list of service Requests. The function parameter is serviceRequestData(serviceName,dataToPassToService). - # Eg. "QA",inputModifierText[0]) means to pass to QA microservice with whatever was in the inputModifierText[0] (The text from the Lucida prompt)) - self.batchedData = [serviceRequestData("QA",[unicode("How old is Johann")]),serviceRequestData("QA",inputModifierText[0])]; - return; - - if(self.currentState==1): - print "State 1"; - # [1] is being passed as the input. This value came from: serviceRequestData("QA",inputModifierText[0]) - # It is based on the positioning of the previous serviceRequestData batch. - # Eg. [0] = serviceRequestData("QA",[unicode("How old is Johann")], [1] = serviceRequestData("QA",inputModifierText[0]) - #That means the second entry from state0 is being passed to it. - self.batchedData = [serviceRequestData("QA",inputModifierText[1])] - self.isEnd = True # This indicates the workflow is complete - return; - - - -class QAWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("QA",inputModifierText[0])]; - self.isEnd = True; - return; - -class IMMWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("IMM",inputModifierImage[0])]; - self.isEnd = True; - return; - -class CAWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("CA",inputModifierText[0])]; - self.isEnd = True; - return; - -class IMCWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("IMC",inputModifierImage[0])]; - self.isEnd = True; - return; - -class FACEWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("FACE",inputModifierImage[0])]; - self.isEnd = True; - return; - -class DIGWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("DIG",inputModifierImage[0])]; - self.isEnd = True; - return; - -class ENSEMBLEWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("ENSEMBLE",inputModifierText[0])]; - self.isEnd = True; - return; - - -class MSWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("MS",inputModifierText[0])]; - self.isEnd = True; - return; - - -WFList = { - "IMMWF" : IMMWF(), - "firstWorkFlow" : firstWorkflow(), - "QAWF" : QAWF(), - "CAWF" : CAWF(), - "IMCWF" : IMCWF(), - "FACEWF" : FACEWF(), - "DIGWF" : DIGWF(), - "ENSEMBLEWF" : ENSEMBLEWF(), - "MSWF" : MSWF() - -} - - - - -# Pre-configured services. -# The ThriftClient assumes that the following services are running. -# Host IP addresses are resolved dynamically: -# either set by Kubernetes or localhost. - -SERVICES = { - 'IMM' : Service('IMM', int(port_dic["imm_port"]), 'image', 'image'), - 'QA' : Service('QA', int(port_dic["qa_port"]), 'text', 'text'), - 'CA' : Service('CA', int(port_dic["ca_port"]), 'text', None), - 'IMC' : Service('IMC', int(port_dic["imc_port"]), 'image', None), - 'FACE' : Service('FACE', int(port_dic["face_port"]), 'image', None), - 'DIG' : Service('DIG', int(port_dic["dig_port"]), 'image', None), - 'WE' : Service('WE', int(port_dic["we_port"]), 'text', None), - 'MS' : Service('MS', int(port_dic["ms_port"]), 'text', None), - } - -CLASSIFIER_DESCRIPTIONS = { - 'text' : { 'class_QA' : Graph([Node('QAWF')]), - 'class_CA' : Graph([Node('CAWF')]), - 'class_WE' : Graph([Node('WEWF')]), - 'class_MS' : Graph([Node('MSWF')]) }, - 'image' : { 'class_IMM' : Graph([Node('IMMWF')]), - 'class_IMC' : Graph([Node('IMCWF')]), - 'class_FACE' : Graph([Node('FACEWF')]), - 'class_DIG' : Graph([Node('DIGWF')]) }, - 'text_image' : { 'class_QA': Graph([Node('QAWF')]), - 'class_IMM' : Graph([Node('IMMWF')]), - 'class_IMC' : Graph([Node('IMCWF')]), - 'class_FACE' : Graph([Node('FACEWF')]), - 'class_DIG' : Graph([Node('DIGWF')]), } - } - -# TODO: Should I have this in its own Config file? -# Structure used to save the state/context across requests in a session -# example: -# SESSION = { : -# 'graph': , -# 'data': -# } -SESSION = {} + """ + How does a workflow work? Reference firstWorkFlow as a walkthrough example. + This really should not be used anymore as its too difficult to generate a workflow by hand. + Instead, use the GUI tool which will automatically compile this + """ + + def processCurrentState(self,inputModifierText,inputModifierImage): + print "Executing state logic" + + if(self.currentState==0): + print "State 0" + self.currentState = 1; # This decides what state to go to next + # batchedData contains a list of service Requests. The function parameter is serviceRequestData(serviceName,dataToPassToService). + # Eg. "QA",inputModifierText[0]) means to pass to QA microservice with whatever was in the inputModifierText[0] (The text from the Lucida prompt)) + self.batchedData = [serviceRequestData("QA",[unicode("How old is Johann")]),serviceRequestData("QA",inputModifierText[0])] + return + + if(self.currentState==1): + print "State 1" + # [1] is being passed as the input. This value came from: serviceRequestData("QA",inputModifierText[0]) + # It is based on the positioning of the previous serviceRequestData batch. + # Eg. [0] = serviceRequestData("QA",[unicode("How old is Johann")], [1] = serviceRequestData("QA",inputModifierText[0]) + #That means the second entry from state0 is being passed to it. + self.batchedData = [serviceRequestData("QA",inputModifierText[1])] + self.isEnd = True # This indicates the workflow is complete + return + +def load_config(): + """ + Update the config needed for Lucida + """ + + # Load mongodb + db = database.db + for input_t in LEARNERS: + del LEARNERS[input_t][:] + # Update service list + SERVICES.clear() + service_list = db["service_info"].find() + count_service = service_list.count() + for i in range(count_service): + service_obj = service_list[i] + acn = service_obj['acronym'] + instance = service_obj['instance'] + input_type = service_obj['input'] + learn_type = service_obj['learn'] + _id = str(service_obj['_id']) + # check if uninitialized + if acn == '': + if acn not in SERVICES: + SERVICES[acn] = 1 + else: + SERVICES[acn] += 1 + continue + # get num of available and uninitialized instance + num = len(instance) + avail_instance = [x for x in instance if x['name'] != ''] + avail = len(avail_instance) + unini = num-avail + SERVICES[acn] = Service(acn, input_type, learn_type, unini, avail, avail_instance, _id) + # update learners + if learn_type == 'none': + pass + else: + LEARNERS[learn_type].append(_id) + + # Update workflow list, current only support single service workflow + for input_t in CLASSIFIER_DESCRIPTIONS: + CLASSIFIER_DESCRIPTIONS[input_t].clear() + WFList.clear() + workflow_list = db["workflow_info"].find() + count_workflow = workflow_list.count() + for i in range(count_workflow): + workflow_obj = workflow_list[i] + name = workflow_obj['name'] + input_list = workflow_obj['input'] + classifier = workflow_obj['classifier'] + # check if uninitialized + if name == '': + continue + CLASSIFIER_PATH['class_'+name] = classifier + code = workflow_obj['code'] + exec(code) + for input_t in input_list: + CLASSIFIER_DESCRIPTIONS[input_t]['class_'+name] = Graph([Node(name)]) + if len(input_list) == 2: + CLASSIFIER_DESCRIPTIONS['text_image']['class_'+name] = Graph([Node(name)]) + WFList[name] = eval(name+"()") + return 0 + +def get_service_withid(_id): + for service in SERVICES: + if service == '': + continue + if SERVICES[service]._id == _id: + return SERVICES[service] + +load_config() diff --git a/lucida/commandcenter/controllers/ConfigChecker.py b/lucida/commandcenter/controllers/ConfigChecker.py.backup similarity index 100% rename from lucida/commandcenter/controllers/ConfigChecker.py rename to lucida/commandcenter/controllers/ConfigChecker.py.backup diff --git a/lucida/commandcenter/controllers/Create.py b/lucida/commandcenter/controllers/Create.py index bd73e3f7c..02ebbd6eb 100644 --- a/lucida/commandcenter/controllers/Create.py +++ b/lucida/commandcenter/controllers/Create.py @@ -1,7 +1,11 @@ from flask import * +from QueryClassifier import query_classifier from AccessManagement import login_required from ThriftClient import thrift_client from Service import Service +from Utilities import log +import Config +import socket create = Blueprint('create', __name__, template_folder='templates') @@ -9,14 +13,42 @@ @login_required def create_route(): options = {} + if request.method == 'POST': + if 'request' in request.form: + if request.form['request'] == 'Update': + Config.load_config() + query_classifier.__init__(Config.TRAIN_OR_LOAD, Config.CLASSIFIER_DESCRIPTIONS) + try: - # Retrieve pre-configured services. services_list = [] for service in thrift_client.SERVICES.values(): if isinstance(service, Service): - host, port = service.get_host_port() - services_list.append((service.name, host, port)) - options['service_list']= sorted(services_list, key=lambda i: i[0]) + services_list.append(service.name) + else: + for i in range(service): + services_list.append('(undef)') + options['service_list'] = sorted(services_list, key=lambda i: i[0]) + # Retrieve pre-configured services. + instances_list = [] + for service in thrift_client.SERVICES.values(): + if isinstance(service, Service): + for i in range(service.num): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + name = service.instance[i]['name'] + host = service.instance[i]['host'] + port = service.instance[i]['port'] + result = 1 + sock.settimeout(1) + result = sock.connect_ex((host, port)) + sock.settimeout(None) + if result == 0: + instances_list.append((service.name, name, host, port, "running")) + else: + instances_list.append((service.name, name, host, port, "stop")) + sock.close() + for i in range(service.unini): + instances_list.append((service.name, '(undef)', '', '', '')) + options['instance_list'] = sorted(instances_list, key=lambda i: i[0]) except Exception as e: log(e) options['error'] = e diff --git a/lucida/commandcenter/controllers/Database.py b/lucida/commandcenter/controllers/Database.py index 8c81af8fd..36aba118a 100644 --- a/lucida/commandcenter/controllers/Database.py +++ b/lucida/commandcenter/controllers/Database.py @@ -3,9 +3,14 @@ from base64 import b64encode from Utilities import log import os -import Config from Memcached import memcached +MAX_DOC_NUM_PER_USER = 30 +""" +MAX_DOC_NUM_PER_USER = 30 # non-negative integer +The maximum number of texts or images for each user. +This is to prevent the server from over-loading. +""" class Database(object): # Name of the algorithm to use for password encryption. @@ -92,27 +97,27 @@ def get_username(self, interface, interface_uid): return None # Adds the uploaded image. - def add_image(self, username, image_data, label, image_id): + def add_image(self, username, image_data, label, image_id, _id): self.get_image_collection(username).insert_one( {'label': label, 'data': b64encode(image_data), # encoded - 'image_id': image_id}) + 'image_id': image_id, 'service_id': _id}) # Deletes the specified image. - def delete_image(self, username, image_id): - self.get_image_collection(username).remove({'image_id': image_id}) + def delete_image(self, username, image_id, _id): + self.get_image_collection(username).remove({'image_id': image_id, 'service_id': _id}) # Returns all the images by username. - def get_images(self, username): + def get_images(self, username, _id): log('Retrieving all images from images_' + username) # Notice image['data'] was encoded using Base64. - return [image for image in self.get_image_collection(username).find({}, { '_id': 0 })] + return [image for image in self.get_image_collection(username).find({'service_id': _id}, { '_id': 0 })] # Checks whether the user can add one more image. def check_add_image(self, username): if self.get_image_collection(username).count() >= \ - Config.MAX_DOC_NUM_PER_USER: + MAX_DOC_NUM_PER_USER: raise RuntimeError('Sorry. You can only add ' + - str(Config.MAX_DOC_NUM_PER_USER) + \ + str(MAX_DOC_NUM_PER_USER) + \ ' images at most') # Returns the number of images by username. def count_images(self, username): @@ -120,27 +125,27 @@ def count_images(self, username): return self.get_image_collection(username).count() # Adds the knowledge text. - def add_text(self, username, text_type, text_data, text_id): + def add_text(self, username, text_type, text_data, text_id, _id): self.get_text_collection(username).insert_one( {'type': text_type, 'text_data': text_data, - 'text_id': text_id}) + 'text_id': text_id, 'service_id': _id}) # Deletes the knowledge text. - def delete_text(self, username, text_id): + def delete_text(self, username, text_id, _id): self.get_text_collection(username).delete_one( - {'text_id': text_id}) + {'text_id': text_id, 'service_id': _id}) # Returns the knowledge text by username. - def get_text(self, username): + def get_text(self, username, _id): log('Retrieving text from text_' + username) - return [text for text in self.get_text_collection(username).find({}, { '_id': 0 })] + return [text for text in self.get_text_collection(username).find({'service_id': _id}, { '_id': 0 })] # Checks whether the user can add one more piece of text. def check_add_text(self, username): if self.get_text_collection(username).count() >= \ - Config.MAX_DOC_NUM_PER_USER: + MAX_DOC_NUM_PER_USER: raise RuntimeError('Sorry. You can only add ' + - str(Config.MAX_DOC_NUM_PER_USER) + \ + str(MAX_DOC_NUM_PER_USER) + \ ' pieces of text at most') database = Database() diff --git a/lucida/commandcenter/controllers/Infer.py b/lucida/commandcenter/controllers/Infer.py index be7f7d42f..e9e48e413 100644 --- a/lucida/commandcenter/controllers/Infer.py +++ b/lucida/commandcenter/controllers/Infer.py @@ -4,7 +4,6 @@ from ThriftClient import thrift_client from QueryClassifier import query_classifier from Utilities import log, check_image_extension -from Parser import port_dic import Config import os import json @@ -14,10 +13,6 @@ @login_required def generic_infer_route(form, upload_file): options = {} - if os.environ.get('ASR_ADDR_PORT'): - options['asr_addr_port'] = os.environ.get('ASR_ADDR_PORT') - else: - options['asr_addr_port'] = 'ws://localhost:' + port_dic["cmd_port"] try: # Deal with POST requests. if request.method == 'POST': @@ -47,15 +42,15 @@ def generic_infer_route(form, upload_file): options['dates'] = options['result'] options['result'] = None except Exception as e: - log(e) - options['errno'] = 500 - options['error'] = str(e) - if 'code' in e and re.match("^4\d\d$", str(e.code)): - options['errno'] = e.code - if str(e) == 'TSocket read 0 bytes': - options['error'] = 'Back-end service encountered a problem' - if str(e).startswith('Could not connect to'): - options['error'] = 'Back-end service is not running' + log(e) + options['errno'] = 500 + options['error'] = str(e) + if 'code' in e and re.match("^4\d\d$", str(e.code)): + options['errno'] = e.code + if str(e) == 'TSocket read 0 bytes': + options['error'] = 'Back-end service encountered a problem' + if str(e).startswith('Could not connect to'): + options['error'] = 'Back-end service is not running' return options @infer.route('/infer', methods=['GET', 'POST']) @@ -65,7 +60,7 @@ def infer_route(): if os.environ.get('ASR_ADDR_PORT'): options['asr_addr_port'] = os.environ.get('ASR_ADDR_PORT') else: - options['asr_addr_port'] = 'ws://localhost:' + port_dic["cmd_port"] + options['asr_addr_port'] = 'ws://localhost:8081' if request.method == 'POST': options = generic_infer_route(request.form, request.files['file'] if 'file' in request.files else None) return render_template('infer.html', **options) @@ -82,5 +77,5 @@ def api_infer_route(): options = generic_infer_route(request.form, request.files['file'] if 'file' in request.files else None) if 'errno' in options: - return json.dumps(options), options['errno'] + return json.dumps(options), options['errno'] return json.dumps(options), 200 diff --git a/lucida/commandcenter/controllers/Learn.py b/lucida/commandcenter/controllers/Learn.py index 4ea454644..6dc902473 100644 --- a/lucida/commandcenter/controllers/Learn.py +++ b/lucida/commandcenter/controllers/Learn.py @@ -5,6 +5,7 @@ from Database import database from ThriftClient import thrift_client from Utilities import log, check_image_extension, check_text_input +import Config import re learn = Blueprint('learn', __name__, template_folder='templates') @@ -17,6 +18,7 @@ def generic_learn_route(op, form, upload_file): if op == 'add_image': image_type = 'image' label = form['label'] + _id = form['_id'] # Check the uploaded image. if upload_file.filename == '': raise RuntimeError('Empty file is not allowed') @@ -32,21 +34,23 @@ def generic_learn_route(op, form, upload_file): # Send the image to IMM. upload_file.close() thrift_client.learn_image(username, image_type, image_data, - image_id) + image_id, _id) # Add the image into the database. - database.add_image(username, image_data, label, image_id) + database.add_image(username, image_data, label, image_id, _id) # Delete image knowledge. elif op == 'delete_image': image_type = 'unlearn' image_id = form['image_id'] + _id = form['_id'] # Send the unlearn request to IMM. - thrift_client.learn_image(username, image_type, '', image_id) + thrift_client.learn_image(username, image_type, '', image_id, _id) # Delete the image from the database. - database.delete_image(username, image_id) + database.delete_image(username, image_id, _id) # Add text knowledge. elif op == 'add_text' or op == 'add_url': text_type = 'text' if op == 'add_text' else 'url' text_data = form['knowledge'] + _id = form['_id'] # Check the text knowledge. check_text_input(text_data) # Check whether the user can add one more piece of text. @@ -56,17 +60,18 @@ def generic_learn_route(op, form, upload_file): str(datetime.datetime.now())).hexdigest() # Send the text to QA. thrift_client.learn_text(username, text_type, - text_data, text_id) + text_data, text_id, _id) # Add the text knowledge into the database. - database.add_text(username, text_type, text_data, text_id) + database.add_text(username, text_type, text_data, text_id, _id) # Delete text knowledge. elif op == 'delete_text': text_type = 'unlearn' text_id = form['text_id'] + _id = form['_id'] # Send the unlearn request to QA. - thrift_client.learn_text(username, text_type, '', text_id) + thrift_client.learn_text(username, text_type, '', text_id, _id) # Delete the text from into the database. - database.delete_text(username, text_id) + database.delete_text(username, text_id, _id) else: raise RuntimeError('Did you click the button?') except Exception as e: @@ -85,13 +90,48 @@ def generic_learn_route(op, form, upload_file): @login_required def learn_route(): options = {} + # Deal with POST requests. if request.method == 'POST': options = generic_learn_route(request.form['op'], request.form, request.files['file'] if 'file' in request.files else None) + + # home page + if '_id' not in request.args: + options['text_list'] = [] + options['image_list'] = [] + for _id in Config.LEARNERS['text']: + service = Config.get_service_withid(_id) + options['text_list'].append({'name': service.name, '_id': _id}) + for _id in Config.LEARNERS['image']: + service = Config.get_service_withid(_id) + options['image_list'].append({'name': service.name, '_id': _id}) + return render_template('learn_home.html', **options) + try: - # Retrieve knowledge. - options['pictures'] = database.get_images(session['username']) - options['text'] = database.get_text(session['username']) + _id = request.args['_id'] + service = Config.get_service_withid(_id) + + options['_id'] = _id + options['service_name'] = service.name + + if _id in Config.LEARNERS['text']: + options['text_able'] = 1 + else: + options['text_able'] = 0 + + # check if text field is necessary + if _id in Config.LEARNERS['image']: + options['image_able'] = 1 + else: + options['image_able'] = 0 + + if options['text_able'] == 0 and options['image_able'] == 0: + abort(404) + + if options['text_able'] == 0: + options['pictures'] = database.get_images(session['username'], _id) + else: + options['text'] = database.get_text(session['username'], _id) except Exception as e: log(e) options['errno'] = 500 @@ -103,26 +143,20 @@ def learn_route(): def api_learn_add_del_route(op): if not op in allowed_endpoints: abort(404) - session['username'] = database.get_username(request.form['interface'], request.form['username']) - if session['username'] == None: - abort (403) - - session['logged_in'] = True - print '@@@@@@@@', session['username'] options = {} if not op == 'query': options = generic_learn_route(op, request.form, request.files['file'] if 'file' in request.files else None) else: try: - # Retrieve knowledge. + # Retrieve knowledge. if 'type' in request.form and request.form['type'] == 'text': - options['text'] = database.get_text(session['username']) + options['text'] = database.get_text(session['username'], request.form['_id']) elif 'type' in request.form and request.form['type'] == 'image': - options['pictures'] = database.get_images(session['username']) + options['pictures'] = database.get_images(session['username'], request.form['_id']) else: - options['pictures'] = database.get_images(session['username']) - options['text'] = database.get_text(session['username']) + options['pictures'] = database.get_images(session['username'], request.form['_id']) + options['text'] = database.get_text(session['username'], request.form['_id']) except Exception as e: log(e) options['errno'] = 500 diff --git a/lucida/commandcenter/controllers/Parser.py b/lucida/commandcenter/controllers/Parser.py deleted file mode 100644 index 569dc28f4..000000000 --- a/lucida/commandcenter/controllers/Parser.py +++ /dev/null @@ -1,21 +0,0 @@ -import ConfigParser, sys - - -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../config.properties"))) -port_dic = dict(cp.items('asection')) -cmd_port = int(port_dic['cmd_port']) diff --git a/lucida/commandcenter/controllers/QueryClassifier.py b/lucida/commandcenter/controllers/QueryClassifier.py index d11b06e00..e0f8f26bb 100644 --- a/lucida/commandcenter/controllers/QueryClassifier.py +++ b/lucida/commandcenter/controllers/QueryClassifier.py @@ -27,6 +27,10 @@ def __init__(self, query_class_name_in): def predict(self, speech_input): return [self.query_class_name] + +class EmptyClassifier(object): + def predict(self, speech_input): + return [] class QueryClassifier(object): # Constructor. @@ -54,12 +58,16 @@ def train(self, input_type, query_classes): current_dir = os.path.abspath(os.path.dirname(__file__)) # If there is no or only one possible outcomes for the input type, # there is no need to train any classifier. - if len(query_classes) <= 1: + if len(query_classes) == 0: + return EmptyClassifier() + if len(query_classes) == 1: return DummyClassifier(query_classes.keys()[0]) # Build DataFrame by going through all data files. data = DataFrame({'text': [], 'class': []}) for query_class_name in query_classes: - path = current_dir + '/../data/' + query_class_name + '.txt' + path = Config.CLASSIFIER_PATH[query_class_name] + if not os.path.isfile(path): + raise RuntimeError('Query classifer data file cannot found!') log('Opening ' + path) lines = [line.rstrip('\n') for line in open(path)] rows = [] @@ -133,6 +141,8 @@ def predict(self, speech_input, image_input): raise RuntimeError('Text and image cannot be both empty') # Convert speech_input to a single-element list. class_predicted = self.classifiers[input_type].predict([speech_input]) + if len(class_predicted) == 0: + raise RuntimeError('Input type has no service available') class_predicted = class_predicted[0] # ndarray to string log('Query classified as ' + class_predicted) return self.CLASSIFIER_DESCRIPTIONS[input_type][class_predicted] diff --git a/lucida/commandcenter/controllers/Service.py b/lucida/commandcenter/controllers/Service.py index d4fd9ac07..d5da4f792 100644 --- a/lucida/commandcenter/controllers/Service.py +++ b/lucida/commandcenter/controllers/Service.py @@ -2,29 +2,35 @@ from Utilities import log class Service(object): - LEARNERS = { 'audio' : [], 'image' : [], 'text' : [] } # Constructor. - def __init__(self, name, port, input_type, learn_type): + def __init__(self, name, input_type, learn_type, unini, avail, avail_instance, _id): self.name = name - self.port = port + self.count = 0 if not (input_type == 'text' or input_type == 'image'): print 'Can only process text and image' exit() self.input_type = input_type - if not learn_type is None: - if not learn_type in Service.LEARNERS: - print 'Unrecognized learn_type' - exit() - Service.LEARNERS[learn_type].append(self) + self.learn_type = learn_type + self.unini = unini + self.num = avail + self.instance = avail_instance + self._id = _id def get_host_port(self): try: - host = 'localhost' - tcp_addr = os.environ.get(self.name + '_PORT_' + str(self.port) + '_TCP_ADDR') - if tcp_addr: - log('TCP address is resolved to ' + tcp_addr) - host = tcp_addr - return host, self.port + if self.num <= 0: + raise RuntimeError('No available instance for service ' + self.name) + cur_host = self.instance[self.count]['host'] + cur_port = self.instance[self.count]['port'] + self.count = (self.count + 1)%self.num + log('loop ' + str(self.count)) + return cur_host, cur_port except Exception: raise RuntimeError('Cannot access service ' + self.name) + def get_host_port_withid(self, instance_id): + for obj in self.instance: + if obj['id'] == instance_id: + return obj['host'], obj['port'] + + diff --git a/lucida/commandcenter/controllers/ServiceRegistry.py b/lucida/commandcenter/controllers/ServiceRegistry.py new file mode 100644 index 000000000..b48051b3b --- /dev/null +++ b/lucida/commandcenter/controllers/ServiceRegistry.py @@ -0,0 +1,9 @@ +from flask import * +from AccessManagement import login_required + +serviceregistry = Blueprint('serviceregistry', __name__, template_folder='templates') + +@serviceregistry.route('/serviceregistry', methods=['GET', 'POST']) +@login_required +def serviceregistry_route(): + return render_template('serviceregistry.html') diff --git a/lucida/commandcenter/controllers/ThriftClient.py b/lucida/commandcenter/controllers/ThriftClient.py index 85010eacc..8f41be9d6 100644 --- a/lucida/commandcenter/controllers/ThriftClient.py +++ b/lucida/commandcenter/controllers/ThriftClient.py @@ -1,29 +1,23 @@ -from lucidatypes.ttypes import QueryInput, QuerySpec -from lucidaservice import LucidaService -from dcm import* from flask import* - -from Config import WFList +import threading +import os +import sys from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +from lucidatypes.ttypes import QueryInput, QuerySpec +from lucidaservice import LucidaService +from dcm import* +from Config import WFList from Utilities import log from Database import database import Config -import os -import sys + reload(sys) sys.setdefaultencoding('utf8') # to solve the unicode error - -import threading - - - - - #This is basically the thread starter function class FuncThread(threading.Thread): def __init__(self, target, *args): @@ -33,8 +27,6 @@ def __init__(self, target, *args): def run(self): self._target(*self._args) - - class ThriftClient(object): # Constructor. def __init__(self, SERVICES): @@ -54,8 +46,7 @@ def create_query_spec(self, name, query_input_list): query_spec.content = query_input_list return query_spec - def get_client_transport(self, service): - host, port = service.get_host_port() + def get_client_transport(self, host, port): print (host,port) transport = TTransport.TFramedTransport(TSocket.TSocket(host, port)) protocol = TBinaryProtocol.TBinaryProtocol(transport) @@ -65,34 +56,44 @@ def get_client_transport(self, service): def send_query(self, LUCID, service_name, query_input_list): query_spec = self.create_query_spec('query', query_input_list) service = self.SERVICES[service_name] - client, transport = self.get_client_transport(service) + host = query_input_list[0].tags[0] + port = int(query_input_list[0].tags[1]) + client, transport = self.get_client_transport(host, port) log('Sending infer request to ' + service.name) result = client.infer(str(LUCID), query_spec) transport.close() return result - - def learn_image(self, LUCID, image_type, image_data, image_id): - for service in Config.Service.LEARNERS['image']: # add concurrency? - knowledge_input = self.create_query_input( - image_type, [image_data], [image_id]) - client, transport = self.get_client_transport(service) - log('Sending learn_image request to IMM') + def learn_image(self, LUCID, image_type, image_data, image_id, _id): + knowledge_input = self.create_query_input( + image_type, [image_data], [image_id]) + service = Config.get_service_withid(_id) + if service.num == 0: + raise RuntimeError('No available instance to learn knowledge') + for obj in service.instance: + instance_id = obj['id'] + host, port = service.get_host_port_withid(instance_id) + client, transport = self.get_client_transport(host, port) + log('Sending learn_image request to ' + service.name) client.learn(str(LUCID), self.create_query_spec('knowledge', [knowledge_input])) transport.close() - def learn_text(self, LUCID, text_type, text_data, text_id): - for service in Config.Service.LEARNERS['text']: # add concurrency? - knowledge_input = self.create_query_input( - text_type, [text_data], [text_id]) - client, transport = self.get_client_transport(service) - log('Sending learn_text request to QA') + def learn_text(self, LUCID, text_type, text_data, text_id, _id): + knowledge_input = self.create_query_input( + text_type, [text_data], [text_id]) + service = Config.get_service_withid(_id) + if service.num == 0: + raise RuntimeError('No available instance to learn knowledge') + for obj in service.instance: + instance_id = obj['id'] + host, port = service.get_host_port_withid(instance_id) + client, transport = self.get_client_transport(host, port) + log('Sending learn_text request to ' + service.name) client.learn(str(LUCID), self.create_query_spec('knowledge', [knowledge_input])) transport.close() - - + # Example usage def executeThreadServiceRequest(self,service_name, inputData, LUCID, threadIDValue): print("Thread ", threadIDValue, "executing", service_name, "with input", inputData) @@ -102,14 +103,9 @@ def executeThreadServiceRequest(self,service_name, inputData, LUCID, threadIDVal query_input_list = [self.create_query_input(service.input_type, inputData, tag_list)] resultText = self.send_query(LUCID, service_name, query_input_list) self.threadResults.insert(threadIDValue, resultText) - - - - -# TODO: split function into separate functions (DCM, creating QuerySpec) + def infer(self, LUCID, workflow_name, text_data, image_data): - response_data = { 'text': text_data, 'image': image_data } self.threadResults = [] @@ -120,16 +116,18 @@ def infer(self, LUCID, workflow_name, text_data, image_data): resultText = response_data['text'] resultImage = [response_data['image']] - + passArgs = dict() while not workflow.isEnd: - + batchedDataReturn = dict() + print "-------------NEXT ITERATION:STATE" + str(workflow.currentState) i = 0 for x in resultText: - resultText[i] = [unicode(resultText)] # Text information must be unicode'd and array'd to be properly passed. IMAGE DATA DOES NOT HAVE THIS DONE TO IT. + resultText[i] = unicode(x) # Text information must be unicode'd and array'd to be properly passed. IMAGE DATA DOES NOT HAVE THIS DONE TO IT. i+= 1 # Processes the current workflow state, and in the process finds if this is the final stage or if next stage exists. - workflow.processCurrentState(resultText,resultImage) + print("Acquiring Batch Request") + workflow.processCurrentState(1,batchedDataReturn,passArgs,resultText,resultImage) resultText = [] resultImage = [] @@ -143,6 +141,8 @@ def infer(self, LUCID, workflow_name, text_data, image_data): threadList.append(FuncThread(self.executeThreadServiceRequest, x.serviceName, x.argumentData, LUCID,threadID)) threadList[threadID].start() threadID+=1 + + print("Executed batch request") threadID = 0 #This is where batched execution joins together @@ -150,12 +150,13 @@ def infer(self, LUCID, workflow_name, text_data, image_data): threadList[threadID].join() print "============ThreadID" + str(threadID) print "Output:" + self.threadResults[threadID] + batchedDataReturn[x.batchedDataName] = self.threadResults[threadID] resultText.insert(threadID, self.threadResults[threadID]) threadID+=1 - - - + print("Do stuff after batch request") + workflow.processCurrentState(0,batchedDataReturn,passArgs,resultText,resultImage) + return resultText[0] thrift_client = ThriftClient(Config.SERVICES) diff --git a/lucida/commandcenter/controllers/WebSocket.py b/lucida/commandcenter/controllers/WebSocket.py index 5c6c77398..976f85af8 100644 --- a/lucida/commandcenter/controllers/WebSocket.py +++ b/lucida/commandcenter/controllers/WebSocket.py @@ -24,10 +24,8 @@ import tornado.gen import tornado.concurrent -from Parser import cmd_port - from tornado.options import define -define("port", default=cmd_port, help="run on the given port", type=int) +define("port", default=8081, help="run on the given port", type=int) STATUS_EOS = -1 STATUS_SUCCESS = 0 diff --git a/lucida/commandcenter/controllers/__init__.py b/lucida/commandcenter/controllers/__init__.py index fd97c3acd..0bfef4b3c 100644 --- a/lucida/commandcenter/controllers/__init__.py +++ b/lucida/commandcenter/controllers/__init__.py @@ -1,3 +1,3 @@ __all__ = ['Main', 'AccessManagement', 'WebSocket', 'Service', 'Graph', - 'ThriftClient', 'Create', 'Learn', 'Infer', 'Parser', + 'ThriftClient', 'Create', 'Learn', 'Infer', 'ServiceRegistry', 'QueryClassifier', 'Config', 'User', 'Utilities', 'Database', 'Memcached', 'Decision'] diff --git a/lucida/commandcenter/data/class_CA.txt b/lucida/commandcenter/data/class_CAWF.txt similarity index 100% rename from lucida/commandcenter/data/class_CA.txt rename to lucida/commandcenter/data/class_CAWF.txt diff --git a/lucida/commandcenter/data/class_DIG.txt b/lucida/commandcenter/data/class_DIGWF.txt similarity index 100% rename from lucida/commandcenter/data/class_DIG.txt rename to lucida/commandcenter/data/class_DIGWF.txt diff --git a/lucida/commandcenter/data/class_FACE.txt b/lucida/commandcenter/data/class_FACEWF.txt similarity index 100% rename from lucida/commandcenter/data/class_FACE.txt rename to lucida/commandcenter/data/class_FACEWF.txt diff --git a/lucida/commandcenter/data/class_IMC.txt b/lucida/commandcenter/data/class_IMCWF.txt similarity index 100% rename from lucida/commandcenter/data/class_IMC.txt rename to lucida/commandcenter/data/class_IMCWF.txt diff --git a/lucida/commandcenter/data/class_IMM.txt b/lucida/commandcenter/data/class_IMMWF.txt similarity index 100% rename from lucida/commandcenter/data/class_IMM.txt rename to lucida/commandcenter/data/class_IMMWF.txt diff --git a/lucida/commandcenter/data/class_MS.txt b/lucida/commandcenter/data/class_MSWF.txt similarity index 100% rename from lucida/commandcenter/data/class_MS.txt rename to lucida/commandcenter/data/class_MSWF.txt diff --git a/lucida/commandcenter/data/class_QA.txt b/lucida/commandcenter/data/class_QAWF.txt similarity index 100% rename from lucida/commandcenter/data/class_QA.txt rename to lucida/commandcenter/data/class_QAWF.txt diff --git a/lucida/commandcenter/data/class_WE.txt b/lucida/commandcenter/data/class_WEWF.txt similarity index 100% rename from lucida/commandcenter/data/class_WE.txt rename to lucida/commandcenter/data/class_WEWF.txt diff --git a/lucida/commandcenter/data/class_WFTaco.txt b/lucida/commandcenter/data/class_WFTaco.txt new file mode 100644 index 000000000..fab509ba7 --- /dev/null +++ b/lucida/commandcenter/data/class_WFTaco.txt @@ -0,0 +1,141 @@ +Who is the current president of the United States of America? +Who wrote the Walden? +Who founded Apple, Inc.? +Who is the first man on the moon? +Who founded the Black Panthers organization? +Who discovered prions? +Who are the members of the Rat Pack? +Who did Queen Victoria marry? +Who were the leaders of the French Revolution? +Who ended the Vietnam War? +When did Xi JinPing born? +When did Albert Einstein die? +When will the next iPhone come out? +When was the Munich agreement made? +When is Easter Sunday in 2014? +When was the first flight of the Wright brothers? +When was the Halley's Comet discovered? +When did Mike Tyson win the title? +When was the University of Michigan founded? +When did Hawaii become a state? +Where is Steve Jobs buried? +Where was Jack Ma born? +Where was the first KFC opened? +Where are the Headquarters of Facebook located? +Where does Barack Obama live? +Where will the 2016 Summer Olympics be held? +Where did ISIS come from? +Where is the River Nile located? +Where is an adults-only Club Med? +Where did Christopher Columbus die? +What was the number one song on the Billboard Top 100 in 2014? +What is the most watched video on YouTube of all time? +What nationality is Ted Cruz? +What did Ludwig van Beethoven compose? +What is the last book of the Bible? +What is the total length of the Golden Gate Bridge? +What is the most populated city in the world? +What is the homosexual residential area in San Francisco called? +What is the Avogadro's Constant? +What sport does David Beckham play? +How many weeks are there in a year? +How many passengers does Bay Area Rapid Transit serve annually? +How many people died in World War 2? +How many employees does Google have? +How many cups are in a quart? +How much is 1 dollar to RMB (Chinese Yuan)? +How much water is there in the Lake Erie? +How much does Earth weigh? +How often does a woman have a period? +How fast does a cheetah run? +Who founded Google? +Who launched Facebook? +Who created Amazon? +Who is the vice president of the United States? +Who is the president of France? +Who is the president of China? +Who is the father of C++? +Who invented Java? +Who created Python? +Who designed the first car? +Who is the richest person in the world? +Who is the greatest philanthropist of all time? +Who discovered America? +Who discovered neutron? +Who successful predicted the regular protein secondary structures? +Who first identified and isolated DNA? +Who is the author of the Harry Potter series? +Who is the author of the Canterbury Tales? +Who wrote A Brief History of Time? +Who wrote the Declaration of Independence? +Who composed Für Elise? +Who composed Piano Concerto No.5? +Who presented the Twin Earth thought experiment? +Who led the Confederate Army in the US Civil War? +Who led the Cuban Revolution in 1961? +Who is the first emperor of the Roman Empire? +Who is the first emperor of India? +Who is the last emperor of china? +Who is the last leader of the Soviet Union? +Who gave America its name? +What is my phone number? +Hey, Lucida, gimme my cellphone numbers. +Where is Eiffel Tower located? +Tell me what my favorite resturant is. +Lucida what should I have for breakfast? +What are some healthy breakfast ideas? +Are their any health benefits to coffee? +Is coffee bad for you? +How much coffee can I drink until I get sick? +Is coffee lethal? +What is my favorite kind of coffee? +Remind me what my favorite coffee is. +Where should I get my coffee today? +What’s Taylor Swift’s newest single? +What is the last music video Taylor Swift released? +Are Calvin Harris and Taylor Swift dating? +What is Calvin Harris’s real name? +What are the first 10 numbers of the fibonacci sequence? +What’s the solution to the fibonacci sequence? +Give me the 27th fibonacci number. +What does saranghae mean? +How do you pronounce fromage? +When does BTS’s new album come out? +What does HYYH stand for? +Who is wearing a black jacket? +Who is doing this presentation? +Who is presenting her research report? +Who likes the color blue? +Who likes flowers? +Who likes reading books? +Who likes playing football? +Who loves swimming? +Who is working on Lucida? +Who are the developers of Lucida? +Who are doing research on Sirius? +Who founded click? +Who founded the company? +Who goes to California every year? +Who just went to New York? +Who is my favorite? +Who is my wife? +Who is my husband? +Who is my son? +Who is my daughter? +Who is my grand father? +Who is my grandmother? +Who is my sister? +Who is my brother? +Who is my cousin? +Who is my aunt? +Who is my uncle? +Who behaves like a clown? +Who is funny? +Who is serious? +Who is outgoing? +Who is bad at programming? +Who is good at painting? +Who wants to be president? +Who plays the violin? +Who killed Lincoln? +Who loved her? diff --git a/lucida/commandcenter/static/js/serviceregistry.js b/lucida/commandcenter/static/js/serviceregistry.js new file mode 100644 index 000000000..e69de29bb diff --git a/lucida/commandcenter/templates/create.html b/lucida/commandcenter/templates/create.html index d5cd79d34..c4b99f20f 100644 --- a/lucida/commandcenter/templates/create.html +++ b/lucida/commandcenter/templates/create.html @@ -12,26 +12,59 @@ +
+ + + + {% for i in range(service_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
{{ service_list[i] }}
+
+ +
+
+

List of Service Instances

+
+
+
+ + - {% for i in service_list %} + {% for i in instance_list %} + + {% endfor %}
Service nameName Host PortAvailablity
{{ i[0] }} {{ i[1] }} {{ i[2] }}{{ i[3] }}{{ i[4] }}
+
+
+
+ +
+
+
+ {% endblock %} diff --git a/lucida/commandcenter/templates/learn.html b/lucida/commandcenter/templates/learn.html index 58a6d9eb6..ec2687492 100644 --- a/lucida/commandcenter/templates/learn.html +++ b/lucida/commandcenter/templates/learn.html @@ -13,6 +13,13 @@

{{ error }}

 {% endif %} +
+
+

Service: {{ service_name }}

+
+
+ + {% if text_able == 1 %}

Add text knowledge:

@@ -22,7 +29,8 @@
- + +
@@ -39,7 +47,8 @@
- + +
@@ -58,6 +67,8 @@ + +
  • {% if i['type'] == 'url' %}{% endif %}{{ i['text_data'] }}{% if i['type'] == 'url' %}{% endif %}
    • @@ -67,7 +78,9 @@ {% endfor %} + {% endif %} + {% if image_able == 1 %}

    Add image knowledge:

    @@ -77,7 +90,8 @@
    - + +

    Upload your picture!

    @@ -102,6 +116,8 @@ + +
    @@ -110,8 +126,13 @@
    {% endfor %} + {% endif %}
    + +
    + Back +


    {% endblock %} diff --git a/lucida/commandcenter/templates/learn_home.html b/lucida/commandcenter/templates/learn_home.html new file mode 100644 index 000000000..7908a66ab --- /dev/null +++ b/lucida/commandcenter/templates/learn_home.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block content %} + + + +
    +
    +

    Main » Learn

    +
    + + {% if error %} +

    {{ error }}

     + {% endif %} + +
    +
    +

    Text:

    +
    +
    + +
    + + + + {% for i in range(text_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
    {{ text_list[i]["name"] }}
    +
    + +
    +
    +

    Image:

    +
    +
    + +
    + + + + {% for i in range(image_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
    {{ image_list[i]["name"] }}
    +
    + +
    + +

    +{% endblock %} diff --git a/lucida/commandcenter/templates/serviceregistry.html b/lucida/commandcenter/templates/serviceregistry.html new file mode 100644 index 000000000..d990b487d --- /dev/null +++ b/lucida/commandcenter/templates/serviceregistry.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block content %} + + +
    +
    +

    Main » Service Registry

    +
    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/lucida/config.properties b/lucida/config.properties deleted file mode 100644 index ec9033926..000000000 --- a/lucida/config.properties +++ /dev/null @@ -1,12 +0,0 @@ -# This is the port configuration of LUCIDA -# WARNING: Never assign one port to more than one service. -# TODO: Adding your own service port. -CMD_PORT=8081 -QA_PORT=8082 -IMM_PORT=8083 -CA_PORT=8084 -IMC_PORT=8085 -FACE_PORT=8086 -DIG_PORT=8087 -WE_PORT=8088 -MS_PORT=8089 diff --git a/lucida/djinntonic/dig/DIGServer.cpp b/lucida/djinntonic/dig/DIGServer.cpp index 639fdbc6d..0395fab5e 100644 --- a/lucida/djinntonic/dig/DIGServer.cpp +++ b/lucida/djinntonic/dig/DIGServer.cpp @@ -5,8 +5,8 @@ #include "DIGHandler.h" #include -#include "Parser.h" #include +#include DEFINE_int32(num_of_threads, 4, @@ -16,21 +16,17 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; //using namespace facebook::windtunnel::treadmill::services::dig; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("DIG_PORT", portVal)) { - cout << "DIG port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -41,6 +37,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "DIG at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/dig/Makefile b/lucida/djinntonic/dig/Makefile index 71e1a8edf..26b0b95de 100644 --- a/lucida/djinntonic/dig/Makefile +++ b/lucida/djinntonic/dig/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./DIGServer + @if [ "$(port)" != "" ]; then \ + ./DIGServer $(port); \ + fi start_test: - cd test && ./DIGClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./DIGClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/dig/Parser.h b/lucida/djinntonic/dig/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/dig/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/dig/test/DIGClient.cpp b/lucida/djinntonic/dig/test/DIGClient.cpp index db2d133e3..1d4a40084 100644 --- a/lucida/djinntonic/dig/test/DIGClient.cpp +++ b/lucida/djinntonic/dig/test/DIGClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("DIG_PORT", portVal)) { - cout << "DIG port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/dig/test/Makefile b/lucida/djinntonic/dig/test/Makefile index 21ad08bd6..cad8de2b6 100644 --- a/lucida/djinntonic/dig/test/Makefile +++ b/lucida/djinntonic/dig/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = DIGClient diff --git a/lucida/djinntonic/face/FACEServer.cpp b/lucida/djinntonic/face/FACEServer.cpp index a65704932..564ab3430 100644 --- a/lucida/djinntonic/face/FACEServer.cpp +++ b/lucida/djinntonic/face/FACEServer.cpp @@ -5,7 +5,7 @@ #include "FACEHandler.h" #include -#include "Parser.h" +#include #include DEFINE_int32(num_of_threads, @@ -16,20 +16,16 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("FACE_PORT", portVal)) { - cout << "FACE port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -40,6 +36,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "FACE at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/face/Makefile b/lucida/djinntonic/face/Makefile index 2c29610b5..df944aa43 100644 --- a/lucida/djinntonic/face/Makefile +++ b/lucida/djinntonic/face/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./FACEServer + @if [ "$(port)" != "" ]; then \ + ./FACEServer $(port); \ + fi start_test: - cd test && ./FACEClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./FACEClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/face/Parser.h b/lucida/djinntonic/face/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/face/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/face/test/FACEClient.cpp b/lucida/djinntonic/face/test/FACEClient.cpp index d006c8662..11b8708df 100644 --- a/lucida/djinntonic/face/test/FACEClient.cpp +++ b/lucida/djinntonic/face/test/FACEClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("FACE_PORT", portVal)) { - cout << "FACE port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/face/test/Makefile b/lucida/djinntonic/face/test/Makefile index 5ac008719..6afa69c66 100644 --- a/lucida/djinntonic/face/test/Makefile +++ b/lucida/djinntonic/face/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = FACEClient diff --git a/lucida/djinntonic/imc/IMCServer.cpp b/lucida/djinntonic/imc/IMCServer.cpp index f923a4166..4a2b19c87 100644 --- a/lucida/djinntonic/imc/IMCServer.cpp +++ b/lucida/djinntonic/imc/IMCServer.cpp @@ -5,7 +5,7 @@ #include "IMCHandler.h" #include -#include "Parser.h" +#include #include DEFINE_int32(num_of_threads, @@ -16,21 +16,17 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; //using namespace facebook::windtunnel::treadmill::services::imc; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMC_PORT", portVal)) { - cout << "IMC port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -41,6 +37,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "IMC at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/imc/Makefile b/lucida/djinntonic/imc/Makefile index 3ef3f4f2c..15912f992 100644 --- a/lucida/djinntonic/imc/Makefile +++ b/lucida/djinntonic/imc/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./IMCServer + @if [ "$(port)" != "" ]; then \ + ./IMCServer $(port); \ + fi start_test: - cd test && ./IMCClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./IMCClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/imc/Parser.h b/lucida/djinntonic/imc/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/imc/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/imc/test/IMCClient.cpp b/lucida/djinntonic/imc/test/IMCClient.cpp index ff1e0c03e..1b9270b3c 100644 --- a/lucida/djinntonic/imc/test/IMCClient.cpp +++ b/lucida/djinntonic/imc/test/IMCClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMC_PORT", portVal)) { - cout << "IMC port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/imc/test/Makefile b/lucida/djinntonic/imc/test/Makefile index 1f1008183..69942dded 100644 --- a/lucida/djinntonic/imc/test/Makefile +++ b/lucida/djinntonic/imc/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = IMCClient diff --git a/lucida/imagematching/opencv_imm/Makefile b/lucida/imagematching/opencv_imm/Makefile index b8d290bb1..1e664e087 100644 --- a/lucida/imagematching/opencv_imm/Makefile +++ b/lucida/imagematching/opencv_imm/Makefile @@ -1,11 +1,17 @@ SUBDIRS=server test include ../../../Makefile.common -start_server: - cd server; ./imm_server +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + ./imm_server $(port); \ + fi start_test: - cd test; ./imm_test + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./imm_test $(port); \ + fi docker: cp ../../lucidaservice.thrift .; \ diff --git a/lucida/imagematching/opencv_imm/server/IMMServer.cpp b/lucida/imagematching/opencv_imm/server/IMMServer.cpp index c34fff5e5..a4f77ed1e 100644 --- a/lucida/imagematching/opencv_imm/server/IMMServer.cpp +++ b/lucida/imagematching/opencv_imm/server/IMMServer.cpp @@ -8,10 +8,10 @@ DEFINE_int32(num_of_threads, "Number of threads (default: 4)"); #include "IMMHandler.h" -#include "Parser.h" #include #include #include +#include using namespace folly; using namespace apache::thrift; @@ -28,18 +28,12 @@ using std::to_string; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMM_PORT", portVal)) { - cout << "IMM port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); - } + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); diff --git a/lucida/imagematching/opencv_imm/server/Parser.h b/lucida/imagematching/opencv_imm/server/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/imagematching/opencv_imm/server/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/imagematching/opencv_imm/test/IMMClient.cpp b/lucida/imagematching/opencv_imm/test/IMMClient.cpp index 5d1a1f17e..d137c31e3 100644 --- a/lucida/imagematching/opencv_imm/test/IMMClient.cpp +++ b/lucida/imagematching/opencv_imm/test/IMMClient.cpp @@ -17,7 +17,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "Parser.h" using namespace folly; using namespace apache::thrift; @@ -57,6 +56,12 @@ string getImageData(const string &image_path) { } int main(int argc, char* argv[]) { + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + // Initialize MongoDB C++ driver. mongo::client::initialize(); mongo::DBClientConnection conn; @@ -73,18 +78,6 @@ int main(int argc, char* argv[]) { folly::init(&argc, &argv); EventBase event_base; - - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMM_PORT", portVal)) { - cout << "IMM port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); - } - std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); LucidaServiceAsyncClient client( @@ -122,9 +115,6 @@ int main(int argc, char* argv[]) { // Infer. // Make request. int num_tests = 3; - if (argc == 2) { - num_tests = atoi(argv[1]); - } for (int i = 0; i < num_tests; ++i) { string image = getImageData("test" + to_string(i) + ".jpg"); // Create a QuerySpec. diff --git a/lucida/imagematching/opencv_imm/test/Parser.h b/lucida/imagematching/opencv_imm/test/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/imagematching/opencv_imm/test/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/musicservice/Makefile b/lucida/musicservice/Makefile index 8462d33d8..fa5bbd0f6 100644 --- a/lucida/musicservice/Makefile +++ b/lucida/musicservice/Makefile @@ -15,7 +15,13 @@ clean: rm -rf lucidaservice lucidatypes start_server: - cd server; python server.py + @if [ "$(port)" != "" ]; then \ + cd server; \ + python server.py $(port); \ + fi -start_test: - cd client; python client.py \ No newline at end of file +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python client.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/musicservice/MusicConfig.py b/lucida/musicservice/MusicConfig.py index e1b12cf76..dea17a90e 100644 --- a/lucida/musicservice/MusicConfig.py +++ b/lucida/musicservice/MusicConfig.py @@ -3,11 +3,7 @@ """ from pygn import Pygn -from helper import port_dic - -# Service port number -PORT = int(port_dic['ms_port']) # Music API username and password -clientID = # TODO: Enter your Client ID from developer.gracenote.com here +clientID = '173703779-EA19716F8D2A73DF7ECAF522D5050BF3' # TODO: Enter your Client ID from developer.gracenote.com here userID = Pygn.register(clientID) # Get a User ID from pygn.register() - Only register once per end-user diff --git a/lucida/musicservice/client/client.py b/lucida/musicservice/client/client.py index 637e5706a..7a463e74a 100644 --- a/lucida/musicservice/client/client.py +++ b/lucida/musicservice/client/client.py @@ -3,7 +3,6 @@ import sys sys.path.append('../') -from MusicConfig import PORT from lucidatypes.ttypes import QueryInput, QuerySpec from lucidaservice import LucidaService @@ -12,6 +11,11 @@ from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + # Setup a template input query LUCID = "Clinc" query_input_data = "I want a happy song." diff --git a/lucida/musicservice/helper.py b/lucida/musicservice/helper.py index d039262c6..2de30c997 100644 --- a/lucida/musicservice/helper.py +++ b/lucida/musicservice/helper.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import ConfigParser, sys, re +import re # Mood invert index mood_dic = { @@ -85,20 +85,3 @@ def keyword_scan(question): else: return output[0] -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../../config.properties"))) -port_dic = dict(cp.items('asection')) \ No newline at end of file diff --git a/lucida/musicservice/server/server.py b/lucida/musicservice/server/server.py index 66f746f28..27e7c4ceb 100644 --- a/lucida/musicservice/server/server.py +++ b/lucida/musicservice/server/server.py @@ -20,6 +20,11 @@ import json import re +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + class MusicHandler: def create(self, LUCID, spec): ''' diff --git a/lucida/questionanswering/OpenEphyra/Makefile b/lucida/questionanswering/OpenEphyra/Makefile index 1f3d30d7a..6790eeb3d 100644 --- a/lucida/questionanswering/OpenEphyra/Makefile +++ b/lucida/questionanswering/OpenEphyra/Makefile @@ -14,10 +14,16 @@ thrift: fi start_server: - chmod +x start_server.sh && ./start_server.sh + @if [ "$(port)" != "" ]; then \ + chmod +x start_server.sh; \ + ./start_server.sh $(port); \ + fi start_test: - chmod +x start_test.sh && ./start_test.sh + @if [ "$(port)" != "" ]; then \ + chmod +x start_test.sh; \ + ./start_test.sh $(port); \ + fi clean: rm -rf bin diff --git a/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java b/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java index 21681525a..49c0dabb7 100644 --- a/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java +++ b/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java @@ -11,10 +11,7 @@ import org.apache.thrift.transport.TTransportException; import java.io.IOException; -import java.io.FileInputStream; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Properties; import org.apache.thrift.TException; import org.apache.thrift.TProcessor; @@ -40,11 +37,12 @@ public class QADaemon { */ public static void main(String [] args) throws TTransportException, IOException, InterruptedException { - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("QA_PORT"); - Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + TProcessor proc = new LucidaService.AsyncProcessor( new QAServiceHandler.AsyncQAServiceHandler()); TNonblockingServerTransport transport = new TNonblockingServerSocket(port); @@ -53,7 +51,7 @@ public static void main(String [] args) .protocolFactory(new TBinaryProtocol.Factory()) .transportFactory(new TFramedTransport.Factory()); final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); - System.out.println("QA at port " + port_str); + System.out.println("QA at port " + port); server.serve(); } } diff --git a/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java b/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java index a47fe5e09..028ad2824 100644 --- a/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java +++ b/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java @@ -2,9 +2,6 @@ //Java packages import java.io.IOException; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import java.util.List; import java.util.ArrayList; @@ -55,12 +52,11 @@ private static final QuerySpec createQuerySpec( public static void main(String [] args) throws IOException { - // Collect the port number. - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("QA_PORT"); - final Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + final Integer port = Integer.valueOf(args[0]); // User. String LUCID = "Clinc"; diff --git a/lucida/questionanswering/OpenEphyra/start_server.sh b/lucida/questionanswering/OpenEphyra/start_server.sh index a3d45e624..d8f3cc21b 100755 --- a/lucida/questionanswering/OpenEphyra/start_server.sh +++ b/lucida/questionanswering/OpenEphyra/start_server.sh @@ -1,2 +1,2 @@ #!/bin/bash -java -Djava.library.path=lib/search/ -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.main.QADaemon +java -Djava.library.path=lib/search/ -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.main.QADaemon $1 diff --git a/lucida/questionanswering/OpenEphyra/start_test.sh b/lucida/questionanswering/OpenEphyra/start_test.sh index 48654f3ba..6f1aa5a69 100755 --- a/lucida/questionanswering/OpenEphyra/start_test.sh +++ b/lucida/questionanswering/OpenEphyra/start_test.sh @@ -1,2 +1,2 @@ #!/bin/bash -java -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.test.QAClient +java -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.test.QAClient $1 diff --git a/lucida/template/.gitignore b/lucida/template/.gitignore new file mode 100644 index 000000000..770c66e2b --- /dev/null +++ b/lucida/template/.gitignore @@ -0,0 +1,2 @@ +lucidaservice.thrift +lucidatypes.thrift diff --git a/lucida/template/README.md b/lucida/template/README.md new file mode 100644 index 000000000..635bd10a3 --- /dev/null +++ b/lucida/template/README.md @@ -0,0 +1,14 @@ +# Template for building your own microservice + +This is a template for a user to refer to when creating their own microservice for Lucida. It includes 3 versions of microservice implementation (c++, Java, python). You can choose your favorite language and implement your own service based on the template. We use both Apache Thrift and Facebook Thrift as our Lucida RPC framework. Thrift is an RPC framework with the advantages of being efficient and language-neutral. It was originally developed by Facebook and now developed by both the open-source community (Apache Thrift) and Facebook. + +## Major Dependencies + +- [Facebook Thrift](https://github.com/facebook/fbthrift) +- [gradle](https://gradle.org/) + +# Structure + +- [`cpp`](cpp): implementation of C++ template +- [`java`](java): implementation for Java template +- [`python`](python): implementation for Python template \ No newline at end of file diff --git a/lucida/template/cpp/.gitignore b/lucida/template/cpp/.gitignore new file mode 100644 index 000000000..525b08a05 --- /dev/null +++ b/lucida/template/cpp/.gitignore @@ -0,0 +1,6 @@ +server/gen-cpp2 +server/template_server +test/gen-cpp2 +test/template_test +server/*.o +test/*.o diff --git a/lucida/template/cpp/Makefile b/lucida/template/cpp/Makefile new file mode 100644 index 000000000..9fc5799e6 --- /dev/null +++ b/lucida/template/cpp/Makefile @@ -0,0 +1,28 @@ +all: + cd server ; \ + make all ; \ + cd ../test; \ + make all ; \ + cd ..; + +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + ./template_server $(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./template_test $(port); \ + fi + +clean: + cd server ; \ + make clean ; \ + cd ../test; \ + make clean ; \ + cd ..; + +%: + @: diff --git a/lucida/template/cpp/README.md b/lucida/template/cpp/README.md new file mode 100644 index 000000000..d620b913a --- /dev/null +++ b/lucida/template/cpp/README.md @@ -0,0 +1,72 @@ +# template microservice in C++ + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- [Facebook Thrift](https://github.com/facebook/fbthrift) + +# Structure + +- `server/`: implementation of the template server +- `test/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`server/templateHandler.cpp`](server/templateHandler.cpp). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` + +Update the [`Makefile`](Makefile). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](test/templateClient.cpp) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. + diff --git a/lucida/template/cpp/server/Makefile b/lucida/template/cpp/server/Makefile new file mode 100644 index 000000000..237ed47bc --- /dev/null +++ b/lucida/template/cpp/server/Makefile @@ -0,0 +1,57 @@ +CXX = g++ + +CXXFLAGS = -std=c++11 \ + -fPIC +CXXFLAGS += $(shell if [ `lsb_release -a 2>/dev/null | grep -Poe "(?<=\s)\d+(?=[\d\.]+$$)"` -gt 14 ]; then echo "-std=c++14"; fi;) + +LINKFLAGS = -lrt \ + -lprotobuf \ + -ltesseract \ + -pthread \ + -lboost_program_options \ + -lboost_filesystem \ + -lboost_system \ + -lthrift \ + -lfolly \ + -lwangle \ + -lzstd \ + -lglog \ + -lthriftcpp2 \ + -lgflags \ + -lthriftprotocol \ + -lcrypto + +TARGET = template_server +SOURCES = gen-cpp2/LucidaService_client.cpp \ + gen-cpp2/lucidaservice_constants.cpp \ + gen-cpp2/LucidaService.cpp \ + gen-cpp2/LucidaService_processmap_binary.cpp \ + gen-cpp2/LucidaService_processmap_compact.cpp \ + gen-cpp2/lucidaservice_types.cpp \ + gen-cpp2/lucidatypes_constants.cpp \ + gen-cpp2/lucidatypes_types.cpp \ + $(wildcard *.cpp) +OBJECTS = $(SOURCES:.cpp=.o) + +all: CXXFLAGS += -O3 +all: thrift $(TARGET) + +debug: CXXFLAGS += -g3 +debug: thrift $(TARGET) + +thrift: + @if [ ! -d "gen-cpp2" ]; then \ + python -mthrift_compiler.main --gen cpp2 ../../lucidaservice.thrift; \ + python -mthrift_compiler.main --gen cpp2 ../../lucidatypes.thrift; \ + fi + +$(TARGET): $(OBJECTS) + $(CXX) $(OBJECTS) $(LINKFLAGS) -o $@ + +%.o: %.cpp + $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ + +clean: + rm -rf $(TARGET) *.o gen-cpp2 + +.PHONY: all debug thrift clean \ No newline at end of file diff --git a/lucida/template/cpp/server/templateHandler.cpp b/lucida/template/cpp/server/templateHandler.cpp new file mode 100644 index 000000000..eb1071fd5 --- /dev/null +++ b/lucida/template/cpp/server/templateHandler.cpp @@ -0,0 +1,63 @@ +#include "templateHandler.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; + +using std::cout; +using std::endl; +using std::string; +using std::unique_ptr; +using std::shared_ptr; +using std::getenv; + +// cout lock. +std::mutex cout_lock_cpp; + +namespace cpp2 { +templateHandler::templateHandler() { + // TODO: adding your own initializatin of the handler +} + +// TODO: implement your own future_create function. +folly::Future templateHandler::future_create +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> spec) { + folly::MoveWrapper > promise; + auto future = promise->getFuture(); + promise->setValue(Unit{}); + return future; +} + +// TODO: implement your own future_learn function. +folly::Future templateHandler::future_learn +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> knowledge) { + folly::MoveWrapper > promise; + auto future = promise->getFuture(); + promise->setValue(Unit{}); + return future; +} + +// TODO: implement your own future_infer function. +folly::Future> templateHandler::future_infer +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> query) { + print("@@@@@ Infer; User: " + *(LUCID.get())); + folly::MoveWrapper > > promise; + auto future = promise->getFuture(); + promise->setValue(unique_ptr(new string("This is the sample answer"))); + return future; +} + +// TODO: define the methods needed for your service. +} diff --git a/lucida/template/cpp/server/templateHandler.h b/lucida/template/cpp/server/templateHandler.h new file mode 100644 index 000000000..c50746c8e --- /dev/null +++ b/lucida/template/cpp/server/templateHandler.h @@ -0,0 +1,41 @@ +#ifndef __TEMPLATEHANDLER_H__ +#define __TEMPLATEHANDLER_H__ + +#include +#include + +#include "gen-cpp2/LucidaService.h" + +// Define print for simple logging. +extern std::mutex cout_lock_cpp; +#define print( x ) \ + ( \ + (cout_lock_cpp.lock()), \ + (std::cout << x << endl), \ + (cout_lock_cpp.unlock()), \ + (void)0 \ + ) + +namespace cpp2 { +class templateHandler : virtual public LucidaServiceSvIf { +public: + templateHandler(); + + folly::Future future_create + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> spec); + + folly::Future future_learn + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> knowledge); + + folly::Future> future_infer + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> query); + +private: + // TODO: Define your own private methods +}; +} + +#endif diff --git a/lucida/template/cpp/server/templateServer.cpp b/lucida/template/cpp/server/templateServer.cpp new file mode 100644 index 000000000..0e9950867 --- /dev/null +++ b/lucida/template/cpp/server/templateServer.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include + +DEFINE_int32(num_of_threads, + 4, + "Number of threads (default: 4)"); + +#include "templateHandler.h" +#include +#include + +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; +using namespace cpp2; +using namespace std; + +using std::cout; +using std::endl; +using std::shared_ptr; +using std::unique_ptr; +using std::to_string; + +int main(int argc, char* argv[]) { + folly::init(&argc, &argv); + + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + + auto handler = std::make_shared(); + auto server = folly::make_unique(); + + server->setPort(port); + server->setNWorkerThreads(FLAGS_num_of_threads); + server->setInterface(std::move(handler)); + server->setIdleTimeout(std::chrono::milliseconds(0)); + server->setTaskExpireTime(std::chrono::milliseconds(0)); + + cout << "TPL at port " << to_string(port) << endl; + server->serve(); + + return 0; +} diff --git a/lucida/template/cpp/test/Makefile b/lucida/template/cpp/test/Makefile new file mode 100644 index 000000000..d97d910ea --- /dev/null +++ b/lucida/template/cpp/test/Makefile @@ -0,0 +1,57 @@ +CXX = g++ + +CXXFLAGS = -std=c++11 \ + -fPIC +CXXFLAGS += $(shell if [ `lsb_release -a 2>/dev/null | grep -Poe "(?<=\s)\d+(?=[\d\.]+$$)"` -gt 14 ]; then echo "-std=c++14"; fi;) + +LINKFLAGS = -lrt \ + -lprotobuf \ + -ltesseract \ + -pthread \ + -lboost_program_options \ + -lboost_filesystem \ + -lboost_system \ + -lthrift \ + -lfolly \ + -lwangle \ + -lzstd \ + -lglog \ + -lthriftcpp2 \ + -lgflags \ + -lthriftprotocol \ + -lcrypto + +TARGET = template_test +SOURCES = gen-cpp2/LucidaService_client.cpp \ + gen-cpp2/lucidaservice_constants.cpp \ + gen-cpp2/LucidaService.cpp \ + gen-cpp2/LucidaService_processmap_binary.cpp \ + gen-cpp2/LucidaService_processmap_compact.cpp \ + gen-cpp2/lucidaservice_types.cpp \ + gen-cpp2/lucidatypes_constants.cpp \ + gen-cpp2/lucidatypes_types.cpp \ + $(wildcard *.cpp) +OBJECTS = $(SOURCES:.cpp=.o) + +all: CXXFLAGS += -O3 +all: thrift $(TARGET) + +debug: CXXFLAGS += -g3 +debug: thrift $(TARGET) + +thrift: + @if [ ! -d "gen-cpp2" ]; then \ + python -mthrift_compiler.main --gen cpp2 ../../lucidaservice.thrift; \ + python -mthrift_compiler.main --gen cpp2 ../../lucidatypes.thrift; \ + fi + +$(TARGET): $(OBJECTS) + $(CXX) $(OBJECTS) $(LINKFLAGS) -o $@ + +%.o: %.cpp + $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ + +clean: + rm -rf $(TARGET) *.o gen-cpp2 + +.PHONY: all debug thrift clean diff --git a/lucida/template/cpp/test/templateClient.cpp b/lucida/template/cpp/test/templateClient.cpp new file mode 100644 index 000000000..2a42b8a76 --- /dev/null +++ b/lucida/template/cpp/test/templateClient.cpp @@ -0,0 +1,61 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "gen-cpp2/LucidaService.h" +#include +#include + +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; +using namespace cpp2; +using namespace std; + +DEFINE_string(hostname, + "127.0.0.1", + "Hostname of the server (default: localhost)"); + +int main(int argc, char* argv[]) { + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + + folly::init(&argc, &argv); + EventBase event_base; + std::shared_ptr socket_t( + TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); + LucidaServiceAsyncClient client( + std::unique_ptr( + new HeaderClientChannel(socket_t))); + + // Infer. + QuerySpec query_spec; + // Create a QueryInput for the query image and add it to the QuerySpec. + // TODO: Adding your own sample query + QueryInput query_input; + query_input.type = "query"; + query_input.data.push_back("What is the sample query?"); + query_input.tags.push_back("localhost"); + query_input.tags.push_back(to_string(port)); + query_input.tags.push_back("0"); + query_spec.content.push_back(query_input); + cout << "///// Connecting to template... /////" << endl; + auto result = client.future_infer("Clinc", std::move(query_spec)).then( + [=](folly::Try&& t) mutable { + cout << "///// Result: /////\n" << t.value() << endl; + return t.value(); + }); + event_base.loop(); + return 0; +} diff --git a/lucida/template/java/.gitignore b/lucida/template/java/.gitignore new file mode 100644 index 000000000..17ed65221 --- /dev/null +++ b/lucida/template/java/.gitignore @@ -0,0 +1,5 @@ +.gradle +build +templateClient/thrift +src/main/java/thrift +templateClient/*.class diff --git a/lucida/template/java/Makefile b/lucida/template/java/Makefile new file mode 100644 index 000000000..a300c5f8d --- /dev/null +++ b/lucida/template/java/Makefile @@ -0,0 +1,39 @@ +all: thrift client + ./gradlew build + +thrift: + @if [ ! -d "src/main/java/thrift" ]; then \ + thrift --gen java ../lucidaservice.thrift; \ + thrift --gen java ../lucidatypes.thrift; \ + sed -i '1s/^/package thrift;\n/' gen-java/LucidaService.java; \ + sed -i '1s/^/package thrift;\n/' gen-java/QueryInput.java; \ + sed -i '1s/^/package thrift;\n/' gen-java/QuerySpec.java; \ + mkdir src/main/java/thrift/; \ + mv gen-java/* src/main/java/thrift/; \ + rmdir gen-java/; \ + fi + +client: + cd templateClient/ && ./compile-template-client.sh + +start_server: + @if [ "$(port)" != "" ]; then \ + ./gradlew run -Pargs=$(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd templateClient; \ + ./start-template-client.sh $(port); \ + fi + +clean: + rm -rf src/main/java/thrift ; \ + rm -rf templateClient/thrift ; \ + rm -rf templateClient/*.class ; \ + rm -rf build; + +.PHONY: all thrift gradle start_server start_test clean + +%: + @: diff --git a/lucida/template/java/README.md b/lucida/template/java/README.md new file mode 100644 index 000000000..99b6f3be0 --- /dev/null +++ b/lucida/template/java/README.md @@ -0,0 +1,71 @@ +# template microservice in Java + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- [gradle](https://gradle.org/) + +# Structure + +- `src/main/java/template/`: implementation of the template server +- `templateClient/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`src/main/java/template/TEServiceHandler.java`](src/main/java/template/TEServiceHandler.java). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` and `build.gradle` + +Update the [`Makefile`](Makefile) and [`build.gradle`](build.gradle). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](templateClient/templateClient.java) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. diff --git a/lucida/template/java/build.gradle b/lucida/template/java/build.gradle new file mode 100644 index 000000000..e410230f1 --- /dev/null +++ b/lucida/template/java/build.gradle @@ -0,0 +1,25 @@ +apply plugin: 'java' +apply plugin: 'application' + +mainClassName = 'template.templateDaemon' +sourceCompatibility = 1.7 +targetCompatibility = 1.7 +version = '1.0' + +run { + if( project.hasProperty('args')) { + args project.args.split('\\s') + } +} + +repositories { + mavenCentral() + flatDir { + dirs 'lib' + } +} + +// TODO: Add the dependencies your service needs +dependencies { + compile 'org.apache.thrift:libthrift:0.9.3' +} \ No newline at end of file diff --git a/lucida/template/java/gradle/wrapper/gradle-wrapper.jar b/lucida/template/java/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..5ccda13e9 Binary files /dev/null and b/lucida/template/java/gradle/wrapper/gradle-wrapper.jar differ diff --git a/lucida/template/java/gradle/wrapper/gradle-wrapper.properties b/lucida/template/java/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..c78864717 --- /dev/null +++ b/lucida/template/java/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Mar 30 20:24:24 EDT 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/lucida/template/java/gradlew b/lucida/template/java/gradlew new file mode 100755 index 000000000..9d82f7891 --- /dev/null +++ b/lucida/template/java/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/lucida/template/java/gradlew.bat b/lucida/template/java/gradlew.bat new file mode 100644 index 000000000..72d362daf --- /dev/null +++ b/lucida/template/java/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lucida/template/java/settings.gradle b/lucida/template/java/settings.gradle new file mode 100644 index 000000000..18204127f --- /dev/null +++ b/lucida/template/java/settings.gradle @@ -0,0 +1,19 @@ +/* + * This settings file was auto generated by the Gradle buildInit task + * by 'yba' at '3/11/16 9:55 PM' with Gradle 2.11 + * + * The settings file is used to specify which projects to include in your build. + * In a single project build this file can be empty or even removed. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user guide at https://docs.gradle.org/2.11/userguide/multi_project_builds.html + */ + +/* +// To declare projects as part of a multi-project build use the 'include' method +include 'shared' +include 'api' +include 'services:webservice' +*/ + +rootProject.name = 'template' diff --git a/lucida/template/java/src/main/java/template/TPLServiceHandler.java b/lucida/template/java/src/main/java/template/TPLServiceHandler.java new file mode 100644 index 000000000..b445be07a --- /dev/null +++ b/lucida/template/java/src/main/java/template/TPLServiceHandler.java @@ -0,0 +1,87 @@ +package template; + +import java.util.List; +import java.io.File; +import java.util.ArrayList; + +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; + +import thrift.*; + +/** + * Implementation of the template interface. A client request to any + * method defined in the thrift file is handled by the + * corresponding method here. + */ +public class TPLServiceHandler { + public static void print(String s) { + synchronized (System.out) { + System.out.println(s); + } + } + + public static class SyncTPLServiceHandler implements LucidaService.Iface { + /** + * TODO: Implement your own create function for your service. + * @param LUCID ID of Lucida user + * @param spec spec + */ + @Override + public void create(String LUCID, QuerySpec spec) {} + + /** + * TODO: Implement your own learn function for your service. + * @param LUCID ID of Lucida user + * @param knowledge knowledge + */ + @Override + public void learn(String LUCID, QuerySpec knowledge) {} + + /** + * TODO: Implement your own infer function for your service. + * Here is a sample infer function. + * @param LUCID ID of Lucida user + * @param query query + */ + @Override + public String infer(String LUCID, QuerySpec query) { + print("@@@@@ Infer; User: " + LUCID); + if (query.content.isEmpty() || query.content.get(0).data.isEmpty()) { + throw new IllegalArgumentException(); + } + String query_data = query.content.get(0).data.get(0); + print("Asking: " + query_data); + String answer_data = "This is the sample answer"; + print("Result: " + answer_data); + return answer_data; + } + } + + public static class AsyncTPLServiceHandler implements LucidaService.AsyncIface { + private SyncTPLServiceHandler handler; + + public AsyncTPLServiceHandler() { + handler = new SyncTPLServiceHandler(); + } + + @Override + public void create(String LUCID, QuerySpec spec, AsyncMethodCallback resultHandler) + throws TException { + print("Async Create"); + } + + @Override + public void learn(String LUCID, QuerySpec knowledge, AsyncMethodCallback resultHandler) + throws TException { + print("Async Learn"); + } + + @Override + public void infer(String LUCID, QuerySpec query, AsyncMethodCallback resultHandler) + throws TException { + print("Async Infer"); + resultHandler.onComplete(handler.infer(LUCID, query)); + } + } +} diff --git a/lucida/template/java/src/main/java/template/templateDaemon.java b/lucida/template/java/src/main/java/template/templateDaemon.java new file mode 100644 index 000000000..99bc48165 --- /dev/null +++ b/lucida/template/java/src/main/java/template/templateDaemon.java @@ -0,0 +1,52 @@ +package template; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.ArrayList; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.server.TNonblockingServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadedSelectorServer; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.async.AsyncMethodCallback; + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +import thrift.*; + +/** + * Starts the template server and listens for requests. + */ +public class templateDaemon { + public static void main(String [] args) + throws TTransportException, IOException, InterruptedException { + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + + TProcessor proc = new LucidaService.AsyncProcessor( + new TPLServiceHandler.AsyncTPLServiceHandler()); + TNonblockingServerTransport transport = new TNonblockingServerSocket(port); + TThreadedSelectorServer.Args arguments = new TThreadedSelectorServer.Args(transport) + .processor(proc) + .protocolFactory(new TBinaryProtocol.Factory()) + .transportFactory(new TFramedTransport.Factory()); + final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); + // TODO: Change XXX into your service's acronym + System.out.println("TPL at port " + port_str); + server.serve(); + } +} diff --git a/lucida/template/java/templateClient/compile-template-client.sh b/lucida/template/java/templateClient/compile-template-client.sh new file mode 100755 index 000000000..892236693 --- /dev/null +++ b/lucida/template/java/templateClient/compile-template-client.sh @@ -0,0 +1,25 @@ +#!/bin/bash +printdivision() +{ + echo -e "\n" + for i in $(seq 1 70); do + echo -n "/"; + done + echo -e "\n" +} + +# Generate thrift files +echo -e "./compile-template-client.sh: $(pwd)" +echo -e "./compile-template-client.sh: Compiling thrift source code..." +thrift --gen java ../../lucidaservice.thrift +thrift --gen java ../../lucidatypes.thrift +mv gen-java thrift +printdivision + +# Add jar files to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar + +# Use cp flag to avoid cluttering up the CLASSPATH environment variable +echo -e "javac -cp $JAVA_CLASS_PATH templateClient.java thrift/LucidaService.java thrift/QueryInput.java thrift/QuerySpec.java +\n\n" +javac -cp "$JAVA_CLASS_PATH" templateClient.java thrift/LucidaService.java thrift/QueryInput.java thrift/QuerySpec.java diff --git a/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar b/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar new file mode 100644 index 000000000..f9221a9f9 Binary files /dev/null and b/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar differ diff --git a/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar b/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar new file mode 100644 index 000000000..4dfaaa8ce Binary files /dev/null and b/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar differ diff --git a/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar b/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar new file mode 100644 index 000000000..8c3b624ff Binary files /dev/null and b/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar differ diff --git a/lucida/template/java/templateClient/start-template-client.sh b/lucida/template/java/templateClient/start-template-client.sh new file mode 100755 index 000000000..8d5379ca7 --- /dev/null +++ b/lucida/template/java/templateClient/start-template-client.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Add Thrift classes to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:thrift +# Add other jar files to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar + +# run the server +java -cp "$JAVA_CLASS_PATH" templateClient "$1" diff --git a/lucida/template/java/templateClient/templateClient.java b/lucida/template/java/templateClient/templateClient.java new file mode 100755 index 000000000..5130c7936 --- /dev/null +++ b/lucida/template/java/templateClient/templateClient.java @@ -0,0 +1,59 @@ +//Java packages +import java.io.IOException; +import java.util.ArrayList; + +//Thrift java libraries +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +//Generated code +import thrift.*; + +/** +* A template Client that get the upcoming events from template Server and prints the results. +*/ +public class templateClient { + public static void main(String [] args) + throws IOException{ + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + + // Query. + // TODO: Adding your own sample query + String LUCID = "Clinc"; + String query_input_data = "What is the sample query?"; + QueryInput query_input = new QueryInput(); + query_input.type = "query"; + query_input.data = new ArrayList(); + query_input.data.add(query_input_data); + QuerySpec query_spec = new QuerySpec(); + query_spec.content = new ArrayList(); + query_spec.content.add(query_input); + + // Initialize thrift objects. + TTransport transport = new TSocket("localhost", port); + TProtocol protocol = new TBinaryProtocol(new TFramedTransport(transport)); + LucidaService.Client client = new LucidaService.Client(protocol); + try { + // Talk to the template server. + transport.open(); + System.out.println(query_input_data); + System.out.println("///// Connecting to template... /////"); + String results = client.infer(LUCID, query_spec); + System.out.println("///// Result: /////"); + System.out.println(results); + transport.close(); + } catch (TException e) { + e.printStackTrace(); + } + + return; + } +} diff --git a/lucida/template/python/.gitignore b/lucida/template/python/.gitignore new file mode 100644 index 000000000..eb761dce6 --- /dev/null +++ b/lucida/template/python/.gitignore @@ -0,0 +1,3 @@ +lucidaservice +lucidatypes +templateConfig.pyc diff --git a/lucida/template/python/Makefile b/lucida/template/python/Makefile new file mode 100644 index 000000000..06c77de2c --- /dev/null +++ b/lucida/template/python/Makefile @@ -0,0 +1,28 @@ +all: thrift + +thrift: + @if [ ! -d "lucidaservice" ]; then \ + thrift --gen py ../lucidaservice.thrift; \ + thrift --gen py ../lucidatypes.thrift; \ + cd gen-py; \ + mv * ..; \ + cd ..; \ + rmdir gen-py; \ + rm __init__.py; \ + fi + +clean: + rm -rf lucidaservice lucidatypes ; \ + rm -rf *.pyc; + +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + python server.py $(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python client.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/template/python/README.md b/lucida/template/python/README.md new file mode 100644 index 000000000..4e6dcffea --- /dev/null +++ b/lucida/template/python/README.md @@ -0,0 +1,71 @@ +# template microservice in Python + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- None + +# Structure + +- `server/`: implementation of the template server +- `client/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`server/server.py`](server/server.py). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` + +Update the [`Makefile`](Makefile). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](client/client.py) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. diff --git a/lucida/template/python/client/client.py b/lucida/template/python/client/client.py new file mode 100644 index 000000000..9dfad229a --- /dev/null +++ b/lucida/template/python/client/client.py @@ -0,0 +1,33 @@ +import sys +sys.path.append('../') + +from lucidatypes.ttypes import QueryInput, QuerySpec +from lucidaservice import LucidaService + +from thrift import Thrift +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + +# TODO: Adding your own sample query +LUCID = "Clinc" +query_input_data = "What is the sample query?" +query_input = QueryInput(type="query", data=[query_input_data]) +query_spec = QuerySpec(content=[query_input]) + +# Initialize thrift objects +transport = TTransport.TFramedTransport(TSocket.TSocket("localhost", PORT)) +protocol = TBinaryProtocol.TBinaryProtocol(transport) +client = LucidaService.Client(protocol) + +transport.open() +print "///// Connecting to template... /////" +results = client.infer(LUCID, query_spec) +print "///// Result: /////" +print "%s" % results +transport.close() \ No newline at end of file diff --git a/lucida/template/python/server/server.py b/lucida/template/python/server/server.py new file mode 100644 index 000000000..d88c6ea12 --- /dev/null +++ b/lucida/template/python/server/server.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +import sys +sys.path.append('../') + +from templateConfig import * + +from lucidatypes.ttypes import QuerySpec +from lucidaservice import LucidaService + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +# TODO: Adding modules your services needed + +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + +class templateHandler(LucidaService.Iface): + def create(self, LUCID, spec): + # TODO: Adding your own infer function. Check the top-level README to + # figure out each parameter represents. + # For the template, do nothing + return + + def learn(self, LUCID, knowledge): + # TODO: Adding your own learn function. Check the top-level README to + # figure out each parameter represents. + # For the template, do nothing + return + + def infer(self, LUCID, query): + # TODO: Adding your own infer function. Check the top-level README to + # figure out each parameter represents. + # For the template, print the query info and return "Anwser is XXX" + print("@@@@@ Infer; User: " + LUCID) + if len(query.content) == 0 or len(query.content[0].data) == 0: + return "error: incorrect query" + query_data = query.content[0].data[-1] + print("Asking: " + query_data) + answer_data = "This is the sample answer" + print("Result: " + answer_data) + return answer_data + +# Set handler to our implementation and setup the server +handler = templateHandler() +processor = LucidaService.Processor(handler) +transport = TSocket.TServerSocket(port=PORT) +tfactory = TTransport.TFramedTransportFactory() +pfactory = TBinaryProtocol.TBinaryProtocolFactory() +server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + +# Display useful information on the command center and start the server +# Change 'XXX' into your service's acronym +print 'TPL at port %d' % PORT +server.serve() diff --git a/lucida/template/python/templateConfig.py b/lucida/template/python/templateConfig.py new file mode 100644 index 000000000..e31bfb1c1 --- /dev/null +++ b/lucida/template/python/templateConfig.py @@ -0,0 +1,5 @@ +""" +This a guide for how to add your own microservice into Lucida interface +""" + +# TODO: Other configuration for your own service diff --git a/lucida/weather/Makefile b/lucida/weather/Makefile index 4d90c1d85..05e30bc98 100644 --- a/lucida/weather/Makefile +++ b/lucida/weather/Makefile @@ -15,7 +15,13 @@ clean: rm -rf lucidaservice lucidatypes start_server: - cd server; python WeatherServer.py + @if [ "$(port)" != "" ]; then \ + cd server; \ + python WeatherServer.py $(port); \ + fi -start_test: - cd client; python WeatherClient.py +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python WeatherClient.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/weather/WeatherConfig.py b/lucida/weather/WeatherConfig.py index 2db625228..0b1fcd8c8 100644 --- a/lucida/weather/WeatherConfig.py +++ b/lucida/weather/WeatherConfig.py @@ -2,33 +2,12 @@ Weather API configuration details """ -import ConfigParser, sys - -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../../config.properties"))) -port_dic = dict(cp.items('asection')) -PORT = int(port_dic['we_port']) - # Weather Underground API key # https://www.wunderground.com/weather/api/ WU_API_URL_BASE = 'http://api.wunderground.com/api/' -WU_API_KEY = # TODO: add your API key here +WU_API_KEY = 'ff76e8e80c802d46' # TODO: add your API key here # Open Weather Map API key # https://openweathermap.org/api OWM_API_URL_BASE = 'http://api.openweathermap.org/data/2.5/weather?' -OWM_API_KEY = # TODO: add your API key here +OWM_API_KEY = '362537b891e6df03a316e82565fe4df3' # TODO: add your API key here diff --git a/lucida/weather/client/WeatherClient.py b/lucida/weather/client/WeatherClient.py index 5a725bbf2..ffb6cf37a 100644 --- a/lucida/weather/client/WeatherClient.py +++ b/lucida/weather/client/WeatherClient.py @@ -3,7 +3,6 @@ import sys sys.path.append('../') -from WeatherConfig import PORT from lucidatypes.ttypes import QueryInput, QuerySpec from lucidaservice import LucidaService @@ -12,6 +11,11 @@ from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + LUCID = "Clinc" query_input_data = "What's the weather in Ann Arbor, MI?" query_input = QueryInput(type="query", data=[query_input_data]) diff --git a/lucida/weather/server/WeatherServer.py b/lucida/weather/server/WeatherServer.py index 8ae390520..276c1bf36 100644 --- a/lucida/weather/server/WeatherServer.py +++ b/lucida/weather/server/WeatherServer.py @@ -16,6 +16,11 @@ import json import urllib +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + class WeatherHandler(LucidaService.Iface): def create(self, LUCID, spec): """ @@ -83,5 +88,5 @@ def infer(self, LUCID, query): pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) -print 'WE at port %d' % PORT +print 'WE at port %d' % PORT server.serve() diff --git a/tools/apt_deps.sh b/tools/apt_deps.sh index 65db094a4..6d607accf 100755 --- a/tools/apt_deps.sh +++ b/tools/apt_deps.sh @@ -75,4 +75,5 @@ apt-get install -y zlib1g-dev \ libffi-dev \ libbz2-dev \ python-yaml \ -&& pip install virtualenv ws4py + pygame \ +&& pip install virtualenv ws4py dill diff --git a/tools/gui_backend_cloud.py b/tools/gui_backend_cloud.py new file mode 100644 index 000000000..2ab1532fe --- /dev/null +++ b/tools/gui_backend_cloud.py @@ -0,0 +1,335 @@ +import os +import sys +import requests +import json +from pymongo import * +from bson.objectid import ObjectId + +class MongoDB(object): + def __init__(self): + mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR') + if mongodb_addr: + self.db = MongoClient(mongodb_addr, 27017).lucida + else: + self.db = MongoClient().lucida + + def get_services(self): + dictReturn = [] + + url = 'http://127.0.0.1:3000/api/v1/service' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['service_list'] + + return dictReturn + + def add_service(self): + """ + return code: + 0: success + 1: name has already exists + 2: acronym has already used + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + # "location": location # location of service in local + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_service(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: service name not found + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + elif error == 'Updated name already used': + return 2 + elif error == 'Updated acronym already used': + return 3 + else: + return -1 + + def delete_service(self, _id): + """ + return code: + 0: success + 1: service not exist + """ + + post = { + "option": "delete", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + else: + return -1 + + def get_workflows(self): + dictReturn = [] + + url = 'http://127.0.0.1:3000/api/v1/workflow' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['workflow_list'] + + return dictReturn + + def add_workflow(self): + """ + return code: + 0: success + 1: workflow name already exists + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_workflow(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: workflow name not found + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + elif error == 'Updated name already used': + return 2 + else: + return -1 + + def delete_workflow(self, _id): + """ + return code: + 0: success + 1: workflow not exists + """ + + post = { + "option": "delete", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + else: + return -1 + + def add_instance(self, _id): + """ + return code: + 0: success + 1: host/port not valid + 2: service name not exist + 3: host/port already used + """ + + # list the attributes for the interface + post = { + "option": "add_empty", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['instance_id'] + else: + error = ret_JSON['error'] + return -1, '' + + def update_instance(self, _id, instance_id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: instance name not found + """ + + post = { + "option": "update", + "_id": _id, # name of service + "instance_id": instance_id, # acronym of service + "op": op, # number of instance + "value": value # host/port pair of instances + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + elif error == 'Host/port pair is not valid': + return 2 + elif error == 'Updated host/port has already been used': + return 3 + else: + return -1 + + def delete_instance(self, _id, instance_id): + """ + return code: + 0: success + 1: instance name not exist + """ + + post = { + "option": "delete", + "_id": _id, # name of service + "instance_id": instance_id # acronym of service + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + else: + return -1 + """ + # import this module and call start_server(name) to start + def start_server(self, name): + location, instance = self.search_path(name) + + # start each instance + for pair in instance: + port = pair['port'] + wrapper_begin = 'gnome-terminal -x bash -c "' + wrapper_end = '"' + code = 'cd ' + location + "; " + code = code + "make start_server port=" + str(port) + os.system(wrapper_begin + code + wrapper_end) + + def search_path(self, name): + # get collection for service information + collection = self.db.service_info + + result = collection.find({'name': name}) + + # check if current service is in MongoDB + count = result.count() + if count != 1: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service not in MongoDB.') + exit(1) + + return result[0]['location'], result[0]['instance'] + """ + +def validate_ip_port(s, p): + """ + Check if ip/port is valid with ipv4 + """ + + a = s.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + if p < 0 or p > 65535: + return False + return True + +db = MongoDB() \ No newline at end of file diff --git a/tools/lucida-gui/.gitignore b/tools/lucida-gui/.gitignore new file mode 100644 index 000000000..337501ef0 --- /dev/null +++ b/tools/lucida-gui/.gitignore @@ -0,0 +1,3 @@ +company_dill.pkl +*.pyc +*.tmp \ No newline at end of file diff --git a/tools/lucida-gui/eztext.py b/tools/lucida-gui/eztext.py new file mode 100755 index 000000000..06d3ec7eb --- /dev/null +++ b/tools/lucida-gui/eztext.py @@ -0,0 +1,153 @@ +# input lib +from pygame.locals import * +import pygame, string + +class ConfigError(KeyError): pass + +class Config: + """ A utility for configuration """ + def __init__(self, options, *look_for): + assertions = [] + for key in look_for: + if key[0] in options.keys(): exec('self.'+key[0]+' = options[\''+key[0]+'\']') + else: exec('self.'+key[0]+' = '+key[1]) + assertions.append(key[0]) + for key in options.keys(): + if key not in assertions: raise ConfigError(key+' not expected as option') + +class Input: + """ A text input for pygame apps """ + def __init__(self, **options): + """ Options: x, y, font, color, restricted, maxlength, prompt """ + self.options = Config(options, ['x', '0'], ['y', '0'], ['font', 'pygame.font.Font(None, 32)'], + ['color', '(0,0,0)'], ['restricted', '\'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!"#$%&\\\'()*+,-./:;<=>?@[\]^_`{|}~\''], + ['maxlength', '-1'], ['prompt', '\'\'']) + self.x = self.options.x; self.y = self.options.y + self.font = self.options.font + self.color = self.options.color + self.restricted = self.options.restricted + self.maxlength = self.options.maxlength + self.prompt = self.options.prompt; self.value = '' + self.shifted = False + + def set_pos(self, x, y): + """ Set the position to x, y """ + self.x = x + self.y = y + + def set_font(self, font): + """ Set the font for the input """ + self.font = font + + def draw(self, surface): + """ Draw the text input to a surface """ + text = self.font.render(self.prompt+self.value, 1, self.color) + surface.blit(text, (self.x, self.y)) + + def update(self, events): + """ Update the input based on passed events """ + for event in events: + if event.type == KEYUP: + if event.key == K_LSHIFT or event.key == K_RSHIFT: self.shifted = False + if event.type == KEYDOWN: + if event.key == K_BACKSPACE: self.value = self.value[:-1] + elif event.key == K_LSHIFT or event.key == K_RSHIFT: self.shifted = True + elif event.key == K_SPACE: self.value += ' ' + if not self.shifted: + if event.key == K_a and 'a' in self.restricted: self.value += 'a' + elif event.key == K_b and 'b' in self.restricted: self.value += 'b' + elif event.key == K_c and 'c' in self.restricted: self.value += 'c' + elif event.key == K_d and 'd' in self.restricted: self.value += 'd' + elif event.key == K_e and 'e' in self.restricted: self.value += 'e' + elif event.key == K_f and 'f' in self.restricted: self.value += 'f' + elif event.key == K_g and 'g' in self.restricted: self.value += 'g' + elif event.key == K_h and 'h' in self.restricted: self.value += 'h' + elif event.key == K_i and 'i' in self.restricted: self.value += 'i' + elif event.key == K_j and 'j' in self.restricted: self.value += 'j' + elif event.key == K_k and 'k' in self.restricted: self.value += 'k' + elif event.key == K_l and 'l' in self.restricted: self.value += 'l' + elif event.key == K_m and 'm' in self.restricted: self.value += 'm' + elif event.key == K_n and 'n' in self.restricted: self.value += 'n' + elif event.key == K_o and 'o' in self.restricted: self.value += 'o' + elif event.key == K_p and 'p' in self.restricted: self.value += 'p' + elif event.key == K_q and 'q' in self.restricted: self.value += 'q' + elif event.key == K_r and 'r' in self.restricted: self.value += 'r' + elif event.key == K_s and 's' in self.restricted: self.value += 's' + elif event.key == K_t and 't' in self.restricted: self.value += 't' + elif event.key == K_u and 'u' in self.restricted: self.value += 'u' + elif event.key == K_v and 'v' in self.restricted: self.value += 'v' + elif event.key == K_w and 'w' in self.restricted: self.value += 'w' + elif event.key == K_x and 'x' in self.restricted: self.value += 'x' + elif event.key == K_y and 'y' in self.restricted: self.value += 'y' + elif event.key == K_z and 'z' in self.restricted: self.value += 'z' + elif event.key == K_0 and '0' in self.restricted: self.value += '0' + elif event.key == K_1 and '1' in self.restricted: self.value += '1' + elif event.key == K_2 and '2' in self.restricted: self.value += '2' + elif event.key == K_3 and '3' in self.restricted: self.value += '3' + elif event.key == K_4 and '4' in self.restricted: self.value += '4' + elif event.key == K_5 and '5' in self.restricted: self.value += '5' + elif event.key == K_6 and '6' in self.restricted: self.value += '6' + elif event.key == K_7 and '7' in self.restricted: self.value += '7' + elif event.key == K_8 and '8' in self.restricted: self.value += '8' + elif event.key == K_9 and '9' in self.restricted: self.value += '9' + elif event.key == K_BACKQUOTE and '`' in self.restricted: self.value += '`' + elif event.key == K_MINUS and '-' in self.restricted: self.value += '-' + elif event.key == K_EQUALS and '=' in self.restricted: self.value += '=' + elif event.key == K_LEFTBRACKET and '[' in self.restricted: self.value += '[' + elif event.key == K_RIGHTBRACKET and ']' in self.restricted: self.value += ']' + elif event.key == K_BACKSLASH and '\\' in self.restricted: self.value += '\\' + elif event.key == K_SEMICOLON and ';' in self.restricted: self.value += ';' + elif event.key == K_QUOTE and '\'' in self.restricted: self.value += '\'' + elif event.key == K_COMMA and ',' in self.restricted: self.value += ',' + elif event.key == K_PERIOD and '.' in self.restricted: self.value += '.' + elif event.key == K_SLASH and '/' in self.restricted: self.value += '/' + elif self.shifted: + if event.key == K_a and 'A' in self.restricted: self.value += 'A' + elif event.key == K_b and 'B' in self.restricted: self.value += 'B' + elif event.key == K_c and 'C' in self.restricted: self.value += 'C' + elif event.key == K_d and 'D' in self.restricted: self.value += 'D' + elif event.key == K_e and 'E' in self.restricted: self.value += 'E' + elif event.key == K_f and 'F' in self.restricted: self.value += 'F' + elif event.key == K_g and 'G' in self.restricted: self.value += 'G' + elif event.key == K_h and 'H' in self.restricted: self.value += 'H' + elif event.key == K_i and 'I' in self.restricted: self.value += 'I' + elif event.key == K_j and 'J' in self.restricted: self.value += 'J' + elif event.key == K_k and 'K' in self.restricted: self.value += 'K' + elif event.key == K_l and 'L' in self.restricted: self.value += 'L' + elif event.key == K_m and 'M' in self.restricted: self.value += 'M' + elif event.key == K_n and 'N' in self.restricted: self.value += 'N' + elif event.key == K_o and 'O' in self.restricted: self.value += 'O' + elif event.key == K_p and 'P' in self.restricted: self.value += 'P' + elif event.key == K_q and 'Q' in self.restricted: self.value += 'Q' + elif event.key == K_r and 'R' in self.restricted: self.value += 'R' + elif event.key == K_s and 'S' in self.restricted: self.value += 'S' + elif event.key == K_t and 'T' in self.restricted: self.value += 'T' + elif event.key == K_u and 'U' in self.restricted: self.value += 'U' + elif event.key == K_v and 'V' in self.restricted: self.value += 'V' + elif event.key == K_w and 'W' in self.restricted: self.value += 'W' + elif event.key == K_x and 'X' in self.restricted: self.value += 'X' + elif event.key == K_y and 'Y' in self.restricted: self.value += 'Y' + elif event.key == K_z and 'Z' in self.restricted: self.value += 'Z' + elif event.key == K_0 and ')' in self.restricted: self.value += ')' + elif event.key == K_1 and '!' in self.restricted: self.value += '!' + elif event.key == K_2 and '@' in self.restricted: self.value += '@' + elif event.key == K_3 and '#' in self.restricted: self.value += '#' + elif event.key == K_4 and '$' in self.restricted: self.value += '$' + elif event.key == K_5 and '%' in self.restricted: self.value += '%' + elif event.key == K_6 and '^' in self.restricted: self.value += '^' + elif event.key == K_7 and '&' in self.restricted: self.value += '&' + elif event.key == K_8 and '*' in self.restricted: self.value += '*' + elif event.key == K_9 and '(' in self.restricted: self.value += '(' + elif event.key == K_BACKQUOTE and '~' in self.restricted: self.value += '~' + elif event.key == K_MINUS and '_' in self.restricted: self.value += '_' + elif event.key == K_EQUALS and '+' in self.restricted: self.value += '+' + elif event.key == K_LEFTBRACKET and '{' in self.restricted: self.value += '{' + elif event.key == K_RIGHTBRACKET and '}' in self.restricted: self.value += '}' + elif event.key == K_BACKSLASH and '|' in self.restricted: self.value += '|' + elif event.key == K_SEMICOLON and ':' in self.restricted: self.value += ':' + elif event.key == K_QUOTE and '"' in self.restricted: self.value += '"' + elif event.key == K_COMMA and '<' in self.restricted: self.value += '<' + elif event.key == K_PERIOD and '>' in self.restricted: self.value += '>' + elif event.key == K_SLASH and '?' in self.restricted: self.value += '?' + + if len(self.value) > self.maxlength and self.maxlength >= 0: self.value = self.value[:-1] diff --git a/tools/lucida-gui/gui.py b/tools/lucida-gui/gui.py new file mode 100644 index 000000000..ddf57693c --- /dev/null +++ b/tools/lucida-gui/gui.py @@ -0,0 +1,1225 @@ +import pygame, sys, os +from time import sleep +from pygame.locals import * +pygame.init() +import math +from gui_backend_cloud import db +import random +import base64 +import dill +from cStringIO import StringIO +import pygame, sys, eztext +from textrect import * + + +# make sure no file named .hiddentempfile is under this folder +def input_impl(): + os.system('gedit .hiddentempfile') + file = open('.hiddentempfile') + code = "" + for line in file: + code = code + line + os.system('rm .hiddentempfile') + print(code) + return code + + +#used for modifying code for state/branch +def writeFileForUseMod(inputWrite): + text_file = open(".hiddentempfile", "w") + text_file.write(inputWrite) + text_file.close() + + + +def getUserInputText(): + # initialize pygame + pygame.init() + # create the screen + screen = pygame.display.set_mode((1280,240)) + # fill the screen w/ white + screen.fill((255,255,255)) + # here is the magic: making the text input + # create an input with a max length of 45, + # and a red color and a prompt saying 'type here: ' + txtbx = eztext.Input(maxlength=245, color=(255,0,0), prompt='type: ') + # create the pygame clock + clock = pygame.time.Clock() + # main loop! + + while 1: + # make sure the program is running at 30 fps + clock.tick(30) + + # events for txtbx + events = pygame.event.get() + # process other events + for event in events: + # close it x button si pressed + if event.type == QUIT: + return txtbx.value + + + # clear the screen + screen.fill((255,255,255)) + # update txtbx + txtbx.update(events) + # blit txtbx on the sceen + txtbx.draw(screen) + # refresh the display + pygame.display.flip() + + + + +class imageObject(object): + def __init__(self,name,xPos,yPos,xWidth,yWidth,imageThing,textInBox=" "): + #These are in 2D vector space. + self.xWidth = xWidth + self.yWidth = yWidth + self.xPos = xPos + self.yPos = yPos + self.text = textInBox + self.imageThing = imageThing + self.drawLevel = 0 + self.name = name + def drawImage(self,screen,screenWidth,screenHeight): + widthInPixels = int(screenWidth*self.xWidth) + heightInPixels = int(screenHeight*self.yWidth) + self.imageThing = pygame.transform.scale(self.imageThing, (int(screenWidth*self.xWidth),int(screenHeight*self.yWidth))) + screen.blit(self.imageThing, (self.xPos*screenWidth, self.yPos*screenHeight)) + + my_rect = pygame.Rect((self.xPos*screenWidth, self.yPos*screenHeight, widthInPixels , heightInPixels)) + + rendered_text = render_textrect(self.text, myfont, my_rect, (0,0,0), (48, 255, 48), 1) + if rendered_text: + screen.blit(rendered_text, my_rect.topleft) + + def setDrawLevel(self,valueSet): + self.drawLevel = valueSet + def isPointOnImage(self,xVectorSpace,yVectorSpace): + if((xVectorSpace>self.xPos) and (xVectorSpace<(self.xPos+self.xWidth)) and (yVectorSpace>self.yPos) and (yVectorSpace<(self.yPos+self.yWidth))): + return "YUP" + return "NOPE" + + +def drawArrow(screen,point0x,point0y,point1x,point1y): + + pos1=point0x,point0y + pos2=point1x,point1y + + pygame.draw.line(screen, (0,0,0), pos1, pos2) + + arrow=pygame.Surface((50,50)) + arrow.fill((255,255,255)) + pygame.draw.line(arrow, (0,0,0), (0,0), (25,25)) + pygame.draw.line(arrow, (0,0,0), (0,50), (25,25)) + arrow.set_colorkey((255,255,255)) + + angle=math.atan2(-(pos1[1]-pos2[1]), pos1[0]-pos2[0]) + angle=math.degrees(angle) + + def drawAng(angle, pos): + nar=pygame.transform.rotate(arrow,angle) + nrect=nar.get_rect(center=pos) + screen.blit(nar, nrect) + + angle+=180 + drawAng(angle, pos2) + + + + + + + +class lineObject(object): + def __init__(self,name,xPos,yPos,xPosEnd,yPosEnd): + #These are in 2D vector space. + self.xPos = xPos + self.yPos = yPos + self.xPosEnd = xPosEnd + self.yPosEnd = yPosEnd + self.drawLevel = 0 + self.name = name + def drawImage(self,screen,screenWidth,screenHeight): + startX = self.xPos*screenWidth + startY = self.yPos*screenHeight + endX = self.xPosEnd*screenWidth + endY = self.yPosEnd*screenHeight + pygame.draw.line(windowScreen.screen,0,(startX,startY),(endX,endY),1) + + + + drawArrow(screen,startX,startY,endX,endY) + + def setDrawLevel(self,valueSet): + self.drawLevel = valueSet + def isPointOnImage(self,xVectorSpace,yVectorSpace): + return "NOPE" + +class windowInformation(object): + def __init__(self,x,y): + self.width = x + self.height = y + self.screen = pygame.display.set_mode((x, y)) + self.listOfObjectsNew = dict(); + + def refreshWindow(self): + self.screen = pygame.display.set_mode((self.width, self.height)) + + # Go through every image object and draw to screen + def blitScreen(self): + print("=======BLITTING SCREEN=====") + self.screen.fill(0) + bgIMGtmp = bgIMG; + bgIMGtmp = pygame.transform.scale(bgIMGtmp, (self.width,self.height)) + self.screen.blit(bgIMGtmp, (0,0)) + drawLevels = -1 + #Some images are supposed to draw ontop of otehrs + while drawLevels!=10: + drawLevels+=1 + for key in self.listOfObjectsNew: + if(self.listOfObjectsNew[key].drawLevel==drawLevels): + self.listOfObjectsNew[key].drawImage(self.screen,self.width,self.height) + pygame.display.flip() + + + #Add an image object onto the screen + def addObject(self,objectName,objectThing): + self.listOfObjectsNew[objectName]= objectThing; + + def removeObject(self,objectName): + del self.listOfObjectsNew[objectName] + + + def getClickOnName(self,xClick,yClick): + xClickVectorSpace = float(xClick)/self.width + yClickVectorSpace = float(yClick)/self.height + + + + #When checking where click, must go through the top to bottom of the image stack + drawLevels = 10 + while drawLevels!=-1: + + for key in self.listOfObjectsNew: + #print(drawLevels,key,self.listOfObjectsNew[key].drawLevel) + if(self.listOfObjectsNew[key].drawLevel==drawLevels): + status = self.listOfObjectsNew[key].isPointOnImage(xClickVectorSpace,yClickVectorSpace) + #print("NODE CHECK", key,xClickVectorSpace,yClickVectorSpace,status,self.listOfObjectsNew[key].drawLevel,drawLevels) + if(status=="YUP"): + return key + drawLevels-=1 + + + return "NOPE" + + + + + + + + + + + + +# Load the image types +yellowBG = pygame.image.load(os.path.join("images/yellowBG.png")) +grayBG = pygame.image.load(os.path.join("images/grayBG.png")) + +nodeActiveIMG = pygame.image.load(os.path.join("images/nodeActive.png")) +nodeIMG = pygame.image.load(os.path.join("images/node.png")) +serviceListButtonIMG = pygame.image.load(os.path.join("images/serviceListButton.png")) +bgIMG = pygame.image.load(os.path.join("images/bg.png")) + + + + + +def getNodeImage(node): + if(node.active==1): + return nodeActiveIMG + return nodeIMG + + + +textOffsetX = 0.025 +textOffsetY = 0.025 + +class lucidaGUI(object): + def __init__(self): + self.dirList = [] + self.WFXOffset = 0.0 + self.WFYOffset = -0.0 + self.currentLevelName = "" + + + #Clicking on level, find what is being clicked on. + def clickLevel(self,xClick,yClick): + clickOnName = windowScreen.getClickOnName(xClick,yClick) + if(clickOnName!="NOPE"): + self.level(clickOnName,1) + + + + def refreshCurrent(self): + windowScreen.refreshWindow() + upDirName = directoriesNames.pop() + lucida.level(upDirName,0) + + def getXYPositionEntry(self,ID): + x = ID%5 + ID /= 5 + y = ID + return [x*0.1,y*0.1] + + def goUpDir(self): + countArr = len(directoriesNames) + + if countArr>1: + upDirName = directoriesNames.pop() + upDirName = directoriesNames.pop() + self.level(upDirName,0) + + + def updateObjectPosition(self,name,xPos,yPos): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.xPos = xPos + objectThing.yPos = yPos + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + def updateObjectPosition2(self,name,xPos,yPos): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.xPosEnd = xPos + objectThing.yPosEnd = yPos + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + + + def updateObjectImage(self,name,imageThing): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.imageThing = imageThing + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + def addToLevel(self,xPos,yPos,xWidth,yWidth,drawLevel,imageObjectName,image,text=" "): + imageObjectInst = imageObject(imageObjectName, xPos,yPos,xWidth,yWidth,image,text) + imageObjectInst.setDrawLevel(drawLevel) + self.dirList.append(imageObjectInst) + + + + def addToLevelLine(self,xPos,yPos,drawLevel,imageObjectName,xPosEnd,yPosEnd): + textObjectInst = lineObject(imageObjectName,xPos,yPos,xPosEnd,yPosEnd) + textObjectInst.setDrawLevel(drawLevel) + self.dirList.append(textObjectInst) + + + # Removes the current display + def undisplayCurrentLevel(self): + + for objectThing in self.dirList: + windowScreen.removeObject(objectThing.name) + self.dirList = [] + # Adds the current (And possibly new) display + def displayCurrentLevel(self): + print("======++DISPLAY CURRENT++======") + for objectThing in self.dirList: + windowScreen.addObject(objectThing.name,objectThing) + windowScreen.blitScreen(); + + def getXYNode(self,node): + XY = [] + XY.append(node.x-self.WFXOffset) + XY.append(node.y-self.WFYOffset) + return XY + + + #Level function draws the level display and the level click functions + def level(self,levelName,directClick): + + #print("Hit level", levelName) + if(directClick==1): + self.currentLevelName = levelName + + + didHit = 0 + goUpDir = 0 + self.undisplayCurrentLevel(); + ### Front: CMD/MS buttons + if(levelName=="root"): + didHit = 1 + self.addToLevel(0.25,0.4,0.5,0.1,0,"wfList",yellowBG,"Workflows") + self.addToLevel(0.25,0.5,0.5,0.1,0,"msList",yellowBG,"Microservices") + self.addToLevel(0.25,0.6,0.5,0.1,0,"bbList",yellowBG,"Blackboxes") + + + # List all of the workflow types + if(levelName=="wfList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.25,0.10,0,"wfListHeader:",yellowBG,"Workflows:") + self.addToLevel(0.50,0.25,0.25,0.10,0,"newWF:",yellowBG,"New") + for WF in workflowList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"wfThing:"+str(countServices),grayBG,WF.name) + countServices+= 1 + + + ## bbList has not been implemented in the GUI yet + ''' + if(levelName=="bbList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.25,0.10,0,"bbListHeader:",yellowBG,"Blackboxes:") + self.addToLevel(0.50,0.25,0.25,0.10,0,"newWF:",yellowBG,"New") + for WF in workflowList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"wfListThing:"+str(countServices),grayBG,WF.name) + countServices+= 1 + ''' + + + #List a specific workflow + if "wfThing:" in levelName: + didHit = 1 + getWFID = int(filter(str.isdigit, levelName)) + wfName = workflowList[getWFID].name + self.addToLevel(0.25,0.25,0.35,0.10,0,"WorkflowNameThing:"+str(getWFID),yellowBG,"WF: " + wfName) + self.addToLevel(0.60,0.25,0.25,0.10,0,"wfListThing"+str(getWFID),yellowBG,"State Graph") + + inputTypes = workflowList[getWFID].dbData['input'][0]; + try: + inputTypes += "," + workflowList[getWFID].dbData['input'][1]; + except: + pass + self.addToLevel(0.25,0.35,0.60,0.10,0,"wfType:"+str(getWFID),yellowBG,"Type:" + inputTypes) + self.addToLevel(0.25,0.45,0.60,0.25,0,"classPath:"+str(getWFID),yellowBG,"ClassPath:" + workflowList[getWFID].dbData['classifier']) + + + + + # This is the actual state graph + if "wfListThing" in levelName: + didHit = 1 + countServices = 0; + getWFID = int(filter(str.isdigit, levelName)) + WF = workflowList[getWFID] + print(WF.nodeList) + + self.addToLevel(0.0,0.0,1.0,1.0,0,"wfBG:",bgIMG) # This gives the background of the state graph + + countServices = 0; + for node in WF.nodeList: + XY = self.getXYNode(node) + + + self.addToLevel(XY[0],XY[1],0.10,0.10,1,"node:"+str(countServices),getNodeImage(node),node.name) + + + #Display the links from this node to another + countLinks = 0 + for link in node.linkList: + toNode = WF.nodeList[link.toNode] + XY2 = self.getXYNode(toNode) + lineThickness = 0.01; + self.addToLevelLine(XY[0],XY[1],2,"IAmLine:"+str(countServices)+"IAmLine:"+str(countLinks),XY2[0],XY2[1]) + + + countLinks += 1 + + countServices += 1 + + + + + + #List microservices + if(levelName=="msList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.25,0.10,0,"msListHeader:",yellowBG,"Microservices:") + self.addToLevel(0.50,0.25,0.25,0.10,0,"newMicroService",yellowBG,"New") + for MS in microServiceList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"msListThing"+str(countServices),grayBG,MS.name) + countServices+= 1 + + #Display a specific microservice + if "msListThing" in levelName: + didHit = 1 + getMSID = int(filter(str.isdigit, levelName)) + msName = microServiceList[getMSID].name + self.addToLevel(0.25,0.25,0.25,0.10,0,"MicroservicesNameThing:"+str(getMSID),yellowBG,"MS: " + msName) + self.addToLevel(0.50,0.25,0.25,0.10,0,"newServer"+str(getMSID),yellowBG,"New") + self.addToLevel(0.25,0.35,0.50,0.05,0,"LearnType:"+str(getMSID),yellowBG,"LearnType: " + microServiceList[getMSID].dbData['learn']) + + countServices = 0 + for server in microServiceList[getMSID].serverList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.40+XY[1],0.10,0.10,0,"serverBB" + str(getMSID) + "serverBB"+str(countServices),grayBG,server.name) + countServices+= 1 + + # An instance of the service + if "serverBB" in levelName: + didHit = 1 + print("BELOW THIS LINE") + idList = levelName.split("serverBB") + msID = int(idList[1]) + boxID = int(idList[2]) + print(levelName.split("serverBB")) + box = microServiceList[msID].serverList[boxID] + msName = microServiceList[msID].name + boxName = box.name + + self.addToLevel(0.25,0.25,0.50,0.10,0,"MS:" + msName,yellowBG,"MS: " + msName) + self.addToLevel(0.25,0.35,0.50,0.10,0,"BoxNameSet:" +str(msID) + "BoxNameSet:" + str(boxID),yellowBG,"Box: " + boxName) + self.addToLevel(0.25,0.45,0.50,0.10,0,"IP:PORT"+str(msID)+"IP:PORT"+str(boxID),yellowBG,"IP:PORT:" + str(box.IP) + ":" + str(box.port)) + + + + #Change the name of a microservice + if "MicroservicesNameThing:" in levelName: + idList = levelName.split("MicroservicesNameThing:") + msID = int(idList[1]) + returnText = getUserInputText() + status = db.update_service( microServiceList[msID].dbData['_id'], "name", returnText) + status = db.update_service( microServiceList[msID].dbData['_id'], "acronym", returnText) + generateMicroServiceList() + self.refreshCurrent() + + # Change the name of a classifier path + if "classPath:" in levelName: + idList = levelName.split("classPath:") + wfID = int(idList[1]) + returnText = getUserInputText() + status = db.update_workflow( workflowList[wfID].dbData['_id'], "classifier", returnText) + generateWorkflowList() + self.refreshCurrent() + + + #Change the name of a workflow + if "WorkflowNameThing:" in levelName: + idList = levelName.split("WorkflowNameThing:") + wfID = int(idList[1]) + returnText = getUserInputText() + status = db.update_workflow( workflowList[wfID].dbData['_id'], "name", returnText) + generateWorkflowList() + workflowData = compileWorkflow(wfID) + db.update_workflow(workflowList[wfID].dbData['_id'], "code",workflowData); + self.refreshCurrent() + + + + #Change the IP:port of a microservice + if "IP:PORT" in levelName: + returnText = getUserInputText() + idList = levelName.split("IP:PORT") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + ipportinfo = returnText.split(":") + box.IP = ipportinfo[0] + box.port = int(ipportinfo[1]) + print(microServiceList[msID].dbData['_id'] , "SPACE" , box.dbData['id'] , "SPACE" + box.IP) + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "host", box.IP) + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "port", box.port) + + generateMicroServiceList() + self.refreshCurrent() + + #Name an instance + if "BoxNameSet:" in levelName: + returnText = getUserInputText() + idList = levelName.split("BoxNameSet:") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "name", returnText) + generateMicroServiceList() + self.refreshCurrent() + + + #Toggles a service between learn types + if "LearnType:" in levelName: + didHit=1 + goUpDir=1 + getMSID = int(filter(str.isdigit, levelName)) + if(microServiceList[getMSID].dbData['learn']=='text'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "image") + if(microServiceList[getMSID].dbData['learn']=='image'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "none") + if(microServiceList[getMSID].dbData['learn']=='none'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "text") + generateMicroServiceList() + + #Toggles a workflow between input types + if "wfType:" in levelName: + didHit=1 + goUpDir=1 + getWFID = int(filter(str.isdigit, levelName)) + doneSomething = 0 + + try: + if((workflowList[getWFID].dbData['input'][0]=='text' and workflowList[getWFID].dbData['input'][1]=='image') or (workflowList[getWFID].dbData['input'][1]=='text' and workflowList[getWFID].dbData['input'][0]=='image')): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['image']) + doneSomething = 1 + except: + pass + + if(workflowList[getWFID].dbData['input'][0]=='image' and doneSomething==0): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text']) + doneSomething = 1 + if(workflowList[getWFID].dbData['input'][0]=='text' and doneSomething==0): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text','image']) + + + generateWorkflowList() + + + + + #New blackbox service registry + if "newServer" in levelName: + didHit=1 + goUpDir=1 + microserviceNameOfServerID = int(levelName.split("newServer",1)[1]) + print(microServiceList[microserviceNameOfServerID].dbData['_id']) + db.add_instance(microServiceList[microserviceNameOfServerID].dbData['_id']) + generateMicroServiceList() + + #A new workflow + if "newWF:" in levelName: + didHit=1 + goUpDir=1 + status,idWF = db.add_workflow() + + db.update_workflow(idWF, "name", "WFTaco"); + db.update_workflow(idWF, "input", ["text"]); + db.update_workflow(idWF, "classifier", "/home/masonhill/lucidas/lucidaapi/lucida-api/lucida/commandcenter/data/class_QAWF.txt"); + + generateWorkflowList() + + #Add a new micro service + if(levelName=="newMicroService"): + didHit=1 + goUpDir=1 + returnVal = 1; + newAppend = 0; + + while(returnVal!=0): + returnVal, serviceID = db.add_service() + newAppend+= 1 + + generateMicroServiceList() + + + + if(didHit==1): + countArr = len(directoriesNames) + if countArr>0: + previousDirName = directoriesNames.pop() + directoriesNames.append(previousDirName) + if(previousDirName!=levelName): + directoriesNames.append(levelName) + if(countArr==0): + directoriesNames.append(levelName) + + #directoriesNames.append(levelName) + self.displayCurrentLevel(); + + if goUpDir==1: + self.goUpDir() + + + #HIt nothing, so need to refresh previous state + if(didHit==0): + upDirName = directoriesNames.pop() + lucida.level(upDirName,0) + + if(not "wfListThing" in directoriesNames[-1]): + currentActiveNode = -1 + + + + + + +class server(object): + def __init__(self,name): + self.name = name + self.IP = "localhost" + self.port = 0 + + + +class microService(object): + def __init__(self,name): + self.name = name + self.serverList = [] + + + def addServer(self,name,ip="",port=0): + serverThing = server(name) + self.serverList.append(serverThing) + serverThing.IP = ip + serverThing.port = port + +class linkObj(object): + def __init__(self,name,toNode): + self.name = name + self.toNode = toNode + self.code = "if(condArgs['pug']=='25 years'):\r\n\treturn True;" + + +class node(object): + def __init__(self,name): + self.name = name + self.linkList = [] # Contains a list of node links + self.x = 0.0; + self.y = 0.0 + self.active = 0 + self.code = "exeMS('pug',\"QA\",\"How old is Johann?\")\nEXE_BATCHED #THIS STATEMENT MUST EXIST AFTER ALL exeMS STATEMENTS. No exeMS is executed UNTIL AFTER THIS STATEMENT\ncondArgs['pug'] = batchedDataReturn['pug']" + + def addLink(self,linkName,toNode): + problemHere = linkObj(linkName,toNode) + self.linkList.append(problemHere) + + +class workflow(object): + def __init__(self,name): + self.name = name + self.nodeList = [] + self.xOffset = 0.0 + self.yOffset = 0.0 + + def addNode(self,node): + self.nodeList.append(node) + +#A workflow needs to convert from stategraph to actual python code +def compileWorkflow(workflowID): + + workflowCompiled = "" + + # Each link needs its own function + + + + workflowName = workflowList[workflowID].name; + + + workflowCompiled += "\nclass "+workflowName+"(workFlow):" + + + + nodeCount = 0 + for node in workflowList[workflowID].nodeList: + + linkCount = 0 + for linkObj in node.linkList: + + + workflowCompiled += "\n\tdef branchCheck"+workflowName+str(nodeCount)+"_"+str(linkCount)+"(self,condArgs,passArgs):"; + + + + + stri = StringIO(linkObj.code) + while True: + nl = stri.readline() + if nl == '': break + workflowCompiled += "\n\t\t"+nl + + + + + linkCount +=1 + + nodeCount +=1 + + + workflowCompiled += "\n\r\n\tdef processCurrentState(self,batchingBit,batchedDataReturn,passArgs,inputModifierText,inputModifierImage):" + + workflowCompiled += "\n\t\tcondArgs = dict()" + nodeCount = 0 + + for node in workflowList[workflowID].nodeList: + workflowCompiled += "\n\t\tif(self.currentState==" + str(nodeCount) +"):" + + workflowCompiled += "\n\t\t\tif(batchingBit==1): self.batchedData = []" + + + stri = StringIO(node.code) + while True: + nl = stri.readline() + appendExeMSString = "" + if nl == '': break + #Need to check the line if it contains a batched request entry. This is [varname] = exeMS( format + currentCount = 0; + for c in nl: + + ''' + ''' + + + nl = nl.replace("exeMS(", "if(batchingBit==1): self.batchedData = appendServiceRequest(self.batchedData,") + nl = nl.replace("EXE_BATCHED", "if(batchingBit==1): return 2") + workflowCompiled += "\n\t\t\t"+nl + + workflowCompiled += "\n\t\t\tif(1==0): QUANTUM_PHYSICS()" + + linkCount = 0 + for linkObj in node.linkList: + + workflowCompiled += "\n\t\t\telif(self.branchCheck"+workflowName+str(nodeCount)+"_"+str(linkCount)+"(condArgs,passArgs)):" + workflowCompiled += "\n\t\t\t\tself.currentState = " + str(linkObj.toNode) + + + + linkCount +=1 + workflowCompiled += "\n\t\t\telse: self.isEnd = True" + + workflowCompiled += "\n\t\t\treturn" + nodeCount +=1 + + return workflowCompiled + + + + + +directoriesNames = [] +workflowList = [] +microServiceList = [] + + + + +# OBJ->DILL->DISK WRITE->DISK READ->BINARY BLOB->HEX64 +#This converts any object to base64. +def objToBase64(workflowThing): + #OBJ->DILL->DISK WRITE + with open('wf.tmp', 'wb') as f: + dill.dump(workflowThing, f) + + #DISK READ->BINARY BLOB->HEX64 + file = open('wf.tmp', 'rb') + file_content = file.read() + base64_two = base64.b64encode(file_content) + return base64_two + + + +#This converts base64 to obj +def base64ToObj(hexData): + #UN-HEX64->BINARY BLOB->DISK WRITE + writeData = hexData.decode('base64'); + + text_file = open('wf2.tmp', 'wb') + text_file.write(writeData) + text_file.close() + #DISK READ->DILL->OBJ + with open('wf2.tmp', 'rb') as f: + test = dill.load( f) + return test + + +def generateMicroServiceList(): + + global microServiceList + microServiceList = [] + serviceList = db.get_services(); + count = 0 + for service in serviceList: + #Create the microservice + if(service['name']==''): service['name'] = "NULL" + microServiceList.append(microService(service['name'])) + microServiceList[count].dbData = service + #And add a list of the valid microservice instances + countInst = 0 + for instance in service['instance']: + if(instance['name']==""): instance['name'] = "NULL" + microServiceList[count].addServer(instance['name'],instance['host'],instance['port']) + microServiceList[count].serverList[countInst].dbData = instance + countInst+=1 + count+= 1 + + + +def generateWorkflowList(): + + + + print("^^^^^^^^^^^^^^^^^^^WORKFLOW^^^^^^^^^^^^^^^^^^^^^") + global workflowList + workflowList = [] + workflows = db.get_workflows(); + count = 0 + for workflowI in workflows: + #Create the microservice + if( workflowI['name']==''): workflowI['name'] = "NULL" + workflowList.append(workflow( workflowI['name'])) + workflowList[count].dbData = workflowI + if( workflowI['stategraph']!=""): + workflowList[count].nodeList = base64ToObj( workflowI['stategraph']) + count+= 1 + + + + +generateMicroServiceList() +generateWorkflowList(); + + + + + + + +windowScreen = windowInformation(512,512) + +myfont = pygame.font.SysFont("arial", 14) +# render text +valueInc = 0 + + +lucida = lucidaGUI() +lucida.level("root",0) + +debounceKey = [] + + + + +#Create the debounce key stack +loop255 = 0; +while loop255!=255: + debounceKey.append(0) + loop255+=1 + + + +debounceMouse = [] +debounceMouse.append(0) +debounceMouse.append(0) +debounceMouse.append(0) + +oldMousePosition = [] +oldMousePosition.append(-1) +oldMousePosition.append(-1) +currentActiveNode = -1 + + +draggingNode = 0 +draggingWF = -1 +draggingNodeID = -1 +draggingX = 0.0 +draggingY = 0.0 + +draggingScreen = 0 +draggingScreenWF = -1 +draggingScreenX = 0.0 +draggingScreenY = 0.0 + + +debounceHm = 0; + +while True: + + + + sleep(0.03) + p = pygame.mouse.get_pos(); + keysPressed = pygame.key.get_pressed() + + #Check if backspace is hit + if(keysPressed[8]==1 and debounceKey[8]==0): + if(currentActiveNode!=-1): + nodeData = WF.nodeList[currentActiveNode] + currentActiveNode = -1 + lucida.updateObjectImage(lucida.currentLevelName,nodeIMG) + nodeData.active = 0 + lucida.goUpDir() + + + + + mousePress = pygame.mouse.get_pressed() + + #Get old mouse position (For moving screen around) + if(mousePress[0]==1 and debounceMouse[0]==1): + if(oldMousePosition[0]==-1): + oldMousePosition[0] = p[0] + oldMousePosition[1] = p[1] + + + #This creates a new node in the WF + if(keysPressed[110]==1 and debounceKey[110]==0 and "wfListThing" in directoriesNames[-1]): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + WF.addNode(node("some state")) + lucida.level(directoriesNames[-1],0); + + + + #This checks if we are moving a node around. + if(mousePress[0]==1 and debounceMouse[0]==1 and "wfListThing" in directoriesNames[-1] and "node:" in lucida.currentLevelName): + draggingNode = 1 + idList = lucida.currentLevelName.split("node:") + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeID = int(idList[1]) + draggingWF = getWFID + draggingNodeID = nodeID + + #Should be able to update now. + nodeData = WF.nodeList[nodeID] + xVector = ((float(p[0]-oldMousePosition[0]))/windowScreen.width) + yVector = ((float(p[1]-oldMousePosition[1]))/windowScreen.height) + + draggingX = xVector+nodeData.x + draggingY = yVector+nodeData.y + + xPlaceNew = xVector+nodeData.x-lucida.WFXOffset + yPlaceNew = yVector+nodeData.y-lucida.WFYOffset + lucida.updateObjectPosition(lucida.currentLevelName,xPlaceNew,yPlaceNew) + lucida.updateObjectPosition("wfListText:"+str(nodeID),xPlaceNew+textOffsetX,yPlaceNew+textOffsetY) + + linkCount = 0 + #This updates the lines at node. + for link in nodeData.linkList: + lucida.updateObjectPosition("IAmLine:" + str(nodeID) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount += 1 + + #But if for instance S0->S1, S0 is updating the line as it owns the node. But if S1 moves, it is not being updated as S1 does not own the node + #So must go through every single node, and find all toNodes and see if match + + fromNodeID = 0 + for nodeCheck in WF.nodeList: + + linkCount = 0; + for link in nodeCheck.linkList: + toNodeID = link.toNode + if(toNodeID==nodeID): # This means eg. S0->S1. We are moving S1. This means S0 has a node to S1, so the line from S0 needs to update. + #So update S0 to repoint to S1. + lucida.updateObjectPosition2("IAmLine:" + str(fromNodeID) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount+=1 + fromNodeID+=1 + + + windowScreen.blitScreen() + + + ## Update the state graph in DB. + if(draggingNode==1 and mousePress[0] == 0): + draggingNode=0 + WF = workflowList[draggingWF] + nodeData = WF.nodeList[draggingNodeID] + nodeData.x =draggingX + nodeData.y = draggingY + base64content = objToBase64(WF.nodeList) + db.update_workflow(workflowList[draggingWF].dbData['_id'], "stategraph", base64content); + + + + # With initial mouse point, now drag around + if(mousePress[0]==1 and debounceMouse[0]==1 and "wfListThing" in directoriesNames[-1] and "wfBG:" in lucida.currentLevelName): + draggingScreen = 1 + differencePoint = [(float(p[0]-oldMousePosition[0]))/windowScreen.width,(float((p[1]-oldMousePosition[1]))/windowScreen.height)] + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + + countServices = 0; + for nodeData in WF.nodeList: + draggingScreenX = -differencePoint[0] + draggingScreenY = -differencePoint[1] + + xPlaceNew = nodeData.x+differencePoint[0]-lucida.WFXOffset + yPlaceNew = nodeData.y+differencePoint[1]-lucida.WFYOffset + lucida.updateObjectPosition("node:"+str(countServices),xPlaceNew,yPlaceNew) + lucida.updateObjectPosition("wfListText:"+str(countServices),xPlaceNew+textOffsetX,yPlaceNew+textOffsetY) + + linkCount = 0 + for link in nodeData.linkList: + + lucida.updateObjectPosition("IAmLine:" + str(countServices) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount += 1 + + # S0->S1. So S0 gets hit. It has link to S1. So S0 updates its from. What is its to? Reference S1. + + linkCount = 0; + for link in nodeData.linkList: + toNodeID = link.toNode + nodeDataTo = WF.nodeList[toNodeID]; + xPlaceNew2 = nodeDataTo.x+differencePoint[0]-lucida.WFXOffset + yPlaceNew2 = nodeDataTo.y+differencePoint[1]-lucida.WFYOffset + + lucida.updateObjectPosition2("IAmLine:" + str(countServices) + "IAmLine:" + str(linkCount),xPlaceNew2,yPlaceNew2) + linkCount += 1 + + countServices += 1 + + + + + + windowScreen.blitScreen() + + + if(draggingScreen==1 and mousePress[0] == 0): + draggingScreen=0 + WF = workflowList[draggingScreenWF] + lucida.WFXOffset += draggingScreenX + lucida.WFYOffset += draggingScreenY + + if(mousePress[0]==1 and debounceMouse[0]==0): + lucida.clickLevel(p[0],p[1]) + + + #Modify a workflow logic + if(currentActiveNode!=-1 and "wfListThing" in directoriesNames[-1] and keysPressed[109]==1 and debounceKey[109]==0): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeThing = WF.nodeList[currentActiveNode] + writeFileForUseMod(nodeThing.code) + nodeThing.code = input_impl() #User code + workflowData = compileWorkflow(getWFID) + db.update_workflow(WF.dbData['_id'], "code",workflowData); + + + + ######## Lets check if I pressed on a line. + if(mousePress[0]==1 and currentActiveNode==-1 and debounceMouse[0]==0 and "wfListThing" in directoriesNames[-1] and "node:" not in lucida.currentLevelName): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + #So go through every single line and see if within certain distance + # And to do that I must go through every single node. + countServices = 0; + floatP = [] + floatP.append(float(p[0])/windowScreen.width) + floatP.append(float(p[1])/windowScreen.height) + print(floatP) + for nodeData in WF.nodeList: + xStart = nodeData.x + yStart = nodeData.y; + + countServices+=1; + linkCount = 0; + for link in nodeData.linkList: + toNodeID = link.toNode + nodeDataTo = WF.nodeList[toNodeID]; + xEnd = nodeDataTo.x + yEnd = nodeDataTo.y + + #no division by 0 + if(xStart==xEnd): + xStart = 1; + m = (yStart-yEnd)/(xStart-xEnd); + b = yStart-m*xStart + d = abs((m*floatP[0])+(floatP[1]*-1) + b)/math.sqrt(m*m+1); + # This line is being pressed. + if(d<0.01 and keysPressed[109]==1 ): + writeFileForUseMod(link.code) + link.code = input_impl() + workflowData = compileWorkflow(getWFID) + db.update_workflow(WF.dbData['_id'], "code",workflowData); + linkCount+=1 + pass + + + # Unselect node, select node, and make link + if(mousePress[0]==1 and debounceMouse[0]==0 and "wfListThing" in directoriesNames[-1] and "node:" in lucida.currentLevelName): + idList = lucida.currentLevelName.split("node:") + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeID = int(idList[1]) + nodeData = WF.nodeList[nodeID] + + if(currentActiveNode==nodeID): # Click again to disable? + print("Unselect node") + currentActiveNode = -1 + lucida.updateObjectImage(lucida.currentLevelName,nodeIMG) + nodeData.active = 0 + elif(currentActiveNode==-1): # No previous selected, so select + print("Select Node:", nodeID) + currentActiveNode = nodeID + lucida.updateObjectImage(lucida.currentLevelName,nodeActiveIMG) + nodeData.active = 1 + else: # There is a selected, and it is not the selected. So make link? + print("Make link", currentActiveNode, nodeID) + #Check if "l" is pressed when attempting to link two nodes. + nodeDataOriginal = WF.nodeList[currentActiveNode] + + #If "l" is pressed, AND not already linked, make link. + if(keysPressed[108]==1): + isLinked = 0 + #Check for if already linked. If no link, link, if link, unlink. + for link in nodeDataOriginal.linkList: + if(link.toNode==nodeID): + isLinked = 1 + + #Link them + if(isLinked==0): + nodeDataOriginal.addLink("linkName",nodeID); + lucida.level(directoriesNames[-1],1); + pass + elif (isLinked==1): #TODO: Unlink + pass + + base64content = objToBase64(WF.nodeList) + db.update_workflow(workflowList[getWFID].dbData['_id'], "stategraph", base64content); + + + + + + #If "l" is not pressed, this means selecting a new node. + if(keysPressed[108]==0): + + lucida.updateObjectImage("node:"+str(currentActiveNode),nodeIMG) + nodeDataOriginal.active = 0 + currentActiveNode = nodeID + lucida.updateObjectImage(lucida.currentLevelName,nodeActiveIMG) + nodeData.active = 1 + + + + + + + + + + + if(mousePress[0]==0): + oldMousePosition = [] + oldMousePosition.append(-1) + oldMousePosition.append(-1) + + + + + #Debounce the key stack + loop255 = 0; + while loop255!=255: + debounceKey[loop255] = keysPressed[loop255] + loop255+=1 + + #Debounce the mouse + debounceMouse[0] = mousePress[0] + + #Makes the program terminate if the x on the window is pressed + for event in pygame.event.get(): + if event.type == pygame.QUIT: + crashTheProgram() + pass + +pygame.quit() diff --git a/tools/lucida-gui/gui_backend_cloud.py b/tools/lucida-gui/gui_backend_cloud.py new file mode 100644 index 000000000..8b69604c6 --- /dev/null +++ b/tools/lucida-gui/gui_backend_cloud.py @@ -0,0 +1,338 @@ +import os +import sys +import requests +import json +from pymongo import * +from bson.objectid import ObjectId + +class MongoDB(object): + def __init__(self): + mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR') + if mongodb_addr: + self.db = MongoClient(mongodb_addr, 27017).lucida + else: + self.db = MongoClient().lucida + + def get_services(self): + dictReturn = [] + + url = 'http://127.0.0.1:3000/api/v1/service' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['service_list'] + + return dictReturn + + def add_service(self): + """ + return code: + 0: success + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + # "location": location # location of service in local + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_service(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: service name not found + 2: name already used + 3: acronym already used + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + elif error == 'Updated name already used': + return 2 + elif error == 'Updated acronym already used': + return 3 + else: + return -1 + + def delete_service(self, _id): + """ + return code: + 0: success + 1: service not exist + """ + + post = { + "option": "delete", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + else: + return -1 + + def get_workflows(self): + dictReturn = [] + + url = 'http://127.0.0.1:3000/api/v1/workflow' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['workflow_list'] + + return dictReturn + + def add_workflow(self): + """ + return code: + 0: success + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_workflow(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: workflow name not found + 2: updated name already used + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + elif error == 'Updated name already used': + return 2 + else: + return -1 + + def delete_workflow(self, _id): + """ + return code: + 0: success + 1: workflow not exists + """ + + post = { + "option": "delete", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + else: + return -1 + + def add_instance(self, _id): + """ + return code: + 0: success + 1: service not valid + """ + + # list the attributes for the interface + post = { + "option": "add_empty", + "_id": _id + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['instance_id'] + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1, '' + else: + return -1, '' + + def update_instance(self, _id, instance_id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: instance not found + 2: host/port not valid + 3: host/port already used + """ + + post = { + "option": "update", + "_id": _id, + "instance_id": instance_id, + "op": op, + "value": value + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + elif error == 'Host/port pair is not valid': + return 2 + elif error == 'Updated host/port has already been used': + return 3 + else: + return -1 + + def delete_instance(self, _id, instance_id): + """ + return code: + 0: success + 1: instance not exist + """ + + post = { + "option": "delete", + "_id": _id, + "instance_id": instance_id + } + + url = 'http://127.0.0.1:3000/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + else: + return -1 + """ + # import this module and call start_server(name) to start + def start_server(self, name): + location, instance = self.search_path(name) + + # start each instance + for pair in instance: + port = pair['port'] + wrapper_begin = 'gnome-terminal -x bash -c "' + wrapper_end = '"' + code = 'cd ' + location + "; " + code = code + "make start_server port=" + str(port) + os.system(wrapper_begin + code + wrapper_end) + + def search_path(self, name): + # get collection for service information + collection = self.db.service_info + + result = collection.find({'name': name}) + + # check if current service is in MongoDB + count = result.count() + if count != 1: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service not in MongoDB.') + exit(1) + + return result[0]['location'], result[0]['instance'] + """ + +def validate_ip_port(s, p): + """ + Check if ip/port is valid with ipv4 + """ + + a = s.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + if p < 0 or p > 65535: + return False + return True + +db = MongoDB() \ No newline at end of file diff --git a/tools/lucida-gui/images/alive.png b/tools/lucida-gui/images/alive.png new file mode 100644 index 000000000..35e40192c Binary files /dev/null and b/tools/lucida-gui/images/alive.png differ diff --git a/tools/lucida-gui/images/bg.png b/tools/lucida-gui/images/bg.png new file mode 100644 index 000000000..55e414e5d Binary files /dev/null and b/tools/lucida-gui/images/bg.png differ diff --git a/tools/lucida-gui/images/dead.png b/tools/lucida-gui/images/dead.png new file mode 100644 index 000000000..1cc68c236 Binary files /dev/null and b/tools/lucida-gui/images/dead.png differ diff --git a/tools/lucida-gui/images/grayBG.png b/tools/lucida-gui/images/grayBG.png new file mode 100644 index 000000000..87f6a2eaf Binary files /dev/null and b/tools/lucida-gui/images/grayBG.png differ diff --git a/tools/lucida-gui/images/node.png b/tools/lucida-gui/images/node.png new file mode 100644 index 000000000..020cbef31 Binary files /dev/null and b/tools/lucida-gui/images/node.png differ diff --git a/tools/lucida-gui/images/nodeActive.png b/tools/lucida-gui/images/nodeActive.png new file mode 100644 index 000000000..365639db7 Binary files /dev/null and b/tools/lucida-gui/images/nodeActive.png differ diff --git a/tools/lucida-gui/images/serviceListButton.png b/tools/lucida-gui/images/serviceListButton.png new file mode 100644 index 000000000..06a082209 Binary files /dev/null and b/tools/lucida-gui/images/serviceListButton.png differ diff --git a/tools/lucida-gui/images/yellowBG.png b/tools/lucida-gui/images/yellowBG.png new file mode 100644 index 000000000..8c3b1b7cd Binary files /dev/null and b/tools/lucida-gui/images/yellowBG.png differ diff --git a/tools/lucida-gui/textrect.py b/tools/lucida-gui/textrect.py new file mode 100755 index 000000000..264b18af2 --- /dev/null +++ b/tools/lucida-gui/textrect.py @@ -0,0 +1,115 @@ +#! /usr/bin/env python + +class TextRectException: + def __init__(self, message = None): + self.message = message + def __str__(self): + return self.message + +def render_textrect(string, font, rect, text_color, background_color, justification=0): + """Returns a surface containing the passed text string, reformatted + to fit within the given rect, word-wrapping as necessary. The text + will be anti-aliased. + + Takes the following arguments: + + string - the text you wish to render. \n begins a new line. + font - a Font object + rect - a rectstyle giving the size of the surface requested. + text_color - a three-byte tuple of the rgb value of the + text color. ex (0, 0, 0) = BLACK + background_color - a three-byte tuple of the rgb value of the surface. + justification - 0 (default) left-justified + 1 horizontally centered + 2 right-justified + + Returns the following values: + + Success - a surface object with the text rendered onto it. + Failure - raises a TextRectException if the text won't fit onto the surface. + """ + + import pygame + + final_lines = [] + + requested_lines = string.splitlines() + + # Create a series of lines that will fit on the provided + # rectangle. + + for requested_line in requested_lines: + if font.size(requested_line)[0] > rect.width: + words = list(requested_line) + # if any of our words are too long to fit, return. + for word in words: + if font.size(word)[0] >= rect.width: + raise TextRectException, "The word " + word + " is too long to fit in the rect passed." + # Start a new line + accumulated_line = "" + for word in words: + test_line = accumulated_line + word + "" + # Build the line while the words fit. + if font.size(test_line)[0] < rect.width: + accumulated_line = test_line + else: + final_lines.append(accumulated_line) + accumulated_line = word + " " + final_lines.append(accumulated_line) + else: + final_lines.append(requested_line) + + # Let's try to write the text out on the surface. + + #surface = pygame.Surface(rect.size) + surface = pygame.Surface([640,480], pygame.SRCALPHA, 32) + surface = surface.convert_alpha() + + # surface.fill(background_color) + + accumulated_height = 0 + for line in final_lines: + if accumulated_height + font.size(line)[1] >= rect.height: + raise TextRectException, "Once word-wrapped, the text string was too tall to fit in the rect." + if line != "": + tempsurface = font.render(line, 1, text_color) + if justification == 0: + surface.blit(tempsurface, (0, accumulated_height)) + elif justification == 1: + surface.blit(tempsurface, ((rect.width - tempsurface.get_width()) / 2, accumulated_height)) + elif justification == 2: + surface.blit(tempsurface, (rect.width - tempsurface.get_width(), accumulated_height)) + else: + raise TextRectException, "Invalid justification argument: " + str(justification) + accumulated_height += font.size(line)[1] + + return surface + + +if __name__ == '__main__': + import pygame + import pygame.font + from pygame.locals import * + + pygame.init() + + display = pygame.display.set_mode((400, 400)) + + my_font = pygame.font.Font(None, 22) + + my_string = "Hi there! I'm a nice bit of wordwrapped text. Won't you be my friend? Honestly, wordwrapping is easy, with David's fancy new render_textrect() function.\nThis is a new line.\n\nThis is another one.\n\n\nAnother line, you lucky dog." + + my_rect = pygame.Rect((40, 40, 300, 300)) + + rendered_text = render_textrect(my_string, my_font, my_rect, (216, 216, 216), (48, 48, 48), 1) + + if rendered_text: + display.blit(rendered_text, my_rect.topleft) + + pygame.display.update() + + while not pygame.event.wait().type in (QUIT, KEYDOWN): + pass + + + diff --git a/tools/start_all_tmux.sh b/tools/start_all_tmux.sh deleted file mode 100755 index 723627df1..000000000 --- a/tools/start_all_tmux.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# This script is used to start all of the microservices -# for the Lucida project by creating a tmux window for each -# server in the background. -# Example calls: -# $./start_all_tmux.sh secure -# $./start_all_tmux.sh -# To attach to the tmux session -# use the following command: -# $tmux a -t lucida - -# source the port properties -. ../lucida/config.properties - -if [ "$1" == "test" ]; then - SESSION_NAME="lu-test" -else - SESSION_NAME="lucida" -fi - -# Check if session already exists -tmux has-session -t ${SESSION_NAME} -if [ $? -eq 0 ]; then - echo "Session ${SESSION_NAME} already exists." - exit 0; -elif [ -n "$TMUX" ]; then - echo "Already in a tmux session" - exit 0; -else - echo "Session ${SESSION_NAME} does not exit. Creating a ${SESSION_NAME} session." -fi - -# Check to see if we should run on http/ws (non-secure) or https/wss (secure) -if [ "$1" == "secure" ]; then - echo "Enabling secure host" - # Getting the host IP address - export ASR_ADDR_PORT="wss://$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'):$CMD_PORT" - export SECURE_HOST=true - - # Generate self-signed certificates - cd $(pwd)/../lucida/commandcenter/ - chmod +x gen_certs.sh - ./gen_certs.sh - cd $(pwd)/../../tools -else - echo "Enabling non-secure host" - export ASR_ADDR_PORT="ws://localhost:$CMD_PORT" -fi - -declare -a commandcenter=("CMD" "$(pwd)/../lucida/commandcenter/") -declare -a questionanswering=("QA" "$(pwd)/../lucida/questionanswering/OpenEphyra/") -declare -a imagematching=("IMM" "$(pwd)/../lucida/imagematching/opencv_imm/") -declare -a calendar=("CA" "$(pwd)/../lucida/calendar/") -declare -a speechrecognition=("ASR" "$(pwd)/../lucida/speechrecognition/kaldi_gstreamer_asr/") -declare -a imageclassification=("IMC" "$(pwd)/../lucida/djinntonic/imc/") -declare -a digitrecognition=("DIG" "$(pwd)/../lucida/djinntonic/dig/") -declare -a facerecognition=("FACE" "$(pwd)/../lucida/djinntonic/face") -declare -a weather=("WE" "$(pwd)/../lucida/weather") -declare -a botframework=("BFI" "$(pwd)/../lucida/botframework-interface") -declare -a musicservice=("MS" "$(pwd)/../lucida/musicservice") - -if [ "$1" == "test" ]; then - declare -a services=( - ) -else - declare -a services=( - commandcenter - speechrecognition) -fi - -services+=( - questionanswering - imagematching - calendar - imageclassification - digitrecognition - facerecognition - weather - botframework - musicservice) - -# Create the session -tmux new-session -s ${SESSION_NAME} -d - -# Create the service windows -TMUX_WIN=0 -for i in "${services[@]}" -do - NAME=$i[0] - SERV_PATH=$i[1] - if [ $TMUX_WIN == 0 ]; then - tmux rename-window -t ${SESSION_NAME}:$TMUX_WIN ${!NAME} - else - tmux new-window -n ${!NAME} -t ${SESSION_NAME} - fi - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "cd ${!SERV_PATH}" C-m - if [ "$1" == "test" ]; then - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "make start_test" C-m - else - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "make start_server" C-m - fi - ((TMUX_WIN++)) -done - -# Start out on the first window when we attach -tmux select-window -t ${SESSION_NAME}:0 - diff --git a/tools/template.sh b/tools/template.sh new file mode 100644 index 000000000..eee344d83 --- /dev/null +++ b/tools/template.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +echo "=============================================" +echo " Lucida Micro Service Registry Tools" +echo " made in May 31, 2017" +echo "=============================================" +echo "" + +create_folder () { + cd ../lucida + cp -rf template/$1 . + mv $1 $2 + cd $2 + find ./ -type f -exec sed -i "s/template/$2/g" {} \; + find ./ -type f -exec sed -i "s/TPL/$2/g" {} \; + find . -depth -name '*template*' -execdir rename "s/template/$2/g" {} \; + find . -depth -name '*TPL*' -execdir rename "s/TPL/$2/g" {} \; + cd .. + echo "[Info] Template folder is ready!" +} + +OP="" +if [ "$1" = "add" ]; then + OP="add" +elif [ "$1" = "delete" ]; then + OP="delete" +else + echo "### Specify what you want to do (add or delete)" + printf "### Enter you want to do: " + read OP + echo "" +fi + +if [ "$OP" = "add" ]; then + NAME_VALID=1 + while [ $NAME_VALID -ne 0 ]; do + echo "### Specify your service name (e.g. musicservice)." + printf "### Enter your service name: " + read NAME + if [ "$NAME" = "" ]; then + echo "[Error] Service name cannot be empty! Please try another one!" + else + NAME_VALID=0 + fi + done + + echo "" + echo "### Specify the programming language you want to you in your programming. If C++/Java/Python, then template will be provided." + printf "### Enter the programming language: " + read LAN + LAN="$(tr [A-Z] [a-z] <<< "$LAN")" + if [ "$LAN" = "c++" ]; then + LAN="cpp" + fi + + if [ "$LAN" = "cpp" -o "$LAN" = "java" -o "$LAN" = "python" ]; then + # do copy template folder of cpp to lucida + if [ -d $NAME ]; then + echo "[Error] service already exists!" + exit 1 + else + create_folder $LAN $NAME + fi + else + # create an empty folder + if [ -d $NAME ]; then + echo "[Error] service already exists!" + exit 1 + else + mkdir $NAME + echo "[Info] Template folder is ready!" + fi + fi +fi \ No newline at end of file