diff --git a/Dockerfile b/Dockerfile index 17fa7a9fc..a53e710b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,18 +24,10 @@ RUN apt-get install -y make ## install lucida RUN mkdir -p /usr/local/lucida ADD . /usr/local/lucida -WORKDIR "/usr/local/lucida/tools" -RUN /bin/bash apt_deps.sh -RUN /bin/bash install_python.sh -RUN /bin/bash install_java.sh -RUN /bin/bash install_opencv.sh -RUN /bin/bash install_thrift.sh -RUN /bin/bash install_fbthrift.sh -RUN /bin/bash install_mongodb.sh -WORKDIR "/usr/local/lucida/lucida" -RUN /usr/bin/make -RUN /bin/bash commandcenter/apache/install_apache.sh -RUN mkdir -p /etc/letsencrypt/live/host +WORKDIR "/usr/local/lucida/" +RUN make dep_core \ + && /bin/bash commandcenter/apache/install_apache.sh \ + && mkdir -p /etc/letsencrypt/live/host ### function docker-flush(){ ### dockerlist=$(docker ps -a -q) diff --git a/Makefile b/Makefile index 370f7e07e..689202649 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ DOCKER_CONTAINER=lucida VERSION=latest +CUR_DIR=$(shell pwd) ifeq ($(THREADS),) THREADS=$(shell grep -c ^processor /proc/cpuinfo) @@ -19,15 +20,6 @@ export LD_LIBRARY_PATH=/usr/local/lib local: cd tools && sudo make && cd ../lucida && make -start_all: - cd tools && ./start_all_tmux.sh - -start_all_secure: - cd tools && ./start_all_tmux.sh secure - -start_test_all: - cd tools && ./start_all_tmux.sh test - all_service: cd lucida && make all @@ -36,3 +28,9 @@ clean_all_service: clean_all_tools: cd tools && make clean + +dep_core: + cd tools && sudo make && cd ../lucida/commandcenter && make + +start_lucida: + cd $(CUR_DIR)/lucida/commandcenter && make start_server diff --git a/lucida/botframework-interface/deps/install_node.sh b/lucida/botframework-interface/deps/install_node.sh index d76bb852c..99822a941 100755 --- a/lucida/botframework-interface/deps/install_node.sh +++ b/lucida/botframework-interface/deps/install_node.sh @@ -1,3 +1,3 @@ #!/bin/bash -sudo apt-get install npm && sudo npm install -g n && sudo n lts +sudo apt-get install -y npm && sudo npm install -g n && sudo n lts diff --git a/lucida/calendar/CalendarClient/CalendarClient.java b/lucida/calendar/CalendarClient/CalendarClient.java index fbe4ac5dd..1960fd792 100755 --- a/lucida/calendar/CalendarClient/CalendarClient.java +++ b/lucida/calendar/CalendarClient/CalendarClient.java @@ -1,9 +1,6 @@ //Java packages -import java.io.FileInputStream; -import java.io.InputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.Properties; //Thrift java libraries import org.apache.thrift.TException; @@ -22,17 +19,11 @@ public class CalendarClient { public static void main(String [] args) throws IOException { - // Collect the port number. - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("CA_PORT"); - Integer port = Integer.valueOf(port_str); - if (args.length == 1) { - port = Integer.parseInt(args[0]); - } else { - System.out.println("Using default port for Calendar Client: " + port); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); } + Integer port = Integer.valueOf(args[0]); // Query. String LUCID = "Clinc"; diff --git a/lucida/calendar/CalendarClient/start-Calendar-client.sh b/lucida/calendar/CalendarClient/start-Calendar-client.sh index d79380b29..6a8a2460d 100755 --- a/lucida/calendar/CalendarClient/start-Calendar-client.sh +++ b/lucida/calendar/CalendarClient/start-Calendar-client.sh @@ -1,8 +1,9 @@ #!/bin/bash + # Add Thrift classes to class path export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:thrift # Add other jar files to class path export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar # run the server -java -cp "$JAVA_CLASS_PATH" CalendarClient "$@" +java -cp "$JAVA_CLASS_PATH" CalendarClient "$1" diff --git a/lucida/calendar/Makefile b/lucida/calendar/Makefile index 472761d01..b58c76f25 100644 --- a/lucida/calendar/Makefile +++ b/lucida/calendar/Makefile @@ -24,10 +24,15 @@ corenlp: fi start_server: - ./gradlew run + @if [ "$(port)" != "" ]; then \ + ./gradlew run -Pargs=$(port); \ + fi start_test: - cd CalendarClient; ./start-Calendar-client.sh + @if [ "$(port)" != "" ]; then \ + cd CalendarClient; \ + ./start-Calendar-client.sh $(port); \ + fi clean: rm -rf src/main/java/thrift && rm -rf CalendarClient/thrift && rm -rf CalendarClient/*.class && rm -rf build && rm -rf lib diff --git a/lucida/calendar/build.gradle b/lucida/calendar/build.gradle index defc0fb1d..66a892c43 100644 --- a/lucida/calendar/build.gradle +++ b/lucida/calendar/build.gradle @@ -39,5 +39,4 @@ dependencies { compile name: 'stanford-corenlp-3.7.0' compile name: 'xom-1.2.10-src' compile name: 'xom' - } diff --git a/lucida/calendar/src/main/java/calendar/CalendarDaemon.java b/lucida/calendar/src/main/java/calendar/CalendarDaemon.java index f16b8d892..1e3272c04 100644 --- a/lucida/calendar/src/main/java/calendar/CalendarDaemon.java +++ b/lucida/calendar/src/main/java/calendar/CalendarDaemon.java @@ -1,13 +1,10 @@ package calendar; -import java.io.FileInputStream; -import java.io.InputStream; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.Properties; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.server.TNonblockingServer; @@ -34,11 +31,12 @@ public class CalendarDaemon { public static void main(String [] args) throws TTransportException, IOException, InterruptedException { - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("CA_PORT"); - Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + TProcessor proc = new LucidaService.AsyncProcessor( new CAServiceHandler.AsyncCAServiceHandler()); TNonblockingServerTransport transport = new TNonblockingServerSocket(port); @@ -47,7 +45,7 @@ public static void main(String [] args) .protocolFactory(new TBinaryProtocol.Factory()) .transportFactory(new TFramedTransport.Factory()); final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); - System.out.println("CA at port " + port_str); + System.out.println("CA at port " + port); server.serve(); } } diff --git a/lucida/commandcenter/api/Blackbox_api.py b/lucida/commandcenter/api/Blackbox_api.py new file mode 100644 index 000000000..95bcbb4fc --- /dev/null +++ b/lucida/commandcenter/api/Blackbox_api.py @@ -0,0 +1,38 @@ +from flask import * +from Database import db + +blackbox_api = Blueprint('blackbox_api', __name__, template_folder='templates') + +@blackbox_api.route('/api/v1/blackbox', methods = ['GET', 'POST']) +def blackbox_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': start + } + """ + if request.method == 'GET': + pass + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'start': + if '_id' not in requestFields or 'instance_id' not in requestFields: + error = {'error': 'Field missing for starting server'} + return jsonify(error), 422 + + ret = db.start_server(requestFields['_id'], requestFields['instance_id']) + print("haha") + if ret == 1: + error = {'error': 'Instance not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Server successfully started!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/Database.py b/lucida/commandcenter/api/Database.py new file mode 100644 index 000000000..a3095bf2f --- /dev/null +++ b/lucida/commandcenter/api/Database.py @@ -0,0 +1,471 @@ +import os +import sys +from pymongo import * +from bson.objectid import ObjectId + +class MongoDB(object): + def __init__(self): + mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR') + if mongodb_addr: + self.db = MongoClient(mongodb_addr, 27017).lucida + else: + self.db = MongoClient().lucida + + def get_services(self): + dictReturn = [] + + service_list = self.db["service_info"].find() + count_service = service_list.count() + for i in range(count_service): + document = service_list[i] + document['_id'] = str(document['_id']) + dictReturn.append(document) + + return dictReturn + + def add_service(self, name, acronym, input_type, learn_type): + """ + return code: + 0: success + 1: name has already exists + 2: acronym has already used + """ + + num = 0 + instance = [] + collection = self.db.service_info + + # check if current service name is used + count = collection.count({'name': name}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service name already used.') + return 1, '' + + # check if current service acronym is used + count = collection.count({'acronym': acronym}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] service acronym already used.') + return 2, '' + + # list the attributes for the interface + post = { + "name": name, # name of service + "acronym": acronym, # acronym of service + "num": num, # biggest id of instance + "instance": instance, # host/port pair of instances + "input": input_type, # input type + "learn": learn_type # learn type + # "location": location # location of service in local + } + + # insert the service information into MongoDB + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def add_empty_service(self): + """ + return code: + 0: success + """ + + collection = self.db.service_info + + name = '' + acronym = '' + input_type = 'text' + learn_type = 'none' + num = 0 + instance = [] + + post = { + "name": name, # name of service + "acronym": acronym, # acronym of service + "num": num, # number of instance + "instance": instance, # host/port pair of instances + "input": input_type, # input type + "learn": learn_type # learn type + # "location": location # location of service in local + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + + def update_service(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: service name not found + 2: updated name already used + 3: updated acronym already used + """ + + collection = self.db.service_info + + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] service not exists in MongoDB.') + return 1 + + # check if update no difference return success + result = collection.find({'_id': ObjectId(_id)})[0] + if result[op] == value: + return 0 + + if op == 'name': + count = collection.count({'name': value}) + if count != 0: + print('[python error] Updated name already used') + return 2 + + if op == 'acronym': + count = collection.count({'acronym': value}) + if count != 0: + print('[python error] Updated acronym already used') + return 3 + + collection.update({'_id': ObjectId(_id)}, {'$set': {op: value }}) + return 0 + + def delete_service(self, _id): + """ + return code: + 0: success + 1: service not exist + """ + + collection = self.db.service_info + + # check if current service is in MongoDB + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] service not exists in MongoDB.') + return 1 + + collection.remove({'_id': ObjectId(_id)}) + return 0 + + def get_workflows(self): + dictReturn = [] + + workflow_list = self.db["workflow_info"].find() + count_workflow = workflow_list.count() + for i in range(count_workflow): + document = workflow_list[i] + document['_id'] = str(document['_id']) + dictReturn.append(document) + + return dictReturn + + def add_workflow(self, name, input_type, classifier_path, class_code, stategraph): + """ + return code: + 0: success + 1: workflow name already exists + """ + + collection = self.db.workflow_info + + # check if current workflow is in MongoDB + count = collection.count({'name': name}) + if count != 0: + #collection.delete_many({"name" : sys.argv[2]}) + print('[python error] workflow name already used.') + return 1, '' + + # list the attributes for the interface + post = { + "name": name, # name of workflow + "input": input_type, # allowed input type + "classifier": classifier_path, # classifier data path + "code": class_code, # code for implementation of the workflow class + "stategraph": stategraph # metadata for state graph + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def add_empty_workflow(self): + """ + return code: + 0: success + """ + + collection = self.db.workflow_info + + name = '' + input_type = [] + classifier_path = '' + class_code = '' + stategraph = '' + + post = { + "name": name, # name of workflow + "input": input_type, # allowed input type + "classifier": classifier_path, # classifier data path + "code": class_code, # code for implementation of the workflow class + "stategraph": stategraph # metadata for state graph + } + + post_id = collection.insert_one(post).inserted_id + return 0, str(post_id) + + def update_workflow(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: workflow name not found + 2: updated name already used + """ + + collection = self.db.workflow_info + + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] workflow not exists') + return 1 + + # check if update no difference return success + result = collection.find({'_id': ObjectId(_id)})[0] + if result[op] == value: + return 0 + + if op == 'name': + count = collection.count({'name': value}) + if count != 0: + print('[python error] Updated name already used') + return 2 + + collection.update({'_id': ObjectId(_id)}, {'$set': {op: value }}) + return 0 + + def delete_workflow(self, _id): + """ + return code: + 0: success + 1: workflow not exists + """ + + collection = self.db.workflow_info + + # check if current workflow is in MongoDB + count = collection.count({'_id': ObjectId(_id)}) + if count == 0: + print('[python error] workflow not exists') + return 1 + + collection.remove({'_id': ObjectId(_id)}) + return 0 + + def add_instance(self, _id, name, host, port, location): + """ + return code: + 0: success + 1: host/port not valid + 2: service not exist + 3: host/port already used + """ + + collection = self.db.service_info + if host == 'localhost': + host = '127.0.0.1' + + if not validate_ip_port(host, port): + print('[python error] Host/port pair is not valid.') + return 1, '' + + # check if current service is in MongoDB + object_id = ObjectId(_id) + count = collection.count({'_id': object_id}) + if count != 1: + print('[python error] service not exists in MongoDB.') + return 2, '' + + """ + # check if name is used + result = collection.find({'name': service_name, 'instance.name': name}) + if result.count() != 0: + print('[python error] Instance name has already been used.') + return 3 + """ + + # check if host and port is used + result = collection.find({'instance' : { '$elemMatch': { 'host': host, 'port': port}}}) + if result.count() != 0: + print('[python error] Host/port has already been used.') + return 3, '' + + result = collection.find({'_id': object_id}) + instance_id = str(result[0]['num']) + collection.update_one({'_id': object_id}, {'$inc': {'num': 1}}) + collection.update_one({'_id': object_id}, {'$push': {'instance': { + 'name': name, + 'host': host, + 'port': port, + 'location': location, + 'id': instance_id + }}}) + return 0, instance_id + + def add_empty_instance(self, _id): + """ + return code: + 0: success + 1: service not exists + """ + + collection = self.db.service_info + + name = '' + host = '127.0.0.1' + port = 0 + location = '' + object_id = ObjectId(_id) + count = collection.count({'_id': object_id}) + if count != 1: + print('[python error] service not exists in MongoDB.') + return 1, '' + + result = collection.find({'_id': object_id}) + instance_id = str(result[0]['num']) + collection.update_one({'_id': object_id}, {'$inc': {'num': 1}}) + collection.update_one({'_id': object_id}, {'$push': {'instance': { + 'name': name, + 'host': host, + 'port': port, + 'location': location, + 'id': instance_id + }}}) + return 0, instance_id + + def update_instance(self, _id, instance_id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: instance not found + 2: host/port pair not valid + 3: host/port pair already used + """ + + collection = self.db.service_info + if op == 'host': + if value == 'localhost': + value = '127.0.0.1' + + # check if current service is in MongoDB + object_id = ObjectId(_id) + result = collection.find({'_id': object_id, 'instance.id': instance_id}) + if result.count() != 1: + print('[python error] Instance name not exists.') + return 1 + + # check update nothing + cur_instance = {} + instance_result = result[0]['instance'] + for instance in instance_result: + if instance['id'] == instance_id: + cur_instance = instance + if instance[op] == value: + return 0 + + if op == 'host': + old_port = cur_instance['port'] + + if not validate_ip_port(value, old_port): + print('[python error] Host/port pair is not valid.') + return 2 + + result = collection.find({'instance': {'$elemMatch': {'host': value, 'port': old_port}}}) + if result.count() != 0: + print('[python error] Updated host/port has already been used') + return 3 + + if op == 'port': + old_host = cur_instance['host'] + + if not validate_ip_port(old_host, value): + print('[python error] Host/port pair is not valid.') + return 2 + + result = collection.find({'instance': {'$elemMatch': {'host': old_host, 'port': value}}}) + if result.count() != 0: + print('[python error] Updated host/port has already been used') + return 3 + + op = 'instance.$.'+op + collection.update({'_id': object_id, 'instance.id': instance_id}, {'$set': {op: value}}) + return 0 + + def delete_instance(self, _id, instance_id): + """ + return code: + 0: success + 1: instance not exist + """ + collection = self.db.service_info + + # check if current service is in MongoDB + object_id = ObjectId(_id) + result = collection.find({'_id': object_id, 'instance.id': instance_id}) + if result.count() != 1: + print('[python error] Instance name not exists.') + return 1 + + collection.update({'_id':object_id}, {'$pull': {'instance': {'id': instance_id}}}) + return 0 + + def start_server(self, _id, instance_id): + collection = self.db.service_info + + # check if current service is in MongoDB + object_id = ObjectId(_id) + result = collection.find({'_id': object_id, 'instance.id': instance_id}) + if result.count() != 1: + print('[python error] Instance name not exists.') + return 1 + + cur_instance = {} + instance_result = result[0]['instance'] + for instance in instance_result: + if instance['id'] == instance_id: + cur_instance = instance + + port = cur_instance['port'] + location = cur_instance['location'] + + # following code may change depend on how to start server + wrapper_begin = 'gnome-terminal -x bash -c "' + wrapper_end = '"' + code = 'cd ' + location + "; " + code = code + "make start_server port=" + str(port) + os.system(wrapper_begin + code + wrapper_end) + return 0 + +def validate_ip_port(s, p): + """ + Check if ip/port is valid with ipv4 + """ + if s == 'localhost': + s = '127.0.0.1' + a = s.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + if p < 0 or p > 65535: + return False + return True + +db = MongoDB() \ No newline at end of file diff --git a/lucida/commandcenter/api/Instance_api.py b/lucida/commandcenter/api/Instance_api.py new file mode 100644 index 000000000..d3ba07e18 --- /dev/null +++ b/lucida/commandcenter/api/Instance_api.py @@ -0,0 +1,97 @@ +from flask import * +from Database import db + +instance_api = Blueprint('instance_api', __name__, template_folder='templates') + +@instance_api.route('/api/v1/instance', methods = ['GET', 'POST']) +def instance_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + pass + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if '_id' not in requestFields or 'name' not in requestFields or \ + 'host' not in requestFields or 'port' not in requestFields or \ + 'location' not in requestFields: + error = {'error': 'Field missing for adding instance'} + return jsonify(error), 422 + + ret, instance_id = db.add_instance(requestFields['_id'], requestFields['name'], + requestFields['host'], requestFields['port'], requestFields['location']) + + if ret == 1: + error = {'error': 'Host/port pair not valid'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Host/port pair already used'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Instance successfully added!', 'instance_id': instance_id} + return jsonify(result), 200 + + elif option == 'add_empty': + if '_id' not in requestFields: + error = {'error': 'Field missing for adding instance'} + return jsonify(error), 422 + + ret, instance_id = db.add_empty_instance(requestFields['_id']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Instance successfully added!', 'instance_id': instance_id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'instance_id' not in requestFields or \ + 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating instance'} + return jsonify(error), 422 + + ret = db.update_instance(requestFields['_id'], requestFields['instance_id'], + requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Instance not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Host/port pair is not valid'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Updated host/port has already been used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Instance successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields or 'instance_id' not in requestFields: + error = {'error': 'Field missing for deleting instance'} + return jsonify(error), 422 + + ret = db.delete_instance(requestFields['_id'], requestFields['instance_id']) + + if ret == 1: + error = {'error': 'Instance not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Instance successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/Service_api.py b/lucida/commandcenter/api/Service_api.py new file mode 100644 index 000000000..03504c34e --- /dev/null +++ b/lucida/commandcenter/api/Service_api.py @@ -0,0 +1,86 @@ +from flask import * +from Database import db + +service_api = Blueprint('service_api', __name__, template_folder='templates') + +@service_api.route('/api/v1/service', methods = ['GET', 'POST']) +def service_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + result = db.get_services() + JSON_obj = {'service_list': result} + return jsonify(JSON_obj), 200 + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if 'name' not in requestFields or 'acronym' not in requestFields or \ + 'input' not in requestFields or 'learn' not in requestFields: + error = {'error': 'Field missing for adding service'} + return jsonify(error), 422 + + ret, _id = db.add_service(requestFields['name'], requestFields['acronym'], + requestFields['input'], requestFields['learn']) + + if ret == 1: + error = {'error': 'Service name has already existed'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Service acronym has already used'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Service successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'add_empty': + ret, _id = db.add_empty_service() + + if ret == 0: + result = {'success': 'Service successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating service'} + return jsonify(error), 422 + + ret = db.update_service(requestFields['_id'], requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Updated name already used'} + return jsonify(error), 422 + elif ret == 3: + error = {'error': 'Updated acronym already used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Service successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields: + error = {'error': 'Field missing for deleting service'} + return jsonify(error), 422 + + ret = db.delete_service(requestFields['_id']) + + if ret == 1: + error = {'error': 'Service not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Service successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/Workflow_api.py b/lucida/commandcenter/api/Workflow_api.py new file mode 100644 index 000000000..8eada57bb --- /dev/null +++ b/lucida/commandcenter/api/Workflow_api.py @@ -0,0 +1,80 @@ +from flask import * +from Database import db + +workflow_api = Blueprint('workflow_api', __name__, template_folder='templates') + +@workflow_api.route('/api/v1/workflow', methods = ['GET', 'POST']) +def workflow_api_route(): + """ + request json object (see detail in documents of API): + { + 'option': add/update/delete + } + """ + if request.method == 'GET': + result = db.get_workflows() + JSON_obj = {'workflow_list': result} + return jsonify(JSON_obj), 200 + + elif request.method == 'POST': + requestFields = request.get_json() + + if 'option' not in requestFields: + error = {'error': 'No option privided'} + return jsonify(error), 422 + + option = requestFields['option'] + + if option == 'add': + if 'name' not in requestFields or 'input' not in requestFields or \ + 'classifier' not in requestFields or 'code' not in requestFields: + error = {'error': 'Field missing for adding workflow'} + return jsonify(error), 422 + + ret, _id = db.add_workflow(requestFields['name'], requestFields['input'], + requestFields['classifier'], requestFields['code']) + + if ret == 1: + error = {'error': 'Workflow name has already existed'} + return jsonify(error), 422 + elif ret == 0: + result = {'success': 'Workflow successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'add_empty': + ret, _id = db.add_empty_workflow() + + if ret == 0: + result = {'success': 'Workflow successfully added!', '_id': _id} + return jsonify(result), 200 + + elif option == 'update': + if '_id' not in requestFields or 'op' not in requestFields or 'value' not in requestFields: + error = {'error': 'Field missing for updating workflow'} + return jsonify(error), 422 + + ret = db.update_workflow(requestFields['_id'], requestFields['op'], requestFields['value']) + + if ret == 1: + error = {'error': 'Workflow not exists'} + return jsonify(error), 422 + elif ret == 2: + error = {'error': 'Updated name already used'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Workflow successfully updated!'} + return jsonify(success), 200 + + elif option == 'delete': + if '_id' not in requestFields: + error = {'error': 'Field missing for deleting workflow'} + return jsonify(error), 422 + + ret = db.delete_workflow(requestFields['_id']) + + if ret == 1: + error = {'error': 'Workflow not exists'} + return jsonify(error), 422 + elif ret == 0: + success = {'success': 'Workflow successfully deleted!'} + return jsonify(success), 200 \ No newline at end of file diff --git a/lucida/commandcenter/api/__init__.py b/lucida/commandcenter/api/__init__.py new file mode 100644 index 000000000..264e28053 --- /dev/null +++ b/lucida/commandcenter/api/__init__.py @@ -0,0 +1 @@ +__all__ = ['Database', 'Service_api', 'Workflow_api', 'Instance_api', 'Blackbox_api'] \ No newline at end of file diff --git a/lucida/commandcenter/app.py b/lucida/commandcenter/app.py index 92d8f91df..c692a53c7 100644 --- a/lucida/commandcenter/app.py +++ b/lucida/commandcenter/app.py @@ -7,7 +7,7 @@ '/../../tools/thrift-0.9.3/lib/py/build/lib*')[0]) from controllers import * -from controllers.Parser import cmd_port +from api import * from flask import * from threading import Thread import logging @@ -25,6 +25,10 @@ app.register_blueprint(Create.create) app.register_blueprint(Learn.learn) app.register_blueprint(Infer.infer) +app.register_blueprint(Service_api.service_api) +app.register_blueprint(Workflow_api.workflow_api) +app.register_blueprint(Instance_api.instance_api) +app.register_blueprint(Blackbox_api.blackbox_api) # Session. app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' @@ -43,7 +47,7 @@ def flask_listener(): threaded=True) def web_socket_listener(): - print 'Start web socket at ' + str(cmd_port) + print 'Start web socket at 8081' logging.basicConfig(level=logging.DEBUG, format="%(levelname)8s %(asctime)s %(message)s ") logging.debug('Starting up server') @@ -52,13 +56,13 @@ def web_socket_listener(): # For wss (with ASR capability) if os.environ.get('SECURE_HOST'): print 'Starting secure web socket' - WebSocket.Application().listen(cmd_port, ssl_options={ + WebSocket.Application().listen(8081, ssl_options={ "certfile":"certs/server.crt", "keyfile":"certs/server.key"}) # For ws (without ASR capability) else: print 'Starting non-secure web socket' - WebSocket.Application().listen(cmd_port) + WebSocket.Application().listen(8081) WebSocket.tornado.ioloop.IOLoop.instance().start() diff --git a/lucida/commandcenter/controllers/Config.py b/lucida/commandcenter/controllers/Config.py index a843337f4..37e764e3f 100644 --- a/lucida/commandcenter/controllers/Config.py +++ b/lucida/commandcenter/controllers/Config.py @@ -1,176 +1,143 @@ -from Service import Service -from Graph import Graph, Node -from Parser import port_dic -from dcm import* +#!/usr/bin/env python2 -# The maximum number of texts or images for each user. -# This is to prevent the server from over-loading. -MAX_DOC_NUM_PER_USER = 30 # non-negative inetegr +from Service import Service +from pymongo import MongoClient +from Database import database +import os, sys, re -# Train or load the query classifier. -# If you set it to 'load', it is assumed that -# models are already saved in `../models`. TRAIN_OR_LOAD = 'train' # either 'train' or 'load' +""" +Train or load the query classifier. +If you set it to 'load', it is assumed that +models are already saved in `../models`. +""" + +SERVICES = {} +""" +Service dict that store all available services dynamically +Host IP addresses are resolved dynamically: +either set by Kubernetes or localhost. +""" + +WFList = {} +""" +Workflow dict that store all available workflows dynamically +""" - - -####################### How does a workflow work? Reference firstWorkFlow as a walkthrough example. - -#Contains serviceName and data to pass. Needed for batch (and thereby parallel) processing. -class serviceRequestData(object): - - def __init__(self,nameOfService,argData): - self.argumentData = argData - self.serviceName = nameOfService - - -class workFlow(object): - def __init__(self): - self.currentState = 0; # What state on the state graph - self.isEnd = False; - self.batchedData = [] - - - - - -class firstWorkflow(workFlow): - - - def processCurrentState(self,inputModifierText,inputModifierImage): - print "Executing state logic"; - - if(self.currentState==0): - print "State 0"; - self.currentState = 1; # This decides what state to go to next - # batchedData contains a list of service Requests. The function parameter is serviceRequestData(serviceName,dataToPassToService). - # Eg. "QA",inputModifierText[0]) means to pass to QA microservice with whatever was in the inputModifierText[0] (The text from the Lucida prompt)) - self.batchedData = [serviceRequestData("QA",[unicode("How old is Johann")]),serviceRequestData("QA",inputModifierText[0])]; - return; - - if(self.currentState==1): - print "State 1"; - # [1] is being passed as the input. This value came from: serviceRequestData("QA",inputModifierText[0]) - # It is based on the positioning of the previous serviceRequestData batch. - # Eg. [0] = serviceRequestData("QA",[unicode("How old is Johann")], [1] = serviceRequestData("QA",inputModifierText[0]) - #That means the second entry from state0 is being passed to it. - self.batchedData = [serviceRequestData("QA",inputModifierText[1])] - self.isEnd = True # This indicates the workflow is complete - return; - - - -class QAWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("QA",inputModifierText[0])]; - self.isEnd = True; - return; - -class IMMWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("IMM",inputModifierImage[0])]; - self.isEnd = True; - return; - -class CAWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("CA",inputModifierText[0])]; - self.isEnd = True; - return; - -class IMCWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("IMC",inputModifierImage[0])]; - self.isEnd = True; - return; - -class FACEWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("FACE",inputModifierImage[0])]; - self.isEnd = True; - return; - -class DIGWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("DIG",inputModifierImage[0])]; - self.isEnd = True; - return; - -class ENSEMBLEWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("ENSEMBLE",inputModifierText[0])]; - self.isEnd = True; - return; - - -class MSWF(workFlow): - def processCurrentState(self,inputModifierText,inputModifierImage): - if(self.currentState==0): - self.batchedData = [serviceRequestData("MS",inputModifierText[0])]; - self.isEnd = True; - return; - - -WFList = { - "IMMWF" : IMMWF(), - "firstWorkFlow" : firstWorkflow(), - "QAWF" : QAWF(), - "CAWF" : CAWF(), - "IMCWF" : IMCWF(), - "FACEWF" : FACEWF(), - "DIGWF" : DIGWF(), - "ENSEMBLEWF" : ENSEMBLEWF(), - "MSWF" : MSWF() - +CLASSIFIER_DESCRIPTIONS = { + 'text': [], + 'image': [], + 'text_image': [] } +""" +Classifier stored for each workflow +""" +CLASSIFIER_PATH = {} +""" +Classifier data path for each workflow +""" +SESSION = {} +""" +Structure used to save the state/context across requests in a session +example: +SESSION = { : +} +""" +LEARNERS = { 'audio' : [], 'image' : [], 'text' : [] } +""" +Store all service supporting learn +""" -# Pre-configured services. -# The ThriftClient assumes that the following services are running. -# Host IP addresses are resolved dynamically: -# either set by Kubernetes or localhost. - -SERVICES = { - 'IMM' : Service('IMM', int(port_dic["imm_port"]), 'image', 'image'), - 'QA' : Service('QA', int(port_dic["qa_port"]), 'text', 'text'), - 'CA' : Service('CA', int(port_dic["ca_port"]), 'text', None), - 'IMC' : Service('IMC', int(port_dic["imc_port"]), 'image', None), - 'FACE' : Service('FACE', int(port_dic["face_port"]), 'image', None), - 'DIG' : Service('DIG', int(port_dic["dig_port"]), 'image', None), - 'WE' : Service('WE', int(port_dic["we_port"]), 'text', None), - 'MS' : Service('MS', int(port_dic["ms_port"]), 'text', None), - } +def appendServiceRequest(data,arg1,arg2,arg3): + data.append(serviceRequestData(arg1,arg2,[arg3])) + return data -CLASSIFIER_DESCRIPTIONS = { - 'text' : { 'class_QA' : Graph([Node('QAWF')]), - 'class_CA' : Graph([Node('CAWF')]), - 'class_WE' : Graph([Node('WEWF')]), - 'class_MS' : Graph([Node('MSWF')]) }, - 'image' : { 'class_IMM' : Graph([Node('IMMWF')]), - 'class_IMC' : Graph([Node('IMCWF')]), - 'class_FACE' : Graph([Node('FACEWF')]), - 'class_DIG' : Graph([Node('DIGWF')]) }, - 'text_image' : { 'class_QA': Graph([Node('QAWF')]), - 'class_IMM' : Graph([Node('IMMWF')]), - 'class_IMC' : Graph([Node('IMCWF')]), - 'class_FACE' : Graph([Node('FACEWF')]), - 'class_DIG' : Graph([Node('DIGWF')]), } - } +class serviceRequestData(object): +#Contains serviceName and data to pass. Needed for batch (and thereby parallel) processing. -# TODO: Should I have this in its own Config file? -# Structure used to save the state/context across requests in a session -# example: -# SESSION = { : -# 'graph': , -# 'data': -# } -SESSION = {} + def __init__(self,batchedDataName,nameOfService,argData): + self.argumentData = argData + self.serviceName = nameOfService + self.batchedDataName = batchedDataName + +class workFlow(object): + + def __init__(self): + self.currentState = 0 # What state on the state graph + self.isEnd = False + self.pause = False + self.batchedData = [] + +def load_config(): + """ + Update the config needed for Lucida + """ + + # Load mongodb + db = database.db + for input_t in LEARNERS: + del LEARNERS[input_t][:] + SESSION.clear() + # Update service list + SERVICES.clear() + service_list = db["service_info"].find() + count_service = service_list.count() + for i in range(count_service): + service_obj = service_list[i] + acn = service_obj['acronym'] + instance = service_obj['instance'] + input_type = service_obj['input'] + learn_type = service_obj['learn'] + _id = str(service_obj['_id']) + # check if uninitialized + if acn == '': + if acn not in SERVICES: + SERVICES[acn] = 1 + else: + SERVICES[acn] += 1 + continue + # get num of available and uninitialized instance + num = len(instance) + avail_instance = [x for x in instance if x['name'] != ''] + avail = len(avail_instance) + unini = num-avail + SERVICES[acn] = Service(acn, input_type, learn_type, unini, avail, avail_instance, _id) + # update learners + if learn_type == 'none': + pass + else: + LEARNERS[learn_type].append(_id) + + # Update workflow list, current only support single service workflow + for input_t in CLASSIFIER_DESCRIPTIONS: + del CLASSIFIER_DESCRIPTIONS[input_t][:] + WFList.clear() + workflow_list = db["workflow_info"].find() + count_workflow = workflow_list.count() + for i in range(count_workflow): + workflow_obj = workflow_list[i] + name = workflow_obj['name'] + input_list = workflow_obj['input'] + classifier = workflow_obj['classifier'] + # check if uninitialized + if name == '': + continue + CLASSIFIER_PATH[name] = classifier + code = workflow_obj['code'] + exec(code) + for input_t in input_list: + CLASSIFIER_DESCRIPTIONS[input_t].append(name) + WFList[name] = eval(name+"()") + return 0 + +def get_service_withid(_id): + for service in SERVICES: + if service == '': + continue + if SERVICES[service]._id == _id: + return SERVICES[service] + +load_config() diff --git a/lucida/commandcenter/controllers/ConfigChecker.py b/lucida/commandcenter/controllers/ConfigChecker.py deleted file mode 100644 index 5c305c708..000000000 --- a/lucida/commandcenter/controllers/ConfigChecker.py +++ /dev/null @@ -1,43 +0,0 @@ -from Config import * - -# Check Config.py. -if MAX_DOC_NUM_PER_USER <= 0: - print 'MAX_DOC_NUM_PER_USER must be non-negative' - exit() -if not (TRAIN_OR_LOAD == 'train' or TRAIN_OR_LOAD == 'load'): - print 'TRAIN_OR_LOAD must be either train or load' - exit() -for service_name, service_obj in SERVICES.iteritems(): - if not service_name == service_obj.name: - print service_name, 'must be the same as', service_obj.name - exit() -for input_type in CLASSIFIER_DESCRIPTIONS: - print '@@@@@ When query type is ' + input_type + ', there are ' + \ - str(len(CLASSIFIER_DESCRIPTIONS[input_type])) + ' possible classes:' - i = 0 - for query_class_name, graph in \ - CLASSIFIER_DESCRIPTIONS[input_type].iteritems(): - print str(i) + '. ' + query_class_name + ' -- needs to invoke ' \ - + graph.to_string() - for node in graph.node_list: - if not node.service_name in SERVICES: - print 'CLASSIFIER_DESCRIPTIONS misconfigured' - print 'Unrecognized service:', node.sercice_name - exit() - if input_type == 'text': - if SERVICES[node.service_name].input_type != 'text': - print 'CLASSIFIER_DESCRIPTIONS misconfigured' - print node.service_name, 'does not receive text' - exit() - elif input_type == 'image': - if SERVICES[node.service_name].input_type != 'image': - print 'CLASSIFIER_DESCRIPTIONS misconfigured' - print node.service_name, 'does not receive image' - exit() - elif input_type == 'text_image': - pass - else: - print 'CLASSIFIER_DESCRIPTIONS misconfigured' - print 'input type must be either text, image, or text_image' - exit() - i += 1 diff --git a/lucida/commandcenter/controllers/Create.py b/lucida/commandcenter/controllers/Create.py index bd73e3f7c..02ebbd6eb 100644 --- a/lucida/commandcenter/controllers/Create.py +++ b/lucida/commandcenter/controllers/Create.py @@ -1,7 +1,11 @@ from flask import * +from QueryClassifier import query_classifier from AccessManagement import login_required from ThriftClient import thrift_client from Service import Service +from Utilities import log +import Config +import socket create = Blueprint('create', __name__, template_folder='templates') @@ -9,14 +13,42 @@ @login_required def create_route(): options = {} + if request.method == 'POST': + if 'request' in request.form: + if request.form['request'] == 'Update': + Config.load_config() + query_classifier.__init__(Config.TRAIN_OR_LOAD, Config.CLASSIFIER_DESCRIPTIONS) + try: - # Retrieve pre-configured services. services_list = [] for service in thrift_client.SERVICES.values(): if isinstance(service, Service): - host, port = service.get_host_port() - services_list.append((service.name, host, port)) - options['service_list']= sorted(services_list, key=lambda i: i[0]) + services_list.append(service.name) + else: + for i in range(service): + services_list.append('(undef)') + options['service_list'] = sorted(services_list, key=lambda i: i[0]) + # Retrieve pre-configured services. + instances_list = [] + for service in thrift_client.SERVICES.values(): + if isinstance(service, Service): + for i in range(service.num): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + name = service.instance[i]['name'] + host = service.instance[i]['host'] + port = service.instance[i]['port'] + result = 1 + sock.settimeout(1) + result = sock.connect_ex((host, port)) + sock.settimeout(None) + if result == 0: + instances_list.append((service.name, name, host, port, "running")) + else: + instances_list.append((service.name, name, host, port, "stop")) + sock.close() + for i in range(service.unini): + instances_list.append((service.name, '(undef)', '', '', '')) + options['instance_list'] = sorted(instances_list, key=lambda i: i[0]) except Exception as e: log(e) options['error'] = e diff --git a/lucida/commandcenter/controllers/Database.py b/lucida/commandcenter/controllers/Database.py index 8c81af8fd..36aba118a 100644 --- a/lucida/commandcenter/controllers/Database.py +++ b/lucida/commandcenter/controllers/Database.py @@ -3,9 +3,14 @@ from base64 import b64encode from Utilities import log import os -import Config from Memcached import memcached +MAX_DOC_NUM_PER_USER = 30 +""" +MAX_DOC_NUM_PER_USER = 30 # non-negative integer +The maximum number of texts or images for each user. +This is to prevent the server from over-loading. +""" class Database(object): # Name of the algorithm to use for password encryption. @@ -92,27 +97,27 @@ def get_username(self, interface, interface_uid): return None # Adds the uploaded image. - def add_image(self, username, image_data, label, image_id): + def add_image(self, username, image_data, label, image_id, _id): self.get_image_collection(username).insert_one( {'label': label, 'data': b64encode(image_data), # encoded - 'image_id': image_id}) + 'image_id': image_id, 'service_id': _id}) # Deletes the specified image. - def delete_image(self, username, image_id): - self.get_image_collection(username).remove({'image_id': image_id}) + def delete_image(self, username, image_id, _id): + self.get_image_collection(username).remove({'image_id': image_id, 'service_id': _id}) # Returns all the images by username. - def get_images(self, username): + def get_images(self, username, _id): log('Retrieving all images from images_' + username) # Notice image['data'] was encoded using Base64. - return [image for image in self.get_image_collection(username).find({}, { '_id': 0 })] + return [image for image in self.get_image_collection(username).find({'service_id': _id}, { '_id': 0 })] # Checks whether the user can add one more image. def check_add_image(self, username): if self.get_image_collection(username).count() >= \ - Config.MAX_DOC_NUM_PER_USER: + MAX_DOC_NUM_PER_USER: raise RuntimeError('Sorry. You can only add ' + - str(Config.MAX_DOC_NUM_PER_USER) + \ + str(MAX_DOC_NUM_PER_USER) + \ ' images at most') # Returns the number of images by username. def count_images(self, username): @@ -120,27 +125,27 @@ def count_images(self, username): return self.get_image_collection(username).count() # Adds the knowledge text. - def add_text(self, username, text_type, text_data, text_id): + def add_text(self, username, text_type, text_data, text_id, _id): self.get_text_collection(username).insert_one( {'type': text_type, 'text_data': text_data, - 'text_id': text_id}) + 'text_id': text_id, 'service_id': _id}) # Deletes the knowledge text. - def delete_text(self, username, text_id): + def delete_text(self, username, text_id, _id): self.get_text_collection(username).delete_one( - {'text_id': text_id}) + {'text_id': text_id, 'service_id': _id}) # Returns the knowledge text by username. - def get_text(self, username): + def get_text(self, username, _id): log('Retrieving text from text_' + username) - return [text for text in self.get_text_collection(username).find({}, { '_id': 0 })] + return [text for text in self.get_text_collection(username).find({'service_id': _id}, { '_id': 0 })] # Checks whether the user can add one more piece of text. def check_add_text(self, username): if self.get_text_collection(username).count() >= \ - Config.MAX_DOC_NUM_PER_USER: + MAX_DOC_NUM_PER_USER: raise RuntimeError('Sorry. You can only add ' + - str(Config.MAX_DOC_NUM_PER_USER) + \ + str(MAX_DOC_NUM_PER_USER) + \ ' pieces of text at most') database = Database() diff --git a/lucida/commandcenter/controllers/Decision.py b/lucida/commandcenter/controllers/Decision.py deleted file mode 100644 index 57b1ddf24..000000000 --- a/lucida/commandcenter/controllers/Decision.py +++ /dev/null @@ -1,13 +0,0 @@ -import abc - -class Decision(object): - __metaclass__ = abc.ABCMeta - - def __init__(self, next_node=None, lucida_response=''): - self.next_node = next_node - self.lucdia_repsonse = lucida_response - - @abc.abstractmethod - def logic_method(self, response_data, service_graph, dcm_node): - """Decision making logic""" - return diff --git a/lucida/commandcenter/controllers/Graph.py b/lucida/commandcenter/controllers/Graph.py deleted file mode 100644 index 728cde364..000000000 --- a/lucida/commandcenter/controllers/Graph.py +++ /dev/null @@ -1,78 +0,0 @@ -from Service import Service -from Queue import Queue - - -class Node(object): - # Constructor. - def __init__(self, service_name, to_indices=[]): - if to_indices is None: - to_indices = [] - self.service_name = service_name - self.to_indices = to_indices - - def to_string(self): - rtn = [self.service_name, str(len(self.to_indices))] - for to_index in self.to_indices: - rtn.append(str(to_index)) - return str(rtn) - - -class Graph(object): - # Constructor. - def __init__(self, node_list): - self.node_list = node_list - # Start index is always initialized to 0 - # TODO: get rid of start_index and pass it into ThriftClient.infer() - # since it is only used when traversing through the graph - self.start_index = 0 - # Validate. - global_has_seen = set() - start_node = self.get_node(self.start_index) - fringe = Queue() - has_seen = set() - fringe.put(start_node) - has_seen.add(start_node) - while not fringe.empty(): - curr_node = fringe.get() - for to_index in curr_node.to_indices: - to_node = self.get_node(to_index) - if to_node in has_seen: - if 'DCM' not in curr_node.service_name: - print 'Invalid graph: cyclic without decision node' - exit() - else: - fringe.put(to_node) - has_seen.add(to_node) - global_has_seen = has_seen.union(global_has_seen) - if len(global_has_seen) < len(node_list): - print 'Invalid graph: unconnected' - # Create a set of service names for fast look-up. - self.service_names = set() - for node in node_list: - self.service_names.add(node.service_name) - - def get_node(self, index): - if index < 0 or index >= len(self.node_list): - print 'Invalid index' - exit() - else: - return self.node_list[index] - - def get_next_index(self, curr_node, next_service_name): - for index in curr_node.to_indices: - if self.get_node(index).service_name == next_service_name: - return index - print 'Invalid next service ' + next_service_name - exit() - - def to_string(self): - rtn = '' - for node in self.node_list: - rtn += node.to_string() - rtn += ', ' - rtn += 'and start index: ' - rtn += str(self.start_index) - return rtn - - def has_service(self, service_name): - return service_name in self.service_names diff --git a/lucida/commandcenter/controllers/Infer.py b/lucida/commandcenter/controllers/Infer.py index be7f7d42f..6874228d6 100644 --- a/lucida/commandcenter/controllers/Infer.py +++ b/lucida/commandcenter/controllers/Infer.py @@ -4,7 +4,8 @@ from ThriftClient import thrift_client from QueryClassifier import query_classifier from Utilities import log, check_image_extension -from Parser import port_dic +from copy import deepcopy +from Config import WFList import Config import os import json @@ -14,10 +15,6 @@ @login_required def generic_infer_route(form, upload_file): options = {} - if os.environ.get('ASR_ADDR_PORT'): - options['asr_addr_port'] = os.environ.get('ASR_ADDR_PORT') - else: - options['asr_addr_port'] = 'ws://localhost:' + port_dic["cmd_port"] try: # Deal with POST requests. if request.method == 'POST': @@ -27,35 +24,45 @@ def generic_infer_route(form, upload_file): speech_input = form['speech_input'] if 'speech_input' in form \ else '' print '@@@@@@@@@@', speech_input - image_input = [upload_file.read()] if upload_file else None + image_input = [upload_file.read()] if upload_file else [] lucida_id = session['username'] # Check if context is saved for Lucida user # If not, classify query, otherwise restore session if lucida_id not in Config.SESSION: - services_needed = query_classifier.predict(speech_input, upload_file) - speech_input = [speech_input] + workflow_needed = query_classifier.predict(speech_input, upload_file) + text_input = [speech_input] + workflow = deepcopy(WFList[workflow_needed]) + options['result'] = thrift_client.infer(lucida_id, workflow, text_input, image_input) else: - services_needed = Config.SESSION[lucida_id]['graph'] - Config.SESSION[lucida_id]['data']['text'].append(speech_input) - speech_input = Config.SESSION[lucida_id]['data']['text'] - node = services_needed.get_node(0) - options['result'] = thrift_client.infer(lucida_id, node.service_name, speech_input, image_input) + if 'workflow' in Config.SESSION[lucida_id]: + workflow = Config.SESSION[lucida_id]['workflow'] + text_input = Config.SESSION[lucida_id]['resulttext'] + text_input.insert(0, speech_input) + options['result'] = thrift_client.infer(lucida_id, workflow, text_input, image_input) + else: + workflow_needed = query_classifier.predict(speech_input, upload_file) + text_input = Config.SESSION[lucida_id]['resulttext'] + text_input.insert(0, speech_input) + workflow = deepcopy(WFList[workflow_needed]) + options['result'] = thrift_client.infer(lucida_id, workflow, text_input, image_input) log('Result ' + options['result']) + ''' # Check if Calendar service is needed. # If so, JavaScript needs to receive the parsed dates. if services_needed.has_service('CA'): options['dates'] = options['result'] options['result'] = None + ''' except Exception as e: - log(e) - options['errno'] = 500 - options['error'] = str(e) - if 'code' in e and re.match("^4\d\d$", str(e.code)): - options['errno'] = e.code - if str(e) == 'TSocket read 0 bytes': - options['error'] = 'Back-end service encountered a problem' - if str(e).startswith('Could not connect to'): - options['error'] = 'Back-end service is not running' + log(e) + options['errno'] = 500 + options['error'] = str(e) + if 'code' in e and re.match("^4\d\d$", str(e.code)): + options['errno'] = e.code + if str(e) == 'TSocket read 0 bytes': + options['error'] = 'Back-end service encountered a problem' + if str(e).startswith('Could not connect to'): + options['error'] = 'Back-end service is not running' return options @infer.route('/infer', methods=['GET', 'POST']) @@ -65,7 +72,7 @@ def infer_route(): if os.environ.get('ASR_ADDR_PORT'): options['asr_addr_port'] = os.environ.get('ASR_ADDR_PORT') else: - options['asr_addr_port'] = 'ws://localhost:' + port_dic["cmd_port"] + options['asr_addr_port'] = 'ws://localhost:8081' if request.method == 'POST': options = generic_infer_route(request.form, request.files['file'] if 'file' in request.files else None) return render_template('infer.html', **options) @@ -82,5 +89,5 @@ def api_infer_route(): options = generic_infer_route(request.form, request.files['file'] if 'file' in request.files else None) if 'errno' in options: - return json.dumps(options), options['errno'] + return json.dumps(options), options['errno'] return json.dumps(options), 200 diff --git a/lucida/commandcenter/controllers/Learn.py b/lucida/commandcenter/controllers/Learn.py index 4ea454644..6dc902473 100644 --- a/lucida/commandcenter/controllers/Learn.py +++ b/lucida/commandcenter/controllers/Learn.py @@ -5,6 +5,7 @@ from Database import database from ThriftClient import thrift_client from Utilities import log, check_image_extension, check_text_input +import Config import re learn = Blueprint('learn', __name__, template_folder='templates') @@ -17,6 +18,7 @@ def generic_learn_route(op, form, upload_file): if op == 'add_image': image_type = 'image' label = form['label'] + _id = form['_id'] # Check the uploaded image. if upload_file.filename == '': raise RuntimeError('Empty file is not allowed') @@ -32,21 +34,23 @@ def generic_learn_route(op, form, upload_file): # Send the image to IMM. upload_file.close() thrift_client.learn_image(username, image_type, image_data, - image_id) + image_id, _id) # Add the image into the database. - database.add_image(username, image_data, label, image_id) + database.add_image(username, image_data, label, image_id, _id) # Delete image knowledge. elif op == 'delete_image': image_type = 'unlearn' image_id = form['image_id'] + _id = form['_id'] # Send the unlearn request to IMM. - thrift_client.learn_image(username, image_type, '', image_id) + thrift_client.learn_image(username, image_type, '', image_id, _id) # Delete the image from the database. - database.delete_image(username, image_id) + database.delete_image(username, image_id, _id) # Add text knowledge. elif op == 'add_text' or op == 'add_url': text_type = 'text' if op == 'add_text' else 'url' text_data = form['knowledge'] + _id = form['_id'] # Check the text knowledge. check_text_input(text_data) # Check whether the user can add one more piece of text. @@ -56,17 +60,18 @@ def generic_learn_route(op, form, upload_file): str(datetime.datetime.now())).hexdigest() # Send the text to QA. thrift_client.learn_text(username, text_type, - text_data, text_id) + text_data, text_id, _id) # Add the text knowledge into the database. - database.add_text(username, text_type, text_data, text_id) + database.add_text(username, text_type, text_data, text_id, _id) # Delete text knowledge. elif op == 'delete_text': text_type = 'unlearn' text_id = form['text_id'] + _id = form['_id'] # Send the unlearn request to QA. - thrift_client.learn_text(username, text_type, '', text_id) + thrift_client.learn_text(username, text_type, '', text_id, _id) # Delete the text from into the database. - database.delete_text(username, text_id) + database.delete_text(username, text_id, _id) else: raise RuntimeError('Did you click the button?') except Exception as e: @@ -85,13 +90,48 @@ def generic_learn_route(op, form, upload_file): @login_required def learn_route(): options = {} + # Deal with POST requests. if request.method == 'POST': options = generic_learn_route(request.form['op'], request.form, request.files['file'] if 'file' in request.files else None) + + # home page + if '_id' not in request.args: + options['text_list'] = [] + options['image_list'] = [] + for _id in Config.LEARNERS['text']: + service = Config.get_service_withid(_id) + options['text_list'].append({'name': service.name, '_id': _id}) + for _id in Config.LEARNERS['image']: + service = Config.get_service_withid(_id) + options['image_list'].append({'name': service.name, '_id': _id}) + return render_template('learn_home.html', **options) + try: - # Retrieve knowledge. - options['pictures'] = database.get_images(session['username']) - options['text'] = database.get_text(session['username']) + _id = request.args['_id'] + service = Config.get_service_withid(_id) + + options['_id'] = _id + options['service_name'] = service.name + + if _id in Config.LEARNERS['text']: + options['text_able'] = 1 + else: + options['text_able'] = 0 + + # check if text field is necessary + if _id in Config.LEARNERS['image']: + options['image_able'] = 1 + else: + options['image_able'] = 0 + + if options['text_able'] == 0 and options['image_able'] == 0: + abort(404) + + if options['text_able'] == 0: + options['pictures'] = database.get_images(session['username'], _id) + else: + options['text'] = database.get_text(session['username'], _id) except Exception as e: log(e) options['errno'] = 500 @@ -103,26 +143,20 @@ def learn_route(): def api_learn_add_del_route(op): if not op in allowed_endpoints: abort(404) - session['username'] = database.get_username(request.form['interface'], request.form['username']) - if session['username'] == None: - abort (403) - - session['logged_in'] = True - print '@@@@@@@@', session['username'] options = {} if not op == 'query': options = generic_learn_route(op, request.form, request.files['file'] if 'file' in request.files else None) else: try: - # Retrieve knowledge. + # Retrieve knowledge. if 'type' in request.form and request.form['type'] == 'text': - options['text'] = database.get_text(session['username']) + options['text'] = database.get_text(session['username'], request.form['_id']) elif 'type' in request.form and request.form['type'] == 'image': - options['pictures'] = database.get_images(session['username']) + options['pictures'] = database.get_images(session['username'], request.form['_id']) else: - options['pictures'] = database.get_images(session['username']) - options['text'] = database.get_text(session['username']) + options['pictures'] = database.get_images(session['username'], request.form['_id']) + options['text'] = database.get_text(session['username'], request.form['_id']) except Exception as e: log(e) options['errno'] = 500 diff --git a/lucida/commandcenter/controllers/Parser.py b/lucida/commandcenter/controllers/Parser.py deleted file mode 100644 index 569dc28f4..000000000 --- a/lucida/commandcenter/controllers/Parser.py +++ /dev/null @@ -1,21 +0,0 @@ -import ConfigParser, sys - - -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../config.properties"))) -port_dic = dict(cp.items('asection')) -cmd_port = int(port_dic['cmd_port']) diff --git a/lucida/commandcenter/controllers/QueryClassifier.py b/lucida/commandcenter/controllers/QueryClassifier.py index d11b06e00..c8fd0c4eb 100644 --- a/lucida/commandcenter/controllers/QueryClassifier.py +++ b/lucida/commandcenter/controllers/QueryClassifier.py @@ -27,6 +27,10 @@ def __init__(self, query_class_name_in): def predict(self, speech_input): return [self.query_class_name] + +class EmptyClassifier(object): + def predict(self, speech_input): + return [] class QueryClassifier(object): # Constructor. @@ -54,12 +58,16 @@ def train(self, input_type, query_classes): current_dir = os.path.abspath(os.path.dirname(__file__)) # If there is no or only one possible outcomes for the input type, # there is no need to train any classifier. - if len(query_classes) <= 1: - return DummyClassifier(query_classes.keys()[0]) + if len(query_classes) == 0: + return EmptyClassifier() + if len(query_classes) == 1: + return DummyClassifier(query_classes[0]) # Build DataFrame by going through all data files. data = DataFrame({'text': [], 'class': []}) for query_class_name in query_classes: - path = current_dir + '/../data/' + query_class_name + '.txt' + path = Config.CLASSIFIER_PATH[query_class_name] + if not os.path.isfile(path): + raise RuntimeError('Query classifer data file cannot found!') log('Opening ' + path) lines = [line.rstrip('\n') for line in open(path)] rows = [] @@ -133,9 +141,11 @@ def predict(self, speech_input, image_input): raise RuntimeError('Text and image cannot be both empty') # Convert speech_input to a single-element list. class_predicted = self.classifiers[input_type].predict([speech_input]) + if len(class_predicted) == 0: + raise RuntimeError('Input type has no service available') class_predicted = class_predicted[0] # ndarray to string log('Query classified as ' + class_predicted) - return self.CLASSIFIER_DESCRIPTIONS[input_type][class_predicted] + return class_predicted query_classifier = QueryClassifier(Config.TRAIN_OR_LOAD, diff --git a/lucida/commandcenter/controllers/Service.py b/lucida/commandcenter/controllers/Service.py index d4fd9ac07..468acf4ba 100644 --- a/lucida/commandcenter/controllers/Service.py +++ b/lucida/commandcenter/controllers/Service.py @@ -2,29 +2,32 @@ from Utilities import log class Service(object): - LEARNERS = { 'audio' : [], 'image' : [], 'text' : [] } # Constructor. - def __init__(self, name, port, input_type, learn_type): + def __init__(self, name, input_type, learn_type, unini, avail, avail_instance, _id): self.name = name - self.port = port + self.count = 0 if not (input_type == 'text' or input_type == 'image'): - print 'Can only process text and image' + log('Can only process text and image') exit() self.input_type = input_type - if not learn_type is None: - if not learn_type in Service.LEARNERS: - print 'Unrecognized learn_type' - exit() - Service.LEARNERS[learn_type].append(self) + self.learn_type = learn_type + self.unini = unini + self.num = avail + self.instance = avail_instance + self._id = _id def get_host_port(self): try: - host = 'localhost' - tcp_addr = os.environ.get(self.name + '_PORT_' + str(self.port) + '_TCP_ADDR') - if tcp_addr: - log('TCP address is resolved to ' + tcp_addr) - host = tcp_addr - return host, self.port + if self.num <= 0: + raise RuntimeError('No available instance for service ' + self.name) + cur_host = self.instance[self.count]['host'] + cur_port = self.instance[self.count]['port'] + self.count = (self.count + 1)%self.num + return cur_host, cur_port except Exception: raise RuntimeError('Cannot access service ' + self.name) + def get_host_port_withid(self, instance_id): + for obj in self.instance: + if obj['id'] == instance_id: + return obj['host'], obj['port'] diff --git a/lucida/commandcenter/controllers/ThriftClient.py b/lucida/commandcenter/controllers/ThriftClient.py index 85010eacc..97ee997ac 100644 --- a/lucida/commandcenter/controllers/ThriftClient.py +++ b/lucida/commandcenter/controllers/ThriftClient.py @@ -1,135 +1,127 @@ -from lucidatypes.ttypes import QueryInput, QuerySpec -from lucidaservice import LucidaService -from dcm import* -from flask import* - -from Config import WFList +from flask import * +import threading +import os +import sys from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +from lucidatypes.ttypes import QueryInput, QuerySpec +from lucidaservice import LucidaService +from Config import WFList from Utilities import log from Database import database import Config -import os -import sys + reload(sys) sys.setdefaultencoding('utf8') # to solve the unicode error - -import threading - - - - - #This is basically the thread starter function class FuncThread(threading.Thread): - def __init__(self, target, *args): - self._target = target - self._args = args - threading.Thread.__init__(self) - def run(self): - self._target(*self._args) - - + def __init__(self, target, *args): + self._target = target + self._args = args + threading.Thread.__init__(self) + def run(self): + self._target(*self._args) class ThriftClient(object): - # Constructor. - def __init__(self, SERVICES): - self.SERVICES = SERVICES - log('Pre-configured services: ' + str(SERVICES)) - - def create_query_input(self, type, data, tag_list): - query_input = QueryInput() - query_input.type = type - query_input.data = data - query_input.tags = tag_list - return query_input - - def create_query_spec(self, name, query_input_list): - query_spec = QuerySpec() - query_spec.name = name - query_spec.content = query_input_list - return query_spec - - def get_client_transport(self, service): - host, port = service.get_host_port() - print (host,port) - transport = TTransport.TFramedTransport(TSocket.TSocket(host, port)) - protocol = TBinaryProtocol.TBinaryProtocol(transport) - transport.open() - return LucidaService.Client(protocol), transport - - def send_query(self, LUCID, service_name, query_input_list): - query_spec = self.create_query_spec('query', query_input_list) - service = self.SERVICES[service_name] - client, transport = self.get_client_transport(service) - log('Sending infer request to ' + service.name) - result = client.infer(str(LUCID), query_spec) - transport.close() - return result - - - def learn_image(self, LUCID, image_type, image_data, image_id): - for service in Config.Service.LEARNERS['image']: # add concurrency? - knowledge_input = self.create_query_input( - image_type, [image_data], [image_id]) - client, transport = self.get_client_transport(service) - log('Sending learn_image request to IMM') - client.learn(str(LUCID), - self.create_query_spec('knowledge', [knowledge_input])) - transport.close() - - def learn_text(self, LUCID, text_type, text_data, text_id): - for service in Config.Service.LEARNERS['text']: # add concurrency? - knowledge_input = self.create_query_input( - text_type, [text_data], [text_id]) - client, transport = self.get_client_transport(service) - log('Sending learn_text request to QA') - client.learn(str(LUCID), - self.create_query_spec('knowledge', [knowledge_input])) - transport.close() - - - # Example usage - def executeThreadServiceRequest(self,service_name, inputData, LUCID, threadIDValue): - print("Thread ", threadIDValue, "executing", service_name, "with input", inputData) + # Constructor. + def __init__(self, SERVICES): + self.SERVICES = SERVICES + log('Pre-configured services: ' + str(SERVICES)) + + def create_query_input(self, type, data, tag_list): + query_input = QueryInput() + query_input.type = type + query_input.data = data + query_input.tags = tag_list + return query_input + + def create_query_spec(self, name, query_input_list): + query_spec = QuerySpec() + query_spec.name = name + query_spec.content = query_input_list + return query_spec + + def get_client_transport(self, host, port): + transport = TTransport.TFramedTransport(TSocket.TSocket(host, port)) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + transport.open() + return LucidaService.Client(protocol), transport + + def send_query(self, LUCID, service_name, query_input_list): + query_spec = self.create_query_spec('query', query_input_list) + service = self.SERVICES[service_name] + host = query_input_list[0].tags[0] + port = int(query_input_list[0].tags[1]) + client, transport = self.get_client_transport(host, port) + log('Sending infer request to ' + service.name) + result = client.infer(str(LUCID), query_spec) + transport.close() + return result + + def learn_image(self, LUCID, image_type, image_data, image_id, _id): + knowledge_input = self.create_query_input( + image_type, [image_data], [image_id]) + service = Config.get_service_withid(_id) + if service.num == 0: + raise RuntimeError('No available instance to learn knowledge') + for obj in service.instance: + instance_id = obj['id'] + host, port = service.get_host_port_withid(instance_id) + client, transport = self.get_client_transport(host, port) + log('Sending learn_image request to ' + service.name) + client.learn(str(LUCID), + self.create_query_spec('knowledge', [knowledge_input])) + transport.close() + + def learn_text(self, LUCID, text_type, text_data, text_id, _id): + knowledge_input = self.create_query_input( + text_type, [text_data], [text_id]) + service = Config.get_service_withid(_id) + if service.num == 0: + raise RuntimeError('No available instance to learn knowledge') + for obj in service.instance: + instance_id = obj['id'] + host, port = service.get_host_port_withid(instance_id) + client, transport = self.get_client_transport(host, port) + log('Sending learn_text request to ' + service.name) + client.learn(str(LUCID), + self.create_query_spec('knowledge', [knowledge_input])) + transport.close() + + # Example usage + def executeThreadServiceRequest(self,service_name, inputData, LUCID, threadIDValue): + log("Thread "+str(threadIDValue)+" executing "+service_name) service = self.SERVICES[service_name] host, port = service.get_host_port() tag_list = [host, str(port)] query_input_list = [self.create_query_input(service.input_type, inputData, tag_list)] resultText = self.send_query(LUCID, service_name, query_input_list) self.threadResults.insert(threadIDValue, resultText) - - - - -# TODO: split function into separate functions (DCM, creating QuerySpec) - def infer(self, LUCID, workflow_name, text_data, image_data): + + def infer(self, LUCID, workflow, text_data, image_data): - - response_data = { 'text': text_data, 'image': image_data } - self.threadResults = [] + response_data = { 'text': text_data, 'image': image_data } + self.threadResults = [] # workflow_name contains the name of the workflow, NOT the microservice. # This acquires the workflow class. - workflow = WFList[workflow_name] - workflow.__init__() - resultText = response_data['text'] - resultImage = [response_data['image']] + resultText = text_data + resultImage = image_data + passArgs = dict() + pause = False + while not workflow.isEnd and not pause: + batchedDataReturn = dict() + log("-------------NEXT ITERATION:STATE" + str(workflow.currentState)) + resultText = [ unicode(x) for x in resultText ] - while not workflow.isEnd: - - i = 0 - for x in resultText: - resultText[i] = [unicode(resultText)] # Text information must be unicode'd and array'd to be properly passed. IMAGE DATA DOES NOT HAVE THIS DONE TO IT. - i+= 1 - # Processes the current workflow state, and in the process finds if this is the final stage or if next stage exists. - workflow.processCurrentState(resultText,resultImage) + log("Acquiring Batch Request") + workflow.processCurrentState(1,batchedDataReturn,passArgs,resultText,resultImage) resultText = [] resultImage = [] @@ -138,24 +130,47 @@ def infer(self, LUCID, workflow_name, text_data, image_data): threadID = 0 #This is where batched execution initalizes and begins for x in workflow.batchedData: - print "_____Thread" + str(threadID) + "," + str(x.serviceName) + "," + str(x.argumentData) + log("_____Thread" + str(threadID) + "," + str(x.serviceName)) #Execute the desired microservice threadList.append(FuncThread(self.executeThreadServiceRequest, x.serviceName, x.argumentData, LUCID,threadID)) threadList[threadID].start() threadID+=1 + + log("Executed batch request") threadID = 0 #This is where batched execution joins together for x in workflow.batchedData: threadList[threadID].join() - print "============ThreadID" + str(threadID) - print "Output:" + self.threadResults[threadID] + log("============ThreadID" + str(threadID)) + log("Output:" + self.threadResults[threadID]) + batchedDataReturn[x.batchedDataName] = self.threadResults[threadID] resultText.insert(threadID, self.threadResults[threadID]) threadID+=1 + log("Do stuff after batch request") + pause, ret= workflow.processCurrentState(0,batchedDataReturn,passArgs,resultText,resultImage) + log("after processstate") + + # store workflow info into session + if pause: + if LUCID not in Config.SESSION: + Config.SESSION[LUCID] = {} + Config.SESSION[LUCID]['workflow'] = workflow + Config.SESSION[LUCID]['resulttext'] = resultText - - - return resultText[0] + + # pop workflow when workflow ends and store resulttext + if workflow.isEnd: + if LUCID in Config.SESSION: + if 'workflow' in Config.SESSION[LUCID]: + Config.SESSION[LUCID].pop('workflow', None) + Config.SESSION[LUCID]['resulttext'] = resultText + else: + Config.SESSION[LUCID] = {} + Config.SESSION[LUCID]['resulttext'] = resultText + + log("successful return") + return ret thrift_client = ThriftClient(Config.SERVICES) diff --git a/lucida/commandcenter/controllers/WebSocket.py b/lucida/commandcenter/controllers/WebSocket.py index 5c6c77398..976f85af8 100644 --- a/lucida/commandcenter/controllers/WebSocket.py +++ b/lucida/commandcenter/controllers/WebSocket.py @@ -24,10 +24,8 @@ import tornado.gen import tornado.concurrent -from Parser import cmd_port - from tornado.options import define -define("port", default=cmd_port, help="run on the given port", type=int) +define("port", default=8081, help="run on the given port", type=int) STATUS_EOS = -1 STATUS_SUCCESS = 0 diff --git a/lucida/commandcenter/controllers/__init__.py b/lucida/commandcenter/controllers/__init__.py index fd97c3acd..051ff8ff1 100644 --- a/lucida/commandcenter/controllers/__init__.py +++ b/lucida/commandcenter/controllers/__init__.py @@ -1,3 +1,3 @@ -__all__ = ['Main', 'AccessManagement', 'WebSocket', 'Service', 'Graph', - 'ThriftClient', 'Create', 'Learn', 'Infer', 'Parser', - 'QueryClassifier', 'Config', 'User', 'Utilities', 'Database', 'Memcached', 'Decision'] +__all__ = ['Main', 'AccessManagement', 'WebSocket', 'Service', + 'ThriftClient', 'Create', 'Learn', 'Infer', + 'QueryClassifier', 'Config', 'User', 'Utilities', 'Database', 'Memcached'] diff --git a/lucida/commandcenter/controllers/dcm/Decision.py b/lucida/commandcenter/controllers/dcm/Decision.py deleted file mode 100644 index 33dee183b..000000000 --- a/lucida/commandcenter/controllers/dcm/Decision.py +++ /dev/null @@ -1,26 +0,0 @@ -import abc - -class Decision(object): - __metaclass__ = abc.ABCMeta - - - def __init__(self, next_node=None, lucida_response=''): - self.next_node = next_node - self.lucdia_repsonse = lucida_response - - @abc.abstractmethod - def logic_method(self, response_data, service_graph, dcm_node): - """ - Decision logic method. - - Arg: - response_data: response text/image from previous services - service_graph: current workflow - dcm_node: current DCM node in workflow - - Return: - lucida_response: response to send back to user - next_node: next node to go to in workflow - """ - - return diff --git a/lucida/commandcenter/controllers/dcm/IMMDCM.py b/lucida/commandcenter/controllers/dcm/IMMDCM.py deleted file mode 100644 index 5b53a5bba..000000000 --- a/lucida/commandcenter/controllers/dcm/IMMDCM.py +++ /dev/null @@ -1,36 +0,0 @@ -import abc -from Decision import* - -not_monument = 'cow' - -class IMMDCM(Decision): - """ - Decision class for IMM decision making nodes. - """ - - def logic_method(self, response_data, service_graph, dcm_node): - """ - Decision logic for IMMDCM. Decides whether the response from IMM - is a monument or not and eventually asks QA where the image is - located. - - Arg: - response_data: response text/image from previous services - service_graph: current workflow - dcm_node: current DCM node in workflow - - Return: - lucida_response: response to send back to user - next_node: next node to go to in workflow - """ - # Check if image was recognized as a monument (not a cow) - # If not, ask user for more specific information - # Otherwise, send response from IMM (name of monument) to QA - if not_monument in response_data['text'][-1]: - self.lucida_response ='Cannot determine location of image. Please give me another image.' - self.next_node = service_graph.get_next_index(dcm_node, 'IMM') - else: - response_data['text'][-1] = 'Where is the location of ' + response_data['text'][-1] - self.lucida_response = '' - self.next_node = service_graph.get_next_index(dcm_node, 'QA') - return diff --git a/lucida/commandcenter/controllers/dcm/WEDCM.py b/lucida/commandcenter/controllers/dcm/WEDCM.py deleted file mode 100644 index 2010599ee..000000000 --- a/lucida/commandcenter/controllers/dcm/WEDCM.py +++ /dev/null @@ -1,32 +0,0 @@ -import abc -from Decision import* - -class WEDCM(Decision): - """ - Decision class for WE decision making nodes. - """ - - def logic_method(self, response_data, service_graph, dcm_node): - """ - Decision logic for WEDCM. Decides whether the response from WE - is the weather or not (asks the user again) and eventually, gets - the weather for the place. - - Arg: - response_data: response text/image from previous services - service_graph: current workflow - dcm_node: current DCM node in workflow - - Return: - lucida_response: response to send back to user - next_node: next node to go to in workflow - """ - # Check if weather was determined - # If not, ask user for more specific information then call WE - if 'No weather found' in response_data['text'][-1]: - self.lucida_response = 'Please state the city and state you are asking the weather for.' - self.next_node = service_graph.get_next_index(dcm_node, 'WE') - else: - self.lucida_response = '' - self.next_node = None - return diff --git a/lucida/commandcenter/controllers/dcm/__init__.py b/lucida/commandcenter/controllers/dcm/__init__.py deleted file mode 100644 index 338abc0ba..000000000 --- a/lucida/commandcenter/controllers/dcm/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['IMMDCM', 'WEDCM'] diff --git a/lucida/commandcenter/data/class_CA.txt b/lucida/commandcenter/data/class_CAWF.txt similarity index 100% rename from lucida/commandcenter/data/class_CA.txt rename to lucida/commandcenter/data/class_CAWF.txt diff --git a/lucida/commandcenter/data/class_DIG.txt b/lucida/commandcenter/data/class_DIGWF.txt similarity index 100% rename from lucida/commandcenter/data/class_DIG.txt rename to lucida/commandcenter/data/class_DIGWF.txt diff --git a/lucida/commandcenter/data/class_FACE.txt b/lucida/commandcenter/data/class_FACEWF.txt similarity index 100% rename from lucida/commandcenter/data/class_FACE.txt rename to lucida/commandcenter/data/class_FACEWF.txt diff --git a/lucida/commandcenter/data/class_IMC.txt b/lucida/commandcenter/data/class_IMCWF.txt similarity index 100% rename from lucida/commandcenter/data/class_IMC.txt rename to lucida/commandcenter/data/class_IMCWF.txt diff --git a/lucida/commandcenter/data/class_IMM.txt b/lucida/commandcenter/data/class_IMMWF.txt similarity index 100% rename from lucida/commandcenter/data/class_IMM.txt rename to lucida/commandcenter/data/class_IMMWF.txt diff --git a/lucida/commandcenter/data/class_MS.txt b/lucida/commandcenter/data/class_MSWF.txt similarity index 100% rename from lucida/commandcenter/data/class_MS.txt rename to lucida/commandcenter/data/class_MSWF.txt diff --git a/lucida/commandcenter/data/class_QA.txt b/lucida/commandcenter/data/class_QAWF.txt similarity index 100% rename from lucida/commandcenter/data/class_QA.txt rename to lucida/commandcenter/data/class_QAWF.txt diff --git a/lucida/commandcenter/data/class_WE.txt b/lucida/commandcenter/data/class_WEWF.txt similarity index 100% rename from lucida/commandcenter/data/class_WE.txt rename to lucida/commandcenter/data/class_WEWF.txt diff --git a/lucida/commandcenter/data/class_WFTaco.txt b/lucida/commandcenter/data/class_WFTaco.txt new file mode 100644 index 000000000..fab509ba7 --- /dev/null +++ b/lucida/commandcenter/data/class_WFTaco.txt @@ -0,0 +1,141 @@ +Who is the current president of the United States of America? +Who wrote the Walden? +Who founded Apple, Inc.? +Who is the first man on the moon? +Who founded the Black Panthers organization? +Who discovered prions? +Who are the members of the Rat Pack? +Who did Queen Victoria marry? +Who were the leaders of the French Revolution? +Who ended the Vietnam War? +When did Xi JinPing born? +When did Albert Einstein die? +When will the next iPhone come out? +When was the Munich agreement made? +When is Easter Sunday in 2014? +When was the first flight of the Wright brothers? +When was the Halley's Comet discovered? +When did Mike Tyson win the title? +When was the University of Michigan founded? +When did Hawaii become a state? +Where is Steve Jobs buried? +Where was Jack Ma born? +Where was the first KFC opened? +Where are the Headquarters of Facebook located? +Where does Barack Obama live? +Where will the 2016 Summer Olympics be held? +Where did ISIS come from? +Where is the River Nile located? +Where is an adults-only Club Med? +Where did Christopher Columbus die? +What was the number one song on the Billboard Top 100 in 2014? +What is the most watched video on YouTube of all time? +What nationality is Ted Cruz? +What did Ludwig van Beethoven compose? +What is the last book of the Bible? +What is the total length of the Golden Gate Bridge? +What is the most populated city in the world? +What is the homosexual residential area in San Francisco called? +What is the Avogadro's Constant? +What sport does David Beckham play? +How many weeks are there in a year? +How many passengers does Bay Area Rapid Transit serve annually? +How many people died in World War 2? +How many employees does Google have? +How many cups are in a quart? +How much is 1 dollar to RMB (Chinese Yuan)? +How much water is there in the Lake Erie? +How much does Earth weigh? +How often does a woman have a period? +How fast does a cheetah run? +Who founded Google? +Who launched Facebook? +Who created Amazon? +Who is the vice president of the United States? +Who is the president of France? +Who is the president of China? +Who is the father of C++? +Who invented Java? +Who created Python? +Who designed the first car? +Who is the richest person in the world? +Who is the greatest philanthropist of all time? +Who discovered America? +Who discovered neutron? +Who successful predicted the regular protein secondary structures? +Who first identified and isolated DNA? +Who is the author of the Harry Potter series? +Who is the author of the Canterbury Tales? +Who wrote A Brief History of Time? +Who wrote the Declaration of Independence? +Who composed Für Elise? +Who composed Piano Concerto No.5? +Who presented the Twin Earth thought experiment? +Who led the Confederate Army in the US Civil War? +Who led the Cuban Revolution in 1961? +Who is the first emperor of the Roman Empire? +Who is the first emperor of India? +Who is the last emperor of china? +Who is the last leader of the Soviet Union? +Who gave America its name? +What is my phone number? +Hey, Lucida, gimme my cellphone numbers. +Where is Eiffel Tower located? +Tell me what my favorite resturant is. +Lucida what should I have for breakfast? +What are some healthy breakfast ideas? +Are their any health benefits to coffee? +Is coffee bad for you? +How much coffee can I drink until I get sick? +Is coffee lethal? +What is my favorite kind of coffee? +Remind me what my favorite coffee is. +Where should I get my coffee today? +What’s Taylor Swift’s newest single? +What is the last music video Taylor Swift released? +Are Calvin Harris and Taylor Swift dating? +What is Calvin Harris’s real name? +What are the first 10 numbers of the fibonacci sequence? +What’s the solution to the fibonacci sequence? +Give me the 27th fibonacci number. +What does saranghae mean? +How do you pronounce fromage? +When does BTS’s new album come out? +What does HYYH stand for? +Who is wearing a black jacket? +Who is doing this presentation? +Who is presenting her research report? +Who likes the color blue? +Who likes flowers? +Who likes reading books? +Who likes playing football? +Who loves swimming? +Who is working on Lucida? +Who are the developers of Lucida? +Who are doing research on Sirius? +Who founded click? +Who founded the company? +Who goes to California every year? +Who just went to New York? +Who is my favorite? +Who is my wife? +Who is my husband? +Who is my son? +Who is my daughter? +Who is my grand father? +Who is my grandmother? +Who is my sister? +Who is my brother? +Who is my cousin? +Who is my aunt? +Who is my uncle? +Who behaves like a clown? +Who is funny? +Who is serious? +Who is outgoing? +Who is bad at programming? +Who is good at painting? +Who wants to be president? +Who plays the violin? +Who killed Lincoln? +Who loved her? diff --git a/lucida/commandcenter/templates/create.html b/lucida/commandcenter/templates/create.html index d5cd79d34..c4b99f20f 100644 --- a/lucida/commandcenter/templates/create.html +++ b/lucida/commandcenter/templates/create.html @@ -12,26 +12,59 @@ +
+ + + + {% for i in range(service_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
{{ service_list[i] }}
+
+ +
+
+

List of Service Instances

+
+
+
+ + - {% for i in service_list %} + {% for i in instance_list %} + + {% endfor %}
Service nameName Host PortAvailablity
{{ i[0] }} {{ i[1] }} {{ i[2] }}{{ i[3] }}{{ i[4] }}
+
+
+
+ +
+
+
+ {% endblock %} diff --git a/lucida/commandcenter/templates/infer.html b/lucida/commandcenter/templates/infer.html index 65f5cdf80..144694550 100644 --- a/lucida/commandcenter/templates/infer.html +++ b/lucida/commandcenter/templates/infer.html @@ -2,11 +2,13 @@ {% block content %} + + passed to calendar.js {% endif %} +--> diff --git a/lucida/commandcenter/templates/learn.html b/lucida/commandcenter/templates/learn.html index 58a6d9eb6..ec2687492 100644 --- a/lucida/commandcenter/templates/learn.html +++ b/lucida/commandcenter/templates/learn.html @@ -13,6 +13,13 @@

{{ error }}

 {% endif %} +
+
+

Service: {{ service_name }}

+
+
+ + {% if text_able == 1 %}

Add text knowledge:

@@ -22,7 +29,8 @@
- + +
@@ -39,7 +47,8 @@
- + +
@@ -58,6 +67,8 @@ + +
  • {% if i['type'] == 'url' %}{% endif %}{{ i['text_data'] }}{% if i['type'] == 'url' %}{% endif %}
    • @@ -67,7 +78,9 @@ {% endfor %} + {% endif %} + {% if image_able == 1 %}

    Add image knowledge:

    @@ -77,7 +90,8 @@
    - + +

    Upload your picture!

    @@ -102,6 +116,8 @@ + +
    @@ -110,8 +126,13 @@
    {% endfor %} + {% endif %}
    + +
    + Back +


    {% endblock %} diff --git a/lucida/commandcenter/templates/learn_home.html b/lucida/commandcenter/templates/learn_home.html new file mode 100644 index 000000000..7908a66ab --- /dev/null +++ b/lucida/commandcenter/templates/learn_home.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block content %} + + + +
    +
    +

    Main » Learn

    +
    + + {% if error %} +

    {{ error }}

     + {% endif %} + +
    +
    +

    Text:

    +
    +
    + +
    + + + + {% for i in range(text_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
    {{ text_list[i]["name"] }}
    +
    + +
    +
    +

    Image:

    +
    +
    + +
    + + + + {% for i in range(image_list|length) %} + + {% if i%4 == 3 %} + + {% endif %} + {% endfor %} + + +
    {{ image_list[i]["name"] }}
    +
    + +
    + +

    +{% endblock %} diff --git a/lucida/config.properties b/lucida/config.properties deleted file mode 100644 index ec9033926..000000000 --- a/lucida/config.properties +++ /dev/null @@ -1,12 +0,0 @@ -# This is the port configuration of LUCIDA -# WARNING: Never assign one port to more than one service. -# TODO: Adding your own service port. -CMD_PORT=8081 -QA_PORT=8082 -IMM_PORT=8083 -CA_PORT=8084 -IMC_PORT=8085 -FACE_PORT=8086 -DIG_PORT=8087 -WE_PORT=8088 -MS_PORT=8089 diff --git a/lucida/djinntonic/Makefile b/lucida/djinntonic/Makefile index 694ba266e..6d54f1e30 100644 --- a/lucida/djinntonic/Makefile +++ b/lucida/djinntonic/Makefile @@ -3,6 +3,7 @@ fname=djinn-weights-1.0.tar.gz all: cd tools \ + && sudo bash install_fbthrift.sh \ && sh install-protobuf.sh \ && sh install-caffe.sh \ && cp common.hpp caffe/distribute/include/caffe/common.hpp \ diff --git a/lucida/djinntonic/dig/DIGServer.cpp b/lucida/djinntonic/dig/DIGServer.cpp index 639fdbc6d..0395fab5e 100644 --- a/lucida/djinntonic/dig/DIGServer.cpp +++ b/lucida/djinntonic/dig/DIGServer.cpp @@ -5,8 +5,8 @@ #include "DIGHandler.h" #include -#include "Parser.h" #include +#include DEFINE_int32(num_of_threads, 4, @@ -16,21 +16,17 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; //using namespace facebook::windtunnel::treadmill::services::dig; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("DIG_PORT", portVal)) { - cout << "DIG port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -41,6 +37,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "DIG at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/dig/Makefile b/lucida/djinntonic/dig/Makefile index 71e1a8edf..26b0b95de 100644 --- a/lucida/djinntonic/dig/Makefile +++ b/lucida/djinntonic/dig/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./DIGServer + @if [ "$(port)" != "" ]; then \ + ./DIGServer $(port); \ + fi start_test: - cd test && ./DIGClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./DIGClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/dig/Parser.h b/lucida/djinntonic/dig/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/dig/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/dig/test/DIGClient.cpp b/lucida/djinntonic/dig/test/DIGClient.cpp index db2d133e3..1d4a40084 100644 --- a/lucida/djinntonic/dig/test/DIGClient.cpp +++ b/lucida/djinntonic/dig/test/DIGClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("DIG_PORT", portVal)) { - cout << "DIG port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/dig/test/Makefile b/lucida/djinntonic/dig/test/Makefile index 21ad08bd6..cad8de2b6 100644 --- a/lucida/djinntonic/dig/test/Makefile +++ b/lucida/djinntonic/dig/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = DIGClient diff --git a/lucida/djinntonic/face/FACEServer.cpp b/lucida/djinntonic/face/FACEServer.cpp index a65704932..564ab3430 100644 --- a/lucida/djinntonic/face/FACEServer.cpp +++ b/lucida/djinntonic/face/FACEServer.cpp @@ -5,7 +5,7 @@ #include "FACEHandler.h" #include -#include "Parser.h" +#include #include DEFINE_int32(num_of_threads, @@ -16,20 +16,16 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("FACE_PORT", portVal)) { - cout << "FACE port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -40,6 +36,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "FACE at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/face/Makefile b/lucida/djinntonic/face/Makefile index 2c29610b5..df944aa43 100644 --- a/lucida/djinntonic/face/Makefile +++ b/lucida/djinntonic/face/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./FACEServer + @if [ "$(port)" != "" ]; then \ + ./FACEServer $(port); \ + fi start_test: - cd test && ./FACEClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./FACEClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/face/Parser.h b/lucida/djinntonic/face/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/face/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/face/test/FACEClient.cpp b/lucida/djinntonic/face/test/FACEClient.cpp index d006c8662..11b8708df 100644 --- a/lucida/djinntonic/face/test/FACEClient.cpp +++ b/lucida/djinntonic/face/test/FACEClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("FACE_PORT", portVal)) { - cout << "FACE port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/face/test/Makefile b/lucida/djinntonic/face/test/Makefile index 5ac008719..6afa69c66 100644 --- a/lucida/djinntonic/face/test/Makefile +++ b/lucida/djinntonic/face/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = FACEClient diff --git a/lucida/djinntonic/imc/IMCServer.cpp b/lucida/djinntonic/imc/IMCServer.cpp index f923a4166..4a2b19c87 100644 --- a/lucida/djinntonic/imc/IMCServer.cpp +++ b/lucida/djinntonic/imc/IMCServer.cpp @@ -5,7 +5,7 @@ #include "IMCHandler.h" #include -#include "Parser.h" +#include #include DEFINE_int32(num_of_threads, @@ -16,21 +16,17 @@ using namespace apache::thrift; using namespace apache::thrift::async; using namespace cpp2; +using namespace std; //using namespace facebook::windtunnel::treadmill::services::imc; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - Properties props; - props.Read("../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMC_PORT", portVal)) { - cout << "IMC port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); @@ -41,6 +37,8 @@ int main(int argc, char* argv[]) { server->setIdleTimeout(std::chrono::milliseconds(0)); server->setTaskExpireTime(std::chrono::milliseconds(0)); + cout << "IMC at " << port << endl; + server->serve(); return 0; diff --git a/lucida/djinntonic/imc/Makefile b/lucida/djinntonic/imc/Makefile index 3ef3f4f2c..15912f992 100644 --- a/lucida/djinntonic/imc/Makefile +++ b/lucida/djinntonic/imc/Makefile @@ -30,10 +30,15 @@ $(TARGET): $(OBJECTS) $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ start_server: - ./IMCServer + @if [ "$(port)" != "" ]; then \ + ./IMCServer $(port); \ + fi start_test: - cd test && ./IMCClient && cd .. + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./IMCClient $(port); \ + fi client: cd test && make all && cd .. diff --git a/lucida/djinntonic/imc/Parser.h b/lucida/djinntonic/imc/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/djinntonic/imc/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/djinntonic/imc/test/IMCClient.cpp b/lucida/djinntonic/imc/test/IMCClient.cpp index ff1e0c03e..1b9270b3c 100644 --- a/lucida/djinntonic/imc/test/IMCClient.cpp +++ b/lucida/djinntonic/imc/test/IMCClient.cpp @@ -15,7 +15,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "../Parser.h" using namespace folly; using namespace apache::thrift; @@ -45,16 +44,11 @@ int main(int argc, char* argv[]){ folly::init(&argc, &argv); EventBase event_base; - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMC_PORT", portVal)) { - cout << "IMC port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); } + int port = atoi(argv[1]); std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); diff --git a/lucida/djinntonic/imc/test/Makefile b/lucida/djinntonic/imc/test/Makefile index 1f1008183..69942dded 100644 --- a/lucida/djinntonic/imc/test/Makefile +++ b/lucida/djinntonic/imc/test/Makefile @@ -16,12 +16,9 @@ LINKFLAGS = -lopencv_core \ -lprotobuf \ -ltesseract \ -pthread \ - -lmongoclient \ -lboost_program_options \ -lboost_filesystem \ -lboost_system \ - -lboost_thread \ - -lboost_regex \ -lthrift \ -lfolly \ -lwangle \ @@ -30,7 +27,6 @@ LINKFLAGS = -lopencv_core \ -lthriftcpp2 \ -lgflags \ -lthriftprotocol \ - -lssl \ -lcrypto TARGET = IMCClient diff --git a/lucida/djinntonic/tools/check_thrift.thrift b/lucida/djinntonic/tools/check_thrift.thrift new file mode 100644 index 000000000..1081f9e88 --- /dev/null +++ b/lucida/djinntonic/tools/check_thrift.thrift @@ -0,0 +1,3 @@ +service HelloWorld { + string hello(1:string LUCID); +} diff --git a/tools/install_fbthrift.sh b/lucida/djinntonic/tools/install_fbthrift.sh similarity index 95% rename from tools/install_fbthrift.sh rename to lucida/djinntonic/tools/install_fbthrift.sh index 5287d6992..6617141d3 100755 --- a/tools/install_fbthrift.sh +++ b/lucida/djinntonic/tools/install_fbthrift.sh @@ -1,8 +1,5 @@ #!/bin/bash installCheck () { - if [ ! -d fbthrift ]; then - return 1 - fi python -mthrift_compiler.main --gen cpp2 check_thrift.thrift if [ -d gen-cpp2 ]; then rm -rf gen-cpp2 diff --git a/lucida/imagematching/opencv_imm/Makefile b/lucida/imagematching/opencv_imm/Makefile index b8d290bb1..db88b8558 100644 --- a/lucida/imagematching/opencv_imm/Makefile +++ b/lucida/imagematching/opencv_imm/Makefile @@ -1,11 +1,17 @@ -SUBDIRS=server test +SUBDIRS=tools server test include ../../../Makefile.common -start_server: - cd server; ./imm_server +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + ./imm_server $(port); \ + fi start_test: - cd test; ./imm_test + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./imm_test $(port); \ + fi docker: cp ../../lucidaservice.thrift .; \ diff --git a/lucida/imagematching/opencv_imm/server/IMMServer.cpp b/lucida/imagematching/opencv_imm/server/IMMServer.cpp index c34fff5e5..a4f77ed1e 100644 --- a/lucida/imagematching/opencv_imm/server/IMMServer.cpp +++ b/lucida/imagematching/opencv_imm/server/IMMServer.cpp @@ -8,10 +8,10 @@ DEFINE_int32(num_of_threads, "Number of threads (default: 4)"); #include "IMMHandler.h" -#include "Parser.h" #include #include #include +#include using namespace folly; using namespace apache::thrift; @@ -28,18 +28,12 @@ using std::to_string; int main(int argc, char* argv[]) { folly::init(&argc, &argv); - - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMM_PORT", portVal)) { - cout << "IMM port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); - } + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); auto handler = std::make_shared(); auto server = folly::make_unique(); diff --git a/lucida/imagematching/opencv_imm/server/Parser.h b/lucida/imagematching/opencv_imm/server/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/imagematching/opencv_imm/server/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/imagematching/opencv_imm/test/IMMClient.cpp b/lucida/imagematching/opencv_imm/test/IMMClient.cpp index 5d1a1f17e..d137c31e3 100644 --- a/lucida/imagematching/opencv_imm/test/IMMClient.cpp +++ b/lucida/imagematching/opencv_imm/test/IMMClient.cpp @@ -17,7 +17,6 @@ #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include -#include "Parser.h" using namespace folly; using namespace apache::thrift; @@ -57,6 +56,12 @@ string getImageData(const string &image_path) { } int main(int argc, char* argv[]) { + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + // Initialize MongoDB C++ driver. mongo::client::initialize(); mongo::DBClientConnection conn; @@ -73,18 +78,6 @@ int main(int argc, char* argv[]) { folly::init(&argc, &argv); EventBase event_base; - - Properties props; - props.Read("../../../config.properties"); - string portVal; - int port; - if (!props.GetValue("IMM_PORT", portVal)) { - cout << "IMM port not defined" << endl; - return -1; - } else { - port = atoi(portVal.c_str()); - } - std::shared_ptr socket_t( TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); LucidaServiceAsyncClient client( @@ -122,9 +115,6 @@ int main(int argc, char* argv[]) { // Infer. // Make request. int num_tests = 3; - if (argc == 2) { - num_tests = atoi(argv[1]); - } for (int i = 0; i < num_tests; ++i) { string image = getImageData("test" + to_string(i) + ".jpg"); // Create a QuerySpec. diff --git a/lucida/imagematching/opencv_imm/test/Parser.h b/lucida/imagematching/opencv_imm/test/Parser.h deleted file mode 100644 index 33978a105..000000000 --- a/lucida/imagematching/opencv_imm/test/Parser.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -using namespace std; - -class Properties { - -public: - - Properties () {} - - bool Read (const string& strFile) { - ifstream is(strFile.c_str()); - if (!is.is_open()) return false; - while (!is.eof()) { - string strLine; - getline(is,strLine); - strLine.erase(remove_if(strLine.begin(), - strLine.end(), - [](char x){return isspace(x);}), - strLine.end()); - uint nPos = strLine.find('='); - if (strLine.length() == 0 || strLine[0] == '#' || - strLine[0] == '!' || string::npos == nPos) continue; - string strKey = strLine.substr(0,nPos); - string strVal = strLine.substr(nPos + 1, strLine.length() - nPos + 1); - m_map.insert(map::value_type(strKey,strVal)); - } - return true; - } - - bool GetValue(const string& strKey, string& strValue) const { - map::const_iterator i; - i = m_map.find(strKey); - if (i != m_map.end()) { - strValue = i->second; - return true; - } - return false; - } - -protected: - - map m_map; -}; \ No newline at end of file diff --git a/lucida/imagematching/opencv_imm/tools/Makefile b/lucida/imagematching/opencv_imm/tools/Makefile new file mode 100644 index 000000000..3138d5715 --- /dev/null +++ b/lucida/imagematching/opencv_imm/tools/Makefile @@ -0,0 +1,3 @@ +all: + (sudo ./install_fbthrift.sh) || (exit 1) + (sudo ./install_opencv.sh) || (exit 1) diff --git a/tools/check_opencv.cpp b/lucida/imagematching/opencv_imm/tools/check_opencv.cpp similarity index 100% rename from tools/check_opencv.cpp rename to lucida/imagematching/opencv_imm/tools/check_opencv.cpp diff --git a/lucida/imagematching/opencv_imm/tools/check_thrift.thrift b/lucida/imagematching/opencv_imm/tools/check_thrift.thrift new file mode 100644 index 000000000..1081f9e88 --- /dev/null +++ b/lucida/imagematching/opencv_imm/tools/check_thrift.thrift @@ -0,0 +1,3 @@ +service HelloWorld { + string hello(1:string LUCID); +} diff --git a/lucida/imagematching/opencv_imm/tools/install_fbthrift.sh b/lucida/imagematching/opencv_imm/tools/install_fbthrift.sh new file mode 100755 index 000000000..6617141d3 --- /dev/null +++ b/lucida/imagematching/opencv_imm/tools/install_fbthrift.sh @@ -0,0 +1,42 @@ +#!/bin/bash +installCheck () { + python -mthrift_compiler.main --gen cpp2 check_thrift.thrift + if [ -d gen-cpp2 ]; then + rm -rf gen-cpp2 + return 0 + else + return 1 + fi +} + +if installCheck "$0"; then + echo "Facebook Thrift installed"; + exit 0; +fi + +if [ ! -d fbthrift ]; then + git clone https://github.com/facebook/fbthrift.git + if [ $? -ne 0 ]; then + echo "Could not clone FBThrift!!! Please try again later..." + exit 1 + fi +fi + +cd fbthrift/thrift \ + && git checkout v2017.03.20.00 \ + && echo "a1abbb7abcb259acbd94d0d0929b79607a8ce806" > ./build/FOLLY_VERSION \ + && echo "a5503c88e1d6799dcfb337caf09834a877790c92" > ./build/WANGLE_VERSION \ + && ./build/deps_ubuntu_14.04.sh \ + && autoreconf -ivf \ + && ./configure \ + && make \ + && make install \ + && cd ../.. + +if installCheck "$0"; then + echo "Facebook Thrift installed"; + exit 0; +else + echo "Failed to install Facebook Thrift"; + exit 1; +fi diff --git a/tools/install_opencv.sh b/lucida/imagematching/opencv_imm/tools/install_opencv.sh similarity index 98% rename from tools/install_opencv.sh rename to lucida/imagematching/opencv_imm/tools/install_opencv.sh index 2f4d66c74..8c82c0856 100755 --- a/tools/install_opencv.sh +++ b/lucida/imagematching/opencv_imm/tools/install_opencv.sh @@ -1,4 +1,5 @@ #!/bin/bash +THREADS=1 if [ -z $THREADS ]; then THREADS=`nproc` fi diff --git a/lucida/musicservice/Makefile b/lucida/musicservice/Makefile index 8462d33d8..fa5bbd0f6 100644 --- a/lucida/musicservice/Makefile +++ b/lucida/musicservice/Makefile @@ -15,7 +15,13 @@ clean: rm -rf lucidaservice lucidatypes start_server: - cd server; python server.py + @if [ "$(port)" != "" ]; then \ + cd server; \ + python server.py $(port); \ + fi -start_test: - cd client; python client.py \ No newline at end of file +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python client.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/musicservice/MusicConfig.py b/lucida/musicservice/MusicConfig.py index e1b12cf76..dea17a90e 100644 --- a/lucida/musicservice/MusicConfig.py +++ b/lucida/musicservice/MusicConfig.py @@ -3,11 +3,7 @@ """ from pygn import Pygn -from helper import port_dic - -# Service port number -PORT = int(port_dic['ms_port']) # Music API username and password -clientID = # TODO: Enter your Client ID from developer.gracenote.com here +clientID = '173703779-EA19716F8D2A73DF7ECAF522D5050BF3' # TODO: Enter your Client ID from developer.gracenote.com here userID = Pygn.register(clientID) # Get a User ID from pygn.register() - Only register once per end-user diff --git a/lucida/musicservice/client/client.py b/lucida/musicservice/client/client.py index 637e5706a..7a463e74a 100644 --- a/lucida/musicservice/client/client.py +++ b/lucida/musicservice/client/client.py @@ -3,7 +3,6 @@ import sys sys.path.append('../') -from MusicConfig import PORT from lucidatypes.ttypes import QueryInput, QuerySpec from lucidaservice import LucidaService @@ -12,6 +11,11 @@ from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + # Setup a template input query LUCID = "Clinc" query_input_data = "I want a happy song." diff --git a/lucida/musicservice/helper.py b/lucida/musicservice/helper.py index d039262c6..2de30c997 100644 --- a/lucida/musicservice/helper.py +++ b/lucida/musicservice/helper.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import ConfigParser, sys, re +import re # Mood invert index mood_dic = { @@ -85,20 +85,3 @@ def keyword_scan(question): else: return output[0] -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../../config.properties"))) -port_dic = dict(cp.items('asection')) \ No newline at end of file diff --git a/lucida/musicservice/server/server.py b/lucida/musicservice/server/server.py index 66f746f28..27e7c4ceb 100644 --- a/lucida/musicservice/server/server.py +++ b/lucida/musicservice/server/server.py @@ -20,6 +20,11 @@ import json import re +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + class MusicHandler: def create(self, LUCID, spec): ''' diff --git a/lucida/questionanswering/OpenEphyra/Makefile b/lucida/questionanswering/OpenEphyra/Makefile index 1f3d30d7a..6790eeb3d 100644 --- a/lucida/questionanswering/OpenEphyra/Makefile +++ b/lucida/questionanswering/OpenEphyra/Makefile @@ -14,10 +14,16 @@ thrift: fi start_server: - chmod +x start_server.sh && ./start_server.sh + @if [ "$(port)" != "" ]; then \ + chmod +x start_server.sh; \ + ./start_server.sh $(port); \ + fi start_test: - chmod +x start_test.sh && ./start_test.sh + @if [ "$(port)" != "" ]; then \ + chmod +x start_test.sh; \ + ./start_test.sh $(port); \ + fi clean: rm -rf bin diff --git a/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java b/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java index 21681525a..49c0dabb7 100644 --- a/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java +++ b/lucida/questionanswering/OpenEphyra/src/lucida/main/QADaemon.java @@ -11,10 +11,7 @@ import org.apache.thrift.transport.TTransportException; import java.io.IOException; -import java.io.FileInputStream; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Properties; import org.apache.thrift.TException; import org.apache.thrift.TProcessor; @@ -40,11 +37,12 @@ public class QADaemon { */ public static void main(String [] args) throws TTransportException, IOException, InterruptedException { - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("QA_PORT"); - Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + TProcessor proc = new LucidaService.AsyncProcessor( new QAServiceHandler.AsyncQAServiceHandler()); TNonblockingServerTransport transport = new TNonblockingServerSocket(port); @@ -53,7 +51,7 @@ public static void main(String [] args) .protocolFactory(new TBinaryProtocol.Factory()) .transportFactory(new TFramedTransport.Factory()); final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); - System.out.println("QA at port " + port_str); + System.out.println("QA at port " + port); server.serve(); } } diff --git a/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java b/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java index a47fe5e09..028ad2824 100644 --- a/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java +++ b/lucida/questionanswering/OpenEphyra/src/lucida/test/QAClient.java @@ -2,9 +2,6 @@ //Java packages import java.io.IOException; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import java.util.List; import java.util.ArrayList; @@ -55,12 +52,11 @@ private static final QuerySpec createQuerySpec( public static void main(String [] args) throws IOException { - // Collect the port number. - Properties port_cfg = new Properties(); - InputStream input = new FileInputStream("../../config.properties"); - port_cfg.load(input); - String port_str = port_cfg.getProperty("QA_PORT"); - final Integer port = Integer.valueOf(port_str); + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + final Integer port = Integer.valueOf(args[0]); // User. String LUCID = "Clinc"; diff --git a/lucida/questionanswering/OpenEphyra/start_server.sh b/lucida/questionanswering/OpenEphyra/start_server.sh index a3d45e624..d8f3cc21b 100755 --- a/lucida/questionanswering/OpenEphyra/start_server.sh +++ b/lucida/questionanswering/OpenEphyra/start_server.sh @@ -1,2 +1,2 @@ #!/bin/bash -java -Djava.library.path=lib/search/ -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.main.QADaemon +java -Djava.library.path=lib/search/ -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.main.QADaemon $1 diff --git a/lucida/questionanswering/OpenEphyra/start_test.sh b/lucida/questionanswering/OpenEphyra/start_test.sh index 48654f3ba..6f1aa5a69 100755 --- a/lucida/questionanswering/OpenEphyra/start_test.sh +++ b/lucida/questionanswering/OpenEphyra/start_test.sh @@ -1,2 +1,2 @@ #!/bin/bash -java -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.test.QAClient +java -classpath bin:lib/learn/jsoup-1.8.3.jar:lib/ml/maxent.jar:lib/ml/minorthird.jar:lib/nlp/jwnl.jar:lib/nlp/lingpipe.jar:lib/nlp/opennlp-tools.jar:lib/nlp/plingstemmer.jar:lib/nlp/snowball.jar:lib/nlp/stanford-ner.jar:lib/nlp/stanford-parser.jar:lib/nlp/stanford-postagger.jar:lib/qa/javelin.jar:lib/search/bing-search-java-sdk.jar:lib/search/googleapi.jar:lib/search/indri.jar:lib/search/yahoosearch.jar:lib/thrift/libthrift-0.9.2.jar:lib/thrift/log4j-1.2.14.jar:lib/thrift/slf4j-api-1.5.8.jar:lib/thrift/slf4j-log4j12-1.5.8.jar:lib/util/commons-logging.jar:lib/util/gson.jar:lib/util/htmlparser.jar:lib/util/jetty-all.jar:lib/util/log4j.jar:lib/util/servlet-api.jar:lib/util/trove.jar:lib/db/mongo-java-driver-3.2.2.jar lucida.test.QAClient $1 diff --git a/lucida/template/.gitignore b/lucida/template/.gitignore new file mode 100644 index 000000000..770c66e2b --- /dev/null +++ b/lucida/template/.gitignore @@ -0,0 +1,2 @@ +lucidaservice.thrift +lucidatypes.thrift diff --git a/lucida/template/README.md b/lucida/template/README.md new file mode 100644 index 000000000..635bd10a3 --- /dev/null +++ b/lucida/template/README.md @@ -0,0 +1,14 @@ +# Template for building your own microservice + +This is a template for a user to refer to when creating their own microservice for Lucida. It includes 3 versions of microservice implementation (c++, Java, python). You can choose your favorite language and implement your own service based on the template. We use both Apache Thrift and Facebook Thrift as our Lucida RPC framework. Thrift is an RPC framework with the advantages of being efficient and language-neutral. It was originally developed by Facebook and now developed by both the open-source community (Apache Thrift) and Facebook. + +## Major Dependencies + +- [Facebook Thrift](https://github.com/facebook/fbthrift) +- [gradle](https://gradle.org/) + +# Structure + +- [`cpp`](cpp): implementation of C++ template +- [`java`](java): implementation for Java template +- [`python`](python): implementation for Python template \ No newline at end of file diff --git a/lucida/template/cpp/.gitignore b/lucida/template/cpp/.gitignore new file mode 100644 index 000000000..525b08a05 --- /dev/null +++ b/lucida/template/cpp/.gitignore @@ -0,0 +1,6 @@ +server/gen-cpp2 +server/template_server +test/gen-cpp2 +test/template_test +server/*.o +test/*.o diff --git a/lucida/template/cpp/Makefile b/lucida/template/cpp/Makefile new file mode 100644 index 000000000..9fc5799e6 --- /dev/null +++ b/lucida/template/cpp/Makefile @@ -0,0 +1,28 @@ +all: + cd server ; \ + make all ; \ + cd ../test; \ + make all ; \ + cd ..; + +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + ./template_server $(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd test; \ + ./template_test $(port); \ + fi + +clean: + cd server ; \ + make clean ; \ + cd ../test; \ + make clean ; \ + cd ..; + +%: + @: diff --git a/lucida/template/cpp/README.md b/lucida/template/cpp/README.md new file mode 100644 index 000000000..d620b913a --- /dev/null +++ b/lucida/template/cpp/README.md @@ -0,0 +1,72 @@ +# template microservice in C++ + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- [Facebook Thrift](https://github.com/facebook/fbthrift) + +# Structure + +- `server/`: implementation of the template server +- `test/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`server/templateHandler.cpp`](server/templateHandler.cpp). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` + +Update the [`Makefile`](Makefile). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](test/templateClient.cpp) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. + diff --git a/lucida/template/cpp/server/Makefile b/lucida/template/cpp/server/Makefile new file mode 100644 index 000000000..237ed47bc --- /dev/null +++ b/lucida/template/cpp/server/Makefile @@ -0,0 +1,57 @@ +CXX = g++ + +CXXFLAGS = -std=c++11 \ + -fPIC +CXXFLAGS += $(shell if [ `lsb_release -a 2>/dev/null | grep -Poe "(?<=\s)\d+(?=[\d\.]+$$)"` -gt 14 ]; then echo "-std=c++14"; fi;) + +LINKFLAGS = -lrt \ + -lprotobuf \ + -ltesseract \ + -pthread \ + -lboost_program_options \ + -lboost_filesystem \ + -lboost_system \ + -lthrift \ + -lfolly \ + -lwangle \ + -lzstd \ + -lglog \ + -lthriftcpp2 \ + -lgflags \ + -lthriftprotocol \ + -lcrypto + +TARGET = template_server +SOURCES = gen-cpp2/LucidaService_client.cpp \ + gen-cpp2/lucidaservice_constants.cpp \ + gen-cpp2/LucidaService.cpp \ + gen-cpp2/LucidaService_processmap_binary.cpp \ + gen-cpp2/LucidaService_processmap_compact.cpp \ + gen-cpp2/lucidaservice_types.cpp \ + gen-cpp2/lucidatypes_constants.cpp \ + gen-cpp2/lucidatypes_types.cpp \ + $(wildcard *.cpp) +OBJECTS = $(SOURCES:.cpp=.o) + +all: CXXFLAGS += -O3 +all: thrift $(TARGET) + +debug: CXXFLAGS += -g3 +debug: thrift $(TARGET) + +thrift: + @if [ ! -d "gen-cpp2" ]; then \ + python -mthrift_compiler.main --gen cpp2 ../../lucidaservice.thrift; \ + python -mthrift_compiler.main --gen cpp2 ../../lucidatypes.thrift; \ + fi + +$(TARGET): $(OBJECTS) + $(CXX) $(OBJECTS) $(LINKFLAGS) -o $@ + +%.o: %.cpp + $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ + +clean: + rm -rf $(TARGET) *.o gen-cpp2 + +.PHONY: all debug thrift clean \ No newline at end of file diff --git a/lucida/template/cpp/server/templateHandler.cpp b/lucida/template/cpp/server/templateHandler.cpp new file mode 100644 index 000000000..eb1071fd5 --- /dev/null +++ b/lucida/template/cpp/server/templateHandler.cpp @@ -0,0 +1,63 @@ +#include "templateHandler.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; + +using std::cout; +using std::endl; +using std::string; +using std::unique_ptr; +using std::shared_ptr; +using std::getenv; + +// cout lock. +std::mutex cout_lock_cpp; + +namespace cpp2 { +templateHandler::templateHandler() { + // TODO: adding your own initializatin of the handler +} + +// TODO: implement your own future_create function. +folly::Future templateHandler::future_create +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> spec) { + folly::MoveWrapper > promise; + auto future = promise->getFuture(); + promise->setValue(Unit{}); + return future; +} + +// TODO: implement your own future_learn function. +folly::Future templateHandler::future_learn +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> knowledge) { + folly::MoveWrapper > promise; + auto future = promise->getFuture(); + promise->setValue(Unit{}); + return future; +} + +// TODO: implement your own future_infer function. +folly::Future> templateHandler::future_infer +(unique_ptr LUCID, unique_ptr< ::cpp2::QuerySpec> query) { + print("@@@@@ Infer; User: " + *(LUCID.get())); + folly::MoveWrapper > > promise; + auto future = promise->getFuture(); + promise->setValue(unique_ptr(new string("This is the sample answer"))); + return future; +} + +// TODO: define the methods needed for your service. +} diff --git a/lucida/template/cpp/server/templateHandler.h b/lucida/template/cpp/server/templateHandler.h new file mode 100644 index 000000000..c50746c8e --- /dev/null +++ b/lucida/template/cpp/server/templateHandler.h @@ -0,0 +1,41 @@ +#ifndef __TEMPLATEHANDLER_H__ +#define __TEMPLATEHANDLER_H__ + +#include +#include + +#include "gen-cpp2/LucidaService.h" + +// Define print for simple logging. +extern std::mutex cout_lock_cpp; +#define print( x ) \ + ( \ + (cout_lock_cpp.lock()), \ + (std::cout << x << endl), \ + (cout_lock_cpp.unlock()), \ + (void)0 \ + ) + +namespace cpp2 { +class templateHandler : virtual public LucidaServiceSvIf { +public: + templateHandler(); + + folly::Future future_create + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> spec); + + folly::Future future_learn + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> knowledge); + + folly::Future> future_infer + (std::unique_ptr LUCID, + std::unique_ptr< ::cpp2::QuerySpec> query); + +private: + // TODO: Define your own private methods +}; +} + +#endif diff --git a/lucida/template/cpp/server/templateServer.cpp b/lucida/template/cpp/server/templateServer.cpp new file mode 100644 index 000000000..0e9950867 --- /dev/null +++ b/lucida/template/cpp/server/templateServer.cpp @@ -0,0 +1,48 @@ +#include + +#include +#include + +DEFINE_int32(num_of_threads, + 4, + "Number of threads (default: 4)"); + +#include "templateHandler.h" +#include +#include + +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; +using namespace cpp2; +using namespace std; + +using std::cout; +using std::endl; +using std::shared_ptr; +using std::unique_ptr; +using std::to_string; + +int main(int argc, char* argv[]) { + folly::init(&argc, &argv); + + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + + auto handler = std::make_shared(); + auto server = folly::make_unique(); + + server->setPort(port); + server->setNWorkerThreads(FLAGS_num_of_threads); + server->setInterface(std::move(handler)); + server->setIdleTimeout(std::chrono::milliseconds(0)); + server->setTaskExpireTime(std::chrono::milliseconds(0)); + + cout << "TPL at port " << to_string(port) << endl; + server->serve(); + + return 0; +} diff --git a/lucida/template/cpp/test/Makefile b/lucida/template/cpp/test/Makefile new file mode 100644 index 000000000..d97d910ea --- /dev/null +++ b/lucida/template/cpp/test/Makefile @@ -0,0 +1,57 @@ +CXX = g++ + +CXXFLAGS = -std=c++11 \ + -fPIC +CXXFLAGS += $(shell if [ `lsb_release -a 2>/dev/null | grep -Poe "(?<=\s)\d+(?=[\d\.]+$$)"` -gt 14 ]; then echo "-std=c++14"; fi;) + +LINKFLAGS = -lrt \ + -lprotobuf \ + -ltesseract \ + -pthread \ + -lboost_program_options \ + -lboost_filesystem \ + -lboost_system \ + -lthrift \ + -lfolly \ + -lwangle \ + -lzstd \ + -lglog \ + -lthriftcpp2 \ + -lgflags \ + -lthriftprotocol \ + -lcrypto + +TARGET = template_test +SOURCES = gen-cpp2/LucidaService_client.cpp \ + gen-cpp2/lucidaservice_constants.cpp \ + gen-cpp2/LucidaService.cpp \ + gen-cpp2/LucidaService_processmap_binary.cpp \ + gen-cpp2/LucidaService_processmap_compact.cpp \ + gen-cpp2/lucidaservice_types.cpp \ + gen-cpp2/lucidatypes_constants.cpp \ + gen-cpp2/lucidatypes_types.cpp \ + $(wildcard *.cpp) +OBJECTS = $(SOURCES:.cpp=.o) + +all: CXXFLAGS += -O3 +all: thrift $(TARGET) + +debug: CXXFLAGS += -g3 +debug: thrift $(TARGET) + +thrift: + @if [ ! -d "gen-cpp2" ]; then \ + python -mthrift_compiler.main --gen cpp2 ../../lucidaservice.thrift; \ + python -mthrift_compiler.main --gen cpp2 ../../lucidatypes.thrift; \ + fi + +$(TARGET): $(OBJECTS) + $(CXX) $(OBJECTS) $(LINKFLAGS) -o $@ + +%.o: %.cpp + $(CXX) -Wall $(CXXFLAGS) -c $< -o $@ + +clean: + rm -rf $(TARGET) *.o gen-cpp2 + +.PHONY: all debug thrift clean diff --git a/lucida/template/cpp/test/templateClient.cpp b/lucida/template/cpp/test/templateClient.cpp new file mode 100644 index 000000000..2a42b8a76 --- /dev/null +++ b/lucida/template/cpp/test/templateClient.cpp @@ -0,0 +1,61 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "gen-cpp2/LucidaService.h" +#include +#include + +using namespace folly; +using namespace apache::thrift; +using namespace apache::thrift::async; +using namespace cpp2; +using namespace std; + +DEFINE_string(hostname, + "127.0.0.1", + "Hostname of the server (default: localhost)"); + +int main(int argc, char* argv[]) { + if (argc != 2){ + cerr << "Wrong argument!" << endl; + exit(EXIT_FAILURE); + } + int port = atoi(argv[1]); + + folly::init(&argc, &argv); + EventBase event_base; + std::shared_ptr socket_t( + TAsyncSocket::newSocket(&event_base, FLAGS_hostname, port)); + LucidaServiceAsyncClient client( + std::unique_ptr( + new HeaderClientChannel(socket_t))); + + // Infer. + QuerySpec query_spec; + // Create a QueryInput for the query image and add it to the QuerySpec. + // TODO: Adding your own sample query + QueryInput query_input; + query_input.type = "query"; + query_input.data.push_back("What is the sample query?"); + query_input.tags.push_back("localhost"); + query_input.tags.push_back(to_string(port)); + query_input.tags.push_back("0"); + query_spec.content.push_back(query_input); + cout << "///// Connecting to template... /////" << endl; + auto result = client.future_infer("Clinc", std::move(query_spec)).then( + [=](folly::Try&& t) mutable { + cout << "///// Result: /////\n" << t.value() << endl; + return t.value(); + }); + event_base.loop(); + return 0; +} diff --git a/lucida/template/java/.gitignore b/lucida/template/java/.gitignore new file mode 100644 index 000000000..17ed65221 --- /dev/null +++ b/lucida/template/java/.gitignore @@ -0,0 +1,5 @@ +.gradle +build +templateClient/thrift +src/main/java/thrift +templateClient/*.class diff --git a/lucida/template/java/Makefile b/lucida/template/java/Makefile new file mode 100644 index 000000000..a300c5f8d --- /dev/null +++ b/lucida/template/java/Makefile @@ -0,0 +1,39 @@ +all: thrift client + ./gradlew build + +thrift: + @if [ ! -d "src/main/java/thrift" ]; then \ + thrift --gen java ../lucidaservice.thrift; \ + thrift --gen java ../lucidatypes.thrift; \ + sed -i '1s/^/package thrift;\n/' gen-java/LucidaService.java; \ + sed -i '1s/^/package thrift;\n/' gen-java/QueryInput.java; \ + sed -i '1s/^/package thrift;\n/' gen-java/QuerySpec.java; \ + mkdir src/main/java/thrift/; \ + mv gen-java/* src/main/java/thrift/; \ + rmdir gen-java/; \ + fi + +client: + cd templateClient/ && ./compile-template-client.sh + +start_server: + @if [ "$(port)" != "" ]; then \ + ./gradlew run -Pargs=$(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd templateClient; \ + ./start-template-client.sh $(port); \ + fi + +clean: + rm -rf src/main/java/thrift ; \ + rm -rf templateClient/thrift ; \ + rm -rf templateClient/*.class ; \ + rm -rf build; + +.PHONY: all thrift gradle start_server start_test clean + +%: + @: diff --git a/lucida/template/java/README.md b/lucida/template/java/README.md new file mode 100644 index 000000000..99b6f3be0 --- /dev/null +++ b/lucida/template/java/README.md @@ -0,0 +1,71 @@ +# template microservice in Java + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- [gradle](https://gradle.org/) + +# Structure + +- `src/main/java/template/`: implementation of the template server +- `templateClient/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`src/main/java/template/TEServiceHandler.java`](src/main/java/template/TEServiceHandler.java). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` and `build.gradle` + +Update the [`Makefile`](Makefile) and [`build.gradle`](build.gradle). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](templateClient/templateClient.java) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. diff --git a/lucida/template/java/build.gradle b/lucida/template/java/build.gradle new file mode 100644 index 000000000..e410230f1 --- /dev/null +++ b/lucida/template/java/build.gradle @@ -0,0 +1,25 @@ +apply plugin: 'java' +apply plugin: 'application' + +mainClassName = 'template.templateDaemon' +sourceCompatibility = 1.7 +targetCompatibility = 1.7 +version = '1.0' + +run { + if( project.hasProperty('args')) { + args project.args.split('\\s') + } +} + +repositories { + mavenCentral() + flatDir { + dirs 'lib' + } +} + +// TODO: Add the dependencies your service needs +dependencies { + compile 'org.apache.thrift:libthrift:0.9.3' +} \ No newline at end of file diff --git a/lucida/template/java/gradle/wrapper/gradle-wrapper.jar b/lucida/template/java/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..5ccda13e9 Binary files /dev/null and b/lucida/template/java/gradle/wrapper/gradle-wrapper.jar differ diff --git a/lucida/template/java/gradle/wrapper/gradle-wrapper.properties b/lucida/template/java/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..c78864717 --- /dev/null +++ b/lucida/template/java/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Mar 30 20:24:24 EDT 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip diff --git a/lucida/template/java/gradlew b/lucida/template/java/gradlew new file mode 100755 index 000000000..9d82f7891 --- /dev/null +++ b/lucida/template/java/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/lucida/template/java/gradlew.bat b/lucida/template/java/gradlew.bat new file mode 100644 index 000000000..72d362daf --- /dev/null +++ b/lucida/template/java/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/lucida/template/java/settings.gradle b/lucida/template/java/settings.gradle new file mode 100644 index 000000000..18204127f --- /dev/null +++ b/lucida/template/java/settings.gradle @@ -0,0 +1,19 @@ +/* + * This settings file was auto generated by the Gradle buildInit task + * by 'yba' at '3/11/16 9:55 PM' with Gradle 2.11 + * + * The settings file is used to specify which projects to include in your build. + * In a single project build this file can be empty or even removed. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user guide at https://docs.gradle.org/2.11/userguide/multi_project_builds.html + */ + +/* +// To declare projects as part of a multi-project build use the 'include' method +include 'shared' +include 'api' +include 'services:webservice' +*/ + +rootProject.name = 'template' diff --git a/lucida/template/java/src/main/java/template/TPLServiceHandler.java b/lucida/template/java/src/main/java/template/TPLServiceHandler.java new file mode 100644 index 000000000..b445be07a --- /dev/null +++ b/lucida/template/java/src/main/java/template/TPLServiceHandler.java @@ -0,0 +1,87 @@ +package template; + +import java.util.List; +import java.io.File; +import java.util.ArrayList; + +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; + +import thrift.*; + +/** + * Implementation of the template interface. A client request to any + * method defined in the thrift file is handled by the + * corresponding method here. + */ +public class TPLServiceHandler { + public static void print(String s) { + synchronized (System.out) { + System.out.println(s); + } + } + + public static class SyncTPLServiceHandler implements LucidaService.Iface { + /** + * TODO: Implement your own create function for your service. + * @param LUCID ID of Lucida user + * @param spec spec + */ + @Override + public void create(String LUCID, QuerySpec spec) {} + + /** + * TODO: Implement your own learn function for your service. + * @param LUCID ID of Lucida user + * @param knowledge knowledge + */ + @Override + public void learn(String LUCID, QuerySpec knowledge) {} + + /** + * TODO: Implement your own infer function for your service. + * Here is a sample infer function. + * @param LUCID ID of Lucida user + * @param query query + */ + @Override + public String infer(String LUCID, QuerySpec query) { + print("@@@@@ Infer; User: " + LUCID); + if (query.content.isEmpty() || query.content.get(0).data.isEmpty()) { + throw new IllegalArgumentException(); + } + String query_data = query.content.get(0).data.get(0); + print("Asking: " + query_data); + String answer_data = "This is the sample answer"; + print("Result: " + answer_data); + return answer_data; + } + } + + public static class AsyncTPLServiceHandler implements LucidaService.AsyncIface { + private SyncTPLServiceHandler handler; + + public AsyncTPLServiceHandler() { + handler = new SyncTPLServiceHandler(); + } + + @Override + public void create(String LUCID, QuerySpec spec, AsyncMethodCallback resultHandler) + throws TException { + print("Async Create"); + } + + @Override + public void learn(String LUCID, QuerySpec knowledge, AsyncMethodCallback resultHandler) + throws TException { + print("Async Learn"); + } + + @Override + public void infer(String LUCID, QuerySpec query, AsyncMethodCallback resultHandler) + throws TException { + print("Async Infer"); + resultHandler.onComplete(handler.infer(LUCID, query)); + } + } +} diff --git a/lucida/template/java/src/main/java/template/templateDaemon.java b/lucida/template/java/src/main/java/template/templateDaemon.java new file mode 100644 index 000000000..99bc48165 --- /dev/null +++ b/lucida/template/java/src/main/java/template/templateDaemon.java @@ -0,0 +1,52 @@ +package template; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.ArrayList; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.server.TNonblockingServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadedSelectorServer; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.async.AsyncMethodCallback; + +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +import thrift.*; + +/** + * Starts the template server and listens for requests. + */ +public class templateDaemon { + public static void main(String [] args) + throws TTransportException, IOException, InterruptedException { + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + + TProcessor proc = new LucidaService.AsyncProcessor( + new TPLServiceHandler.AsyncTPLServiceHandler()); + TNonblockingServerTransport transport = new TNonblockingServerSocket(port); + TThreadedSelectorServer.Args arguments = new TThreadedSelectorServer.Args(transport) + .processor(proc) + .protocolFactory(new TBinaryProtocol.Factory()) + .transportFactory(new TFramedTransport.Factory()); + final TThreadedSelectorServer server = new TThreadedSelectorServer(arguments); + // TODO: Change XXX into your service's acronym + System.out.println("TPL at port " + port_str); + server.serve(); + } +} diff --git a/lucida/template/java/templateClient/compile-template-client.sh b/lucida/template/java/templateClient/compile-template-client.sh new file mode 100755 index 000000000..892236693 --- /dev/null +++ b/lucida/template/java/templateClient/compile-template-client.sh @@ -0,0 +1,25 @@ +#!/bin/bash +printdivision() +{ + echo -e "\n" + for i in $(seq 1 70); do + echo -n "/"; + done + echo -e "\n" +} + +# Generate thrift files +echo -e "./compile-template-client.sh: $(pwd)" +echo -e "./compile-template-client.sh: Compiling thrift source code..." +thrift --gen java ../../lucidaservice.thrift +thrift --gen java ../../lucidatypes.thrift +mv gen-java thrift +printdivision + +# Add jar files to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar + +# Use cp flag to avoid cluttering up the CLASSPATH environment variable +echo -e "javac -cp $JAVA_CLASS_PATH templateClient.java thrift/LucidaService.java thrift/QueryInput.java thrift/QuerySpec.java +\n\n" +javac -cp "$JAVA_CLASS_PATH" templateClient.java thrift/LucidaService.java thrift/QueryInput.java thrift/QuerySpec.java diff --git a/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar b/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar new file mode 100644 index 000000000..f9221a9f9 Binary files /dev/null and b/lucida/template/java/templateClient/lib/libthrift-0.9.3.jar differ diff --git a/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar b/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar new file mode 100644 index 000000000..4dfaaa8ce Binary files /dev/null and b/lucida/template/java/templateClient/lib/slf4j-api-1.7.13.jar differ diff --git a/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar b/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar new file mode 100644 index 000000000..8c3b624ff Binary files /dev/null and b/lucida/template/java/templateClient/lib/slf4j-simple-1.7.13.jar differ diff --git a/lucida/template/java/templateClient/start-template-client.sh b/lucida/template/java/templateClient/start-template-client.sh new file mode 100755 index 000000000..8d5379ca7 --- /dev/null +++ b/lucida/template/java/templateClient/start-template-client.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Add Thrift classes to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:thrift +# Add other jar files to class path +export JAVA_CLASS_PATH=$JAVA_CLASS_PATH:lib/libthrift-0.9.3.jar:lib/slf4j-api-1.7.13.jar:lib/slf4j-simple-1.7.13.jar + +# run the server +java -cp "$JAVA_CLASS_PATH" templateClient "$1" diff --git a/lucida/template/java/templateClient/templateClient.java b/lucida/template/java/templateClient/templateClient.java new file mode 100755 index 000000000..5130c7936 --- /dev/null +++ b/lucida/template/java/templateClient/templateClient.java @@ -0,0 +1,59 @@ +//Java packages +import java.io.IOException; +import java.util.ArrayList; + +//Thrift java libraries +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; + +//Generated code +import thrift.*; + +/** +* A template Client that get the upcoming events from template Server and prints the results. +*/ +public class templateClient { + public static void main(String [] args) + throws IOException{ + if (args.length != 1){ + System.out.println("Wrong arguments!"); + System.exit(1); + } + Integer port = Integer.valueOf(args[0]); + + // Query. + // TODO: Adding your own sample query + String LUCID = "Clinc"; + String query_input_data = "What is the sample query?"; + QueryInput query_input = new QueryInput(); + query_input.type = "query"; + query_input.data = new ArrayList(); + query_input.data.add(query_input_data); + QuerySpec query_spec = new QuerySpec(); + query_spec.content = new ArrayList(); + query_spec.content.add(query_input); + + // Initialize thrift objects. + TTransport transport = new TSocket("localhost", port); + TProtocol protocol = new TBinaryProtocol(new TFramedTransport(transport)); + LucidaService.Client client = new LucidaService.Client(protocol); + try { + // Talk to the template server. + transport.open(); + System.out.println(query_input_data); + System.out.println("///// Connecting to template... /////"); + String results = client.infer(LUCID, query_spec); + System.out.println("///// Result: /////"); + System.out.println(results); + transport.close(); + } catch (TException e) { + e.printStackTrace(); + } + + return; + } +} diff --git a/lucida/template/python/.gitignore b/lucida/template/python/.gitignore new file mode 100644 index 000000000..eb761dce6 --- /dev/null +++ b/lucida/template/python/.gitignore @@ -0,0 +1,3 @@ +lucidaservice +lucidatypes +templateConfig.pyc diff --git a/lucida/template/python/Makefile b/lucida/template/python/Makefile new file mode 100644 index 000000000..06c77de2c --- /dev/null +++ b/lucida/template/python/Makefile @@ -0,0 +1,28 @@ +all: thrift + +thrift: + @if [ ! -d "lucidaservice" ]; then \ + thrift --gen py ../lucidaservice.thrift; \ + thrift --gen py ../lucidatypes.thrift; \ + cd gen-py; \ + mv * ..; \ + cd ..; \ + rmdir gen-py; \ + rm __init__.py; \ + fi + +clean: + rm -rf lucidaservice lucidatypes ; \ + rm -rf *.pyc; + +start_server: + @if [ "$(port)" != "" ]; then \ + cd server; \ + python server.py $(port); \ + fi + +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python client.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/template/python/README.md b/lucida/template/python/README.md new file mode 100644 index 000000000..4e6dcffea --- /dev/null +++ b/lucida/template/python/README.md @@ -0,0 +1,71 @@ +# template microservice in Python + +This is a guide of microservice built in lucida. To build your own service, follow the steps below. + +## Major Dependencies + +- None + +# Structure + +- `server/`: implementation of the template server +- `client/`: implementation of the template testing client + +### Step 0: design your service workflow + +To get started, first design your service workflow. You can create the workflow that includes the services already in Lucida. After designing your workflow, modify the configuration file [`lucida/commandcenter/controller/Config.py`](../../commandcenter/controllers/Config.py). See the top-level [`README.md`](../../../README.md) for details. + +### Step 1: move the directory + +Place the directory under [`lucida/lucida`](../../) folder, and change the name of your directory into a specified name represent your service. + +### Step 2: change the configuration + +Add the port information for your service in [`config.properties`](../../config.properties) with this format. +``` +_PORT= +``` + +### Step 3: implement your own create/learn/infer methods + +Implement your own create/learn/infer methods in [`server/server.py`](server/server.py). The spec of these three function is in the top-level [`README.md`](../../../README.md). Your are free to import different packages for your service, but remember to add the dependencies correctly. + +### Step 4: update the `Makefile` + +Update the [`Makefile`](Makefile). The default one has included the generating Thrift stubs code. You only need to add the dependencies of your own service. + +### Step 5: test your service individually + +Change the [test application](client/client.py) to fit with your service. Remember to change the test query to make sure your service really works. After that, do the following steps under this directory to test your service. + +- build + + ``` + make all + ``` + +- start server + + ``` + make start_server + ``` + +- start testing + + ``` + make start_test + ``` + +### Step 6: insert your service into Lucida + +Once the test of your service passes, you can insert it into Lucida. Modify the [`tools/start_all_tmux.sh`](../../../tools/start_all_tmux.sh) and [`lucida/Makefile`](../../Makefile) so that `make local` and `make start_all` include your service. + +### Step 7: add training data for your own query class + +Define a custom type of query that your service can handle and create the following file in the [`lucida/commandcenter/data/`](../../commandcenter/data/) directory: + +``` +class_.txt +``` + +, and have at least 40 pieces of text in it, each being one way to ask about the same question. diff --git a/lucida/template/python/client/client.py b/lucida/template/python/client/client.py new file mode 100644 index 000000000..9dfad229a --- /dev/null +++ b/lucida/template/python/client/client.py @@ -0,0 +1,33 @@ +import sys +sys.path.append('../') + +from lucidatypes.ttypes import QueryInput, QuerySpec +from lucidaservice import LucidaService + +from thrift import Thrift +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol + +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + +# TODO: Adding your own sample query +LUCID = "Clinc" +query_input_data = "What is the sample query?" +query_input = QueryInput(type="query", data=[query_input_data]) +query_spec = QuerySpec(content=[query_input]) + +# Initialize thrift objects +transport = TTransport.TFramedTransport(TSocket.TSocket("localhost", PORT)) +protocol = TBinaryProtocol.TBinaryProtocol(transport) +client = LucidaService.Client(protocol) + +transport.open() +print "///// Connecting to template... /////" +results = client.infer(LUCID, query_spec) +print "///// Result: /////" +print "%s" % results +transport.close() \ No newline at end of file diff --git a/lucida/template/python/server/server.py b/lucida/template/python/server/server.py new file mode 100644 index 000000000..d88c6ea12 --- /dev/null +++ b/lucida/template/python/server/server.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +import sys +sys.path.append('../') + +from templateConfig import * + +from lucidatypes.ttypes import QuerySpec +from lucidaservice import LucidaService + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +# TODO: Adding modules your services needed + +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + +class templateHandler(LucidaService.Iface): + def create(self, LUCID, spec): + # TODO: Adding your own infer function. Check the top-level README to + # figure out each parameter represents. + # For the template, do nothing + return + + def learn(self, LUCID, knowledge): + # TODO: Adding your own learn function. Check the top-level README to + # figure out each parameter represents. + # For the template, do nothing + return + + def infer(self, LUCID, query): + # TODO: Adding your own infer function. Check the top-level README to + # figure out each parameter represents. + # For the template, print the query info and return "Anwser is XXX" + print("@@@@@ Infer; User: " + LUCID) + if len(query.content) == 0 or len(query.content[0].data) == 0: + return "error: incorrect query" + query_data = query.content[0].data[-1] + print("Asking: " + query_data) + answer_data = "This is the sample answer" + print("Result: " + answer_data) + return answer_data + +# Set handler to our implementation and setup the server +handler = templateHandler() +processor = LucidaService.Processor(handler) +transport = TSocket.TServerSocket(port=PORT) +tfactory = TTransport.TFramedTransportFactory() +pfactory = TBinaryProtocol.TBinaryProtocolFactory() +server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + +# Display useful information on the command center and start the server +# Change 'XXX' into your service's acronym +print 'TPL at port %d' % PORT +server.serve() diff --git a/lucida/template/python/templateConfig.py b/lucida/template/python/templateConfig.py new file mode 100644 index 000000000..e31bfb1c1 --- /dev/null +++ b/lucida/template/python/templateConfig.py @@ -0,0 +1,5 @@ +""" +This a guide for how to add your own microservice into Lucida interface +""" + +# TODO: Other configuration for your own service diff --git a/lucida/weather/Makefile b/lucida/weather/Makefile index 4d90c1d85..05e30bc98 100644 --- a/lucida/weather/Makefile +++ b/lucida/weather/Makefile @@ -15,7 +15,13 @@ clean: rm -rf lucidaservice lucidatypes start_server: - cd server; python WeatherServer.py + @if [ "$(port)" != "" ]; then \ + cd server; \ + python WeatherServer.py $(port); \ + fi -start_test: - cd client; python WeatherClient.py +start_test: + @if [ "$(port)" != "" ]; then \ + cd client; \ + python WeatherClient.py $(port); \ + fi \ No newline at end of file diff --git a/lucida/weather/WeatherConfig.py b/lucida/weather/WeatherConfig.py index 2db625228..0b1fcd8c8 100644 --- a/lucida/weather/WeatherConfig.py +++ b/lucida/weather/WeatherConfig.py @@ -2,33 +2,12 @@ Weather API configuration details """ -import ConfigParser, sys - -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[asection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - -cp = ConfigParser.SafeConfigParser() -cp.readfp(FakeSecHead(open("../../config.properties"))) -port_dic = dict(cp.items('asection')) -PORT = int(port_dic['we_port']) - # Weather Underground API key # https://www.wunderground.com/weather/api/ WU_API_URL_BASE = 'http://api.wunderground.com/api/' -WU_API_KEY = # TODO: add your API key here +WU_API_KEY = 'ff76e8e80c802d46' # TODO: add your API key here # Open Weather Map API key # https://openweathermap.org/api OWM_API_URL_BASE = 'http://api.openweathermap.org/data/2.5/weather?' -OWM_API_KEY = # TODO: add your API key here +OWM_API_KEY = '362537b891e6df03a316e82565fe4df3' # TODO: add your API key here diff --git a/lucida/weather/client/WeatherClient.py b/lucida/weather/client/WeatherClient.py index 5a725bbf2..ffb6cf37a 100644 --- a/lucida/weather/client/WeatherClient.py +++ b/lucida/weather/client/WeatherClient.py @@ -3,7 +3,6 @@ import sys sys.path.append('../') -from WeatherConfig import PORT from lucidatypes.ttypes import QueryInput, QuerySpec from lucidaservice import LucidaService @@ -12,6 +11,11 @@ from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + LUCID = "Clinc" query_input_data = "What's the weather in Ann Arbor, MI?" query_input = QueryInput(type="query", data=[query_input_data]) diff --git a/lucida/weather/server/WeatherServer.py b/lucida/weather/server/WeatherServer.py index 8ae390520..276c1bf36 100644 --- a/lucida/weather/server/WeatherServer.py +++ b/lucida/weather/server/WeatherServer.py @@ -16,6 +16,11 @@ import json import urllib +if len(sys.argv) != 2: + print('Wrong arguments!') + exit(1) +PORT = int(sys.argv[1]) + class WeatherHandler(LucidaService.Iface): def create(self, LUCID, spec): """ @@ -83,5 +88,5 @@ def infer(self, LUCID, query): pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) -print 'WE at port %d' % PORT +print 'WE at port %d' % PORT server.serve() diff --git a/tools/Makefile b/tools/Makefile index 7dd3ebc8a..39387d080 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -4,11 +4,8 @@ all: (sudo ./apt_deps.sh) || (exit 1) (sudo ./install_python.sh) || (exit 1) (sudo ./install_java.sh) || (exit 1) - (sudo ./install_opencv.sh) || (exit 1) (sudo ./install_thrift.sh) || (exit 1) - (sudo ./install_fbthrift.sh) || (exit 1) (sudo ./install_mongodb.sh) || (exit 1) clean: sudo ./clean.sh - diff --git a/tools/apt_deps.sh b/tools/apt_deps.sh index 65db094a4..29a351b55 100755 --- a/tools/apt_deps.sh +++ b/tools/apt_deps.sh @@ -75,4 +75,5 @@ apt-get install -y zlib1g-dev \ libffi-dev \ libbz2-dev \ python-yaml \ -&& pip install virtualenv ws4py + python-pygame \ +&& pip install virtualenv ws4py dill diff --git a/tools/deploy/asrmaster-controller-https.yaml b/tools/deploy/asrmaster-controller-https.yaml index 4b91bdde2..022245c0f 100644 --- a/tools/deploy/asrmaster-controller-https.yaml +++ b/tools/deploy/asrmaster-controller-https.yaml @@ -14,7 +14,7 @@ spec: name: asrmaster spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] args: ['cd $LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/ && export WSS=True && source $LUCIDAROOT/../tools/python_2_7_12/bin/activate && python kaldigstserver/master_server.py --port=8081'] name: asrmaster diff --git a/tools/deploy/asrmaster-controller.yaml b/tools/deploy/asrmaster-controller.yaml index 8f4a74758..5d89e5b43 100644 --- a/tools/deploy/asrmaster-controller.yaml +++ b/tools/deploy/asrmaster-controller.yaml @@ -14,7 +14,7 @@ spec: name: asrmaster spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] args: ['cd $LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/ && source $LUCIDAROOT/../tools/python_2_7_12/bin/activate && python kaldigstserver/master_server.py --port=8081'] name: asrmaster diff --git a/tools/deploy/asrworker-controller-https.yaml b/tools/deploy/asrworker-controller-https.yaml index 54a30ac8f..e1e186148 100644 --- a/tools/deploy/asrworker-controller-https.yaml +++ b/tools/deploy/asrworker-controller-https.yaml @@ -14,7 +14,7 @@ spec: name: asrworker spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] args: ['cd $LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/ && export GST_PLUGIN_PATH=$LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/kaldi/tools/gst-kaldi-nnet2-online/src && python kaldigstserver/worker.py -u wss://$ASRMASTER_PORT_8081_TCP_ADDR:8081/worker/ws/speech -c sample_english_nnet2.yaml'] name: asrworker diff --git a/tools/deploy/asrworker-controller.yaml b/tools/deploy/asrworker-controller.yaml index 3b40d44bf..ab62aa0cd 100644 --- a/tools/deploy/asrworker-controller.yaml +++ b/tools/deploy/asrworker-controller.yaml @@ -14,7 +14,7 @@ spec: name: asrworker spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] args: ['cd $LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/ && export GST_PLUGIN_PATH=$LUCIDAROOT/speechrecognition/kaldi_gstreamer_asr/kaldi/tools/gst-kaldi-nnet2-online/src && python kaldigstserver/worker.py -u ws://$ASRMASTER_PORT_8081_TCP_ADDR:8081/worker/ws/speech -c sample_english_nnet2.yaml'] name: asrworker diff --git a/tools/deploy/ca-controller.yaml b/tools/deploy/ca-controller.yaml index 47b6b5bd3..b0ba572bb 100644 --- a/tools/deploy/ca-controller.yaml +++ b/tools/deploy/ca-controller.yaml @@ -14,9 +14,9 @@ spec: name: ca spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/calendar/ && ./gradlew run'] + args: ['cd $LUCIDAROOT/calendar/ && make start_server port=8084'] name: ca ports: - containerPort: 8084 diff --git a/tools/deploy/dig-controller.yaml b/tools/deploy/dig-controller.yaml index 93e8eb770..e2dc0415e 100644 --- a/tools/deploy/dig-controller.yaml +++ b/tools/deploy/dig-controller.yaml @@ -14,9 +14,9 @@ spec: name: dig spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/djinntonic/dig && ./DIGServer'] + args: ['cd $LUCIDAROOT/djinntonic/dig && make start_server port=8087'] name: dig ports: - containerPort: 8087 diff --git a/tools/deploy/face-controller.yaml b/tools/deploy/face-controller.yaml index 5b94ae1cb..08bd757f5 100644 --- a/tools/deploy/face-controller.yaml +++ b/tools/deploy/face-controller.yaml @@ -14,9 +14,9 @@ spec: name: face spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/djinntonic/face && ./FACEServer'] + args: ['cd $LUCIDAROOT/djinntonic/face && make start_server port=8086'] name: face ports: - containerPort: 8086 diff --git a/tools/deploy/imc-controller.yaml b/tools/deploy/imc-controller.yaml index 9c5cfb62c..f42d23297 100644 --- a/tools/deploy/imc-controller.yaml +++ b/tools/deploy/imc-controller.yaml @@ -14,9 +14,9 @@ spec: name: imc spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/djinntonic/imc && ./IMCServer'] + args: ['cd $LUCIDAROOT/djinntonic/imc && make start_server port=8085'] name: imc ports: - containerPort: 8085 diff --git a/tools/deploy/imm-controller.yaml b/tools/deploy/imm-controller.yaml index 6da7646c2..443a8f115 100644 --- a/tools/deploy/imm-controller.yaml +++ b/tools/deploy/imm-controller.yaml @@ -14,9 +14,9 @@ spec: name: imm spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/imagematching/opencv_imm/server/ && ./imm_server'] + args: ['cd $LUCIDAROOT/imagematching/opencv_imm/ && make start_server port=8082'] name: imm ports: - containerPort: 8082 diff --git a/tools/deploy/ms-controller.yaml b/tools/deploy/ms-controller.yaml index 00974ef69..a2aa15eca 100644 --- a/tools/deploy/ms-controller.yaml +++ b/tools/deploy/ms-controller.yaml @@ -14,9 +14,9 @@ spec: name: ms spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/musicservice/server/ && python server.py'] + args: ['cd $LUCIDAROOT/musicservice/ && make start_server port=8089'] name: ms ports: - containerPort: 8089 diff --git a/tools/deploy/qa-controller.yaml b/tools/deploy/qa-controller.yaml index 1b5cef0b9..ff137414c 100644 --- a/tools/deploy/qa-controller.yaml +++ b/tools/deploy/qa-controller.yaml @@ -15,11 +15,11 @@ spec: spec: dnsPolicy: Default containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] # Delete " && export wiki_indri_index=/usr/local/lucida/lucida/questionanswering/OpenEphyra/db/wiki_indri_index" # if you don't want to use Wikipedia as an additional knowledge base. - args: ['cd $LUCIDAROOT/questionanswering/OpenEphyra/ && export wiki_indri_index=/usr/local/lucida/lucida/questionanswering/OpenEphyra/db/wiki_indri_index && ./start_server.sh'] + args: ['cd $LUCIDAROOT/questionanswering/OpenEphyra/ && make start_server port=8083'] name: qa ports: - containerPort: 8083 diff --git a/tools/deploy/we-controller.yaml b/tools/deploy/we-controller.yaml index f278376e8..7b044263d 100644 --- a/tools/deploy/we-controller.yaml +++ b/tools/deploy/we-controller.yaml @@ -14,9 +14,9 @@ spec: name: we spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_services:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/weather/server/ && python WeatherServer.py'] + args: ['cd $LUCIDAROOT/weather/ && make start_server port=8088'] name: we ports: - containerPort: 8088 diff --git a/tools/deploy/web-controller-https.yaml b/tools/deploy/web-controller-https.yaml index 10685c160..a839df6f3 100644 --- a/tools/deploy/web-controller-https.yaml +++ b/tools/deploy/web-controller-https.yaml @@ -14,7 +14,7 @@ spec: name: web spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] # Modify the domain name (but leave the port number unchanged). args: ['cd $LUCIDAROOT/commandcenter/apache/ && export ASR_ADDR_PORT=wss://clarity13.eecs.umich.edu:30001 && cp -f $LUCIDAROOT/commandcenter/apache/conf/000-default_https.conf /etc/apache2/sites-available/000-default.conf && sudo a2enmod ssl && env > envs.txt && sudo chown -R www-data:www-data /usr/local/lucida && sudo /usr/sbin/apache2ctl -D FOREGROUND'] diff --git a/tools/deploy/web-controller.yaml b/tools/deploy/web-controller.yaml index d55aeae7f..7da1ed996 100644 --- a/tools/deploy/web-controller.yaml +++ b/tools/deploy/web-controller.yaml @@ -14,9 +14,9 @@ spec: name: web spec: containers: - - image: claritylab/lucida:v2.2 + - image: czx5049/lucida_core:v1.2 command: ['/bin/sh', '-c'] - args: ['cd $LUCIDAROOT/commandcenter/apache/ && export ASR_ADDR_PORT=ws://localhost:30001 && env > envs.txt && sudo chown -R www-data:www-data /usr/local/lucida && sudo /usr/sbin/apache2ctl -D FOREGROUND'] + args: ['cd $LUCIDAROOT/commandcenter/ && make start_server'] name: web ports: - containerPort: 80 diff --git a/tools/deploy/web-service.yaml b/tools/deploy/web-service.yaml index 609debc9c..0c36a75a9 100644 --- a/tools/deploy/web-service.yaml +++ b/tools/deploy/web-service.yaml @@ -8,8 +8,8 @@ spec: type: NodePort ports: - nodePort: 30000 - port: 80 - targetPort: 80 + port: 3000 + targetPort: 3000 protocol: TCP selector: name: web diff --git a/tools/lucida-gui/.gitignore b/tools/lucida-gui/.gitignore new file mode 100644 index 000000000..337501ef0 --- /dev/null +++ b/tools/lucida-gui/.gitignore @@ -0,0 +1,3 @@ +company_dill.pkl +*.pyc +*.tmp \ No newline at end of file diff --git a/tools/lucida-gui/eztext.py b/tools/lucida-gui/eztext.py new file mode 100755 index 000000000..06d3ec7eb --- /dev/null +++ b/tools/lucida-gui/eztext.py @@ -0,0 +1,153 @@ +# input lib +from pygame.locals import * +import pygame, string + +class ConfigError(KeyError): pass + +class Config: + """ A utility for configuration """ + def __init__(self, options, *look_for): + assertions = [] + for key in look_for: + if key[0] in options.keys(): exec('self.'+key[0]+' = options[\''+key[0]+'\']') + else: exec('self.'+key[0]+' = '+key[1]) + assertions.append(key[0]) + for key in options.keys(): + if key not in assertions: raise ConfigError(key+' not expected as option') + +class Input: + """ A text input for pygame apps """ + def __init__(self, **options): + """ Options: x, y, font, color, restricted, maxlength, prompt """ + self.options = Config(options, ['x', '0'], ['y', '0'], ['font', 'pygame.font.Font(None, 32)'], + ['color', '(0,0,0)'], ['restricted', '\'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!"#$%&\\\'()*+,-./:;<=>?@[\]^_`{|}~\''], + ['maxlength', '-1'], ['prompt', '\'\'']) + self.x = self.options.x; self.y = self.options.y + self.font = self.options.font + self.color = self.options.color + self.restricted = self.options.restricted + self.maxlength = self.options.maxlength + self.prompt = self.options.prompt; self.value = '' + self.shifted = False + + def set_pos(self, x, y): + """ Set the position to x, y """ + self.x = x + self.y = y + + def set_font(self, font): + """ Set the font for the input """ + self.font = font + + def draw(self, surface): + """ Draw the text input to a surface """ + text = self.font.render(self.prompt+self.value, 1, self.color) + surface.blit(text, (self.x, self.y)) + + def update(self, events): + """ Update the input based on passed events """ + for event in events: + if event.type == KEYUP: + if event.key == K_LSHIFT or event.key == K_RSHIFT: self.shifted = False + if event.type == KEYDOWN: + if event.key == K_BACKSPACE: self.value = self.value[:-1] + elif event.key == K_LSHIFT or event.key == K_RSHIFT: self.shifted = True + elif event.key == K_SPACE: self.value += ' ' + if not self.shifted: + if event.key == K_a and 'a' in self.restricted: self.value += 'a' + elif event.key == K_b and 'b' in self.restricted: self.value += 'b' + elif event.key == K_c and 'c' in self.restricted: self.value += 'c' + elif event.key == K_d and 'd' in self.restricted: self.value += 'd' + elif event.key == K_e and 'e' in self.restricted: self.value += 'e' + elif event.key == K_f and 'f' in self.restricted: self.value += 'f' + elif event.key == K_g and 'g' in self.restricted: self.value += 'g' + elif event.key == K_h and 'h' in self.restricted: self.value += 'h' + elif event.key == K_i and 'i' in self.restricted: self.value += 'i' + elif event.key == K_j and 'j' in self.restricted: self.value += 'j' + elif event.key == K_k and 'k' in self.restricted: self.value += 'k' + elif event.key == K_l and 'l' in self.restricted: self.value += 'l' + elif event.key == K_m and 'm' in self.restricted: self.value += 'm' + elif event.key == K_n and 'n' in self.restricted: self.value += 'n' + elif event.key == K_o and 'o' in self.restricted: self.value += 'o' + elif event.key == K_p and 'p' in self.restricted: self.value += 'p' + elif event.key == K_q and 'q' in self.restricted: self.value += 'q' + elif event.key == K_r and 'r' in self.restricted: self.value += 'r' + elif event.key == K_s and 's' in self.restricted: self.value += 's' + elif event.key == K_t and 't' in self.restricted: self.value += 't' + elif event.key == K_u and 'u' in self.restricted: self.value += 'u' + elif event.key == K_v and 'v' in self.restricted: self.value += 'v' + elif event.key == K_w and 'w' in self.restricted: self.value += 'w' + elif event.key == K_x and 'x' in self.restricted: self.value += 'x' + elif event.key == K_y and 'y' in self.restricted: self.value += 'y' + elif event.key == K_z and 'z' in self.restricted: self.value += 'z' + elif event.key == K_0 and '0' in self.restricted: self.value += '0' + elif event.key == K_1 and '1' in self.restricted: self.value += '1' + elif event.key == K_2 and '2' in self.restricted: self.value += '2' + elif event.key == K_3 and '3' in self.restricted: self.value += '3' + elif event.key == K_4 and '4' in self.restricted: self.value += '4' + elif event.key == K_5 and '5' in self.restricted: self.value += '5' + elif event.key == K_6 and '6' in self.restricted: self.value += '6' + elif event.key == K_7 and '7' in self.restricted: self.value += '7' + elif event.key == K_8 and '8' in self.restricted: self.value += '8' + elif event.key == K_9 and '9' in self.restricted: self.value += '9' + elif event.key == K_BACKQUOTE and '`' in self.restricted: self.value += '`' + elif event.key == K_MINUS and '-' in self.restricted: self.value += '-' + elif event.key == K_EQUALS and '=' in self.restricted: self.value += '=' + elif event.key == K_LEFTBRACKET and '[' in self.restricted: self.value += '[' + elif event.key == K_RIGHTBRACKET and ']' in self.restricted: self.value += ']' + elif event.key == K_BACKSLASH and '\\' in self.restricted: self.value += '\\' + elif event.key == K_SEMICOLON and ';' in self.restricted: self.value += ';' + elif event.key == K_QUOTE and '\'' in self.restricted: self.value += '\'' + elif event.key == K_COMMA and ',' in self.restricted: self.value += ',' + elif event.key == K_PERIOD and '.' in self.restricted: self.value += '.' + elif event.key == K_SLASH and '/' in self.restricted: self.value += '/' + elif self.shifted: + if event.key == K_a and 'A' in self.restricted: self.value += 'A' + elif event.key == K_b and 'B' in self.restricted: self.value += 'B' + elif event.key == K_c and 'C' in self.restricted: self.value += 'C' + elif event.key == K_d and 'D' in self.restricted: self.value += 'D' + elif event.key == K_e and 'E' in self.restricted: self.value += 'E' + elif event.key == K_f and 'F' in self.restricted: self.value += 'F' + elif event.key == K_g and 'G' in self.restricted: self.value += 'G' + elif event.key == K_h and 'H' in self.restricted: self.value += 'H' + elif event.key == K_i and 'I' in self.restricted: self.value += 'I' + elif event.key == K_j and 'J' in self.restricted: self.value += 'J' + elif event.key == K_k and 'K' in self.restricted: self.value += 'K' + elif event.key == K_l and 'L' in self.restricted: self.value += 'L' + elif event.key == K_m and 'M' in self.restricted: self.value += 'M' + elif event.key == K_n and 'N' in self.restricted: self.value += 'N' + elif event.key == K_o and 'O' in self.restricted: self.value += 'O' + elif event.key == K_p and 'P' in self.restricted: self.value += 'P' + elif event.key == K_q and 'Q' in self.restricted: self.value += 'Q' + elif event.key == K_r and 'R' in self.restricted: self.value += 'R' + elif event.key == K_s and 'S' in self.restricted: self.value += 'S' + elif event.key == K_t and 'T' in self.restricted: self.value += 'T' + elif event.key == K_u and 'U' in self.restricted: self.value += 'U' + elif event.key == K_v and 'V' in self.restricted: self.value += 'V' + elif event.key == K_w and 'W' in self.restricted: self.value += 'W' + elif event.key == K_x and 'X' in self.restricted: self.value += 'X' + elif event.key == K_y and 'Y' in self.restricted: self.value += 'Y' + elif event.key == K_z and 'Z' in self.restricted: self.value += 'Z' + elif event.key == K_0 and ')' in self.restricted: self.value += ')' + elif event.key == K_1 and '!' in self.restricted: self.value += '!' + elif event.key == K_2 and '@' in self.restricted: self.value += '@' + elif event.key == K_3 and '#' in self.restricted: self.value += '#' + elif event.key == K_4 and '$' in self.restricted: self.value += '$' + elif event.key == K_5 and '%' in self.restricted: self.value += '%' + elif event.key == K_6 and '^' in self.restricted: self.value += '^' + elif event.key == K_7 and '&' in self.restricted: self.value += '&' + elif event.key == K_8 and '*' in self.restricted: self.value += '*' + elif event.key == K_9 and '(' in self.restricted: self.value += '(' + elif event.key == K_BACKQUOTE and '~' in self.restricted: self.value += '~' + elif event.key == K_MINUS and '_' in self.restricted: self.value += '_' + elif event.key == K_EQUALS and '+' in self.restricted: self.value += '+' + elif event.key == K_LEFTBRACKET and '{' in self.restricted: self.value += '{' + elif event.key == K_RIGHTBRACKET and '}' in self.restricted: self.value += '}' + elif event.key == K_BACKSLASH and '|' in self.restricted: self.value += '|' + elif event.key == K_SEMICOLON and ':' in self.restricted: self.value += ':' + elif event.key == K_QUOTE and '"' in self.restricted: self.value += '"' + elif event.key == K_COMMA and '<' in self.restricted: self.value += '<' + elif event.key == K_PERIOD and '>' in self.restricted: self.value += '>' + elif event.key == K_SLASH and '?' in self.restricted: self.value += '?' + + if len(self.value) > self.maxlength and self.maxlength >= 0: self.value = self.value[:-1] diff --git a/tools/lucida-gui/gui.py b/tools/lucida-gui/gui.py new file mode 100644 index 000000000..ec485e429 --- /dev/null +++ b/tools/lucida-gui/gui.py @@ -0,0 +1,1338 @@ +import pygame, sys, os +from time import sleep +from pygame.locals import * +pygame.init() +import math +from gui_backend_cloud import db +import random +import base64 +import dill +from cStringIO import StringIO +import pygame, sys, eztext +from textrect import * + + +# make sure no file named .hiddentempfile is under this folder +def input_impl(): + os.system('gedit .hiddentempfile') + file = open('.hiddentempfile') + code = "" + for line in file: + code = code + line + os.system('rm .hiddentempfile') + print(code) + return code + + +#used for modifying code for state/branch +def writeFileForUseMod(inputWrite): + text_file = open(".hiddentempfile", "w") + text_file.write(inputWrite) + text_file.close() + + + +def getUserInputText(): + # initialize pygame + pygame.init() + # create the screen + screen = pygame.display.set_mode((1280,240)) + # fill the screen w/ white + screen.fill((255,255,255)) + # here is the magic: making the text input + # create an input with a max length of 45, + # and a red color and a prompt saying 'type here: ' + txtbx = eztext.Input(maxlength=245, color=(255,0,0), prompt='type: ') + # create the pygame clock + clock = pygame.time.Clock() + # main loop! + + while 1: + # make sure the program is running at 30 fps + clock.tick(30) + + # events for txtbx + events = pygame.event.get() + # process other events + for event in events: + # close it x button si pressed + if event.type == QUIT: + return txtbx.value + + + # clear the screen + screen.fill((255,255,255)) + # update txtbx + txtbx.update(events) + # blit txtbx on the sceen + txtbx.draw(screen) + # refresh the display + pygame.display.flip() + + + + +class imageObject(object): + def __init__(self,name,xPos,yPos,xWidth,yWidth,imageThing,textInBox=" "): + #These are in 2D vector space. + self.xWidth = xWidth + self.yWidth = yWidth + self.xPos = xPos + self.yPos = yPos + self.text = textInBox + self.imageThing = imageThing + self.drawLevel = 0 + self.name = name + def drawImage(self,screen,screenWidth,screenHeight): + widthInPixels = int(screenWidth*self.xWidth) + heightInPixels = int(screenHeight*self.yWidth) + self.imageThing = pygame.transform.scale(self.imageThing, (int(screenWidth*self.xWidth),int(screenHeight*self.yWidth))) + screen.blit(self.imageThing, (self.xPos*screenWidth, self.yPos*screenHeight)) + + my_rect = pygame.Rect((self.xPos*screenWidth, self.yPos*screenHeight, widthInPixels , heightInPixels)) + + rendered_text = render_textrect(self.text, myfont, my_rect, (0,0,0), (48, 255, 48), 1) + if rendered_text: + screen.blit(rendered_text, my_rect.topleft) + + def setDrawLevel(self,valueSet): + self.drawLevel = valueSet + def isPointOnImage(self,xVectorSpace,yVectorSpace): + if((xVectorSpace>self.xPos) and (xVectorSpace<(self.xPos+self.xWidth)) and (yVectorSpace>self.yPos) and (yVectorSpace<(self.yPos+self.yWidth))): + return "YUP" + return "NOPE" + + +def drawArrow(screen,point0x,point0y,point1x,point1y): + + pos1=point0x,point0y + pos2=point1x,point1y + + pygame.draw.line(screen, (0,0,0), pos1, pos2) + + arrow=pygame.Surface((50,50)) + arrow.fill((255,255,255)) + pygame.draw.line(arrow, (0,0,0), (0,0), (25,25)) + pygame.draw.line(arrow, (0,0,0), (0,50), (25,25)) + arrow.set_colorkey((255,255,255)) + + angle=math.atan2(-(pos1[1]-pos2[1]), pos1[0]-pos2[0]) + angle=math.degrees(angle) + + def drawAng(angle, pos): + nar=pygame.transform.rotate(arrow,angle) + nrect=nar.get_rect(center=pos) + screen.blit(nar, nrect) + + angle+=180 + drawAng(angle, pos2) + + + + + + + +class lineObject(object): + def __init__(self,name,xPos,yPos,xPosEnd,yPosEnd): + #These are in 2D vector space. + self.xPos = xPos + self.yPos = yPos + self.xPosEnd = xPosEnd + self.yPosEnd = yPosEnd + self.drawLevel = 0 + self.name = name + def drawImage(self,screen,screenWidth,screenHeight): + startX = self.xPos*screenWidth + startY = self.yPos*screenHeight + endX = self.xPosEnd*screenWidth + endY = self.yPosEnd*screenHeight + pygame.draw.line(windowScreen.screen,0,(startX,startY),(endX,endY),1) + + + + drawArrow(screen,startX,startY,endX,endY) + + def setDrawLevel(self,valueSet): + self.drawLevel = valueSet + def isPointOnImage(self,xVectorSpace,yVectorSpace): + return "NOPE" + +class windowInformation(object): + def __init__(self,x,y): + self.width = x + self.height = y + self.screen = pygame.display.set_mode((x, y)) + self.listOfObjectsNew = dict(); + + def refreshWindow(self): + self.screen = pygame.display.set_mode((self.width, self.height)) + + # Go through every image object and draw to screen + def blitScreen(self): + print("=======BLITTING SCREEN=====") + self.screen.fill(0) + bgIMGtmp = bgIMG; + bgIMGtmp = pygame.transform.scale(bgIMGtmp, (self.width,self.height)) + self.screen.blit(bgIMGtmp, (0,0)) + drawLevels = -1 + #Some images are supposed to draw ontop of otehrs + while drawLevels!=10: + drawLevels+=1 + for key in self.listOfObjectsNew: + if(self.listOfObjectsNew[key].drawLevel==drawLevels): + self.listOfObjectsNew[key].drawImage(self.screen,self.width,self.height) + pygame.display.flip() + + + #Add an image object onto the screen + def addObject(self,objectName,objectThing): + self.listOfObjectsNew[objectName]= objectThing; + + def removeObject(self,objectName): + del self.listOfObjectsNew[objectName] + + + def getClickOnName(self,xClick,yClick): + xClickVectorSpace = float(xClick)/self.width + yClickVectorSpace = float(yClick)/self.height + + + + #When checking where click, must go through the top to bottom of the image stack + drawLevels = 10 + while drawLevels!=-1: + + for key in self.listOfObjectsNew: + #print(drawLevels,key,self.listOfObjectsNew[key].drawLevel) + if(self.listOfObjectsNew[key].drawLevel==drawLevels): + status = self.listOfObjectsNew[key].isPointOnImage(xClickVectorSpace,yClickVectorSpace) + #print("NODE CHECK", key,xClickVectorSpace,yClickVectorSpace,status,self.listOfObjectsNew[key].drawLevel,drawLevels) + if(status=="YUP"): + return key + drawLevels-=1 + + + return "NOPE" + + + + + + + + + + + + +# Load the image types +yellowBG = pygame.image.load(os.path.join("images/yellowBG.png")) +grayBG = pygame.image.load(os.path.join("images/grayBG.png")) +dead = pygame.image.load(os.path.join("images/dead.png")) + +nodeActiveIMG = pygame.image.load(os.path.join("images/nodeActive.png")) +nodeIMG = pygame.image.load(os.path.join("images/node.png")) +serviceListButtonIMG = pygame.image.load(os.path.join("images/serviceListButton.png")) +bgIMG = pygame.image.load(os.path.join("images/bg.png")) + + + + + +def getNodeImage(node): + if(node.active==1): + return nodeActiveIMG + return nodeIMG + + + +textOffsetX = 0.025 +textOffsetY = 0.025 + +class lucidaGUI(object): + def __init__(self): + self.dirList = [] + self.WFXOffset = 0.0 + self.WFYOffset = -0.0 + self.currentLevelName = "" + + + #Clicking on level, find what is being clicked on. + def clickLevel(self,xClick,yClick): + clickOnName = windowScreen.getClickOnName(xClick,yClick) + if(clickOnName!="NOPE"): + self.level(clickOnName,1) + + + + def refreshCurrent(self): + windowScreen.refreshWindow() + upDirName = directoriesNames.pop() + lucida.level(upDirName,0) + + def getXYPositionEntry(self,ID): + x = ID%5 + ID /= 5 + y = ID + return [x*0.1,y*0.1] + + def goUpDir(self, time): + countArr = len(directoriesNames) + + if countArr>1: + upDirName = directoriesNames.pop() + for count in range(time): + upDirName = directoriesNames.pop() + self.level(upDirName,0) + + + def updateObjectPosition(self,name,xPos,yPos): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.xPos = xPos + objectThing.yPos = yPos + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + def updateObjectPosition2(self,name,xPos,yPos): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.xPosEnd = xPos + objectThing.yPosEnd = yPos + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + + + def updateObjectImage(self,name,imageThing): + for objectThing in self.dirList: + if(objectThing.name==name): + objectThing.imageThing = imageThing + objectThing.drawImage(windowScreen.screen,windowScreen.width,windowScreen.height) + + def addToLevel(self,xPos,yPos,xWidth,yWidth,drawLevel,imageObjectName,image,text=" "): + imageObjectInst = imageObject(imageObjectName, xPos,yPos,xWidth,yWidth,image,text) + imageObjectInst.setDrawLevel(drawLevel) + self.dirList.append(imageObjectInst) + + + + def addToLevelLine(self,xPos,yPos,drawLevel,imageObjectName,xPosEnd,yPosEnd): + textObjectInst = lineObject(imageObjectName,xPos,yPos,xPosEnd,yPosEnd) + textObjectInst.setDrawLevel(drawLevel) + self.dirList.append(textObjectInst) + + + # Removes the current display + def undisplayCurrentLevel(self): + + for objectThing in self.dirList: + windowScreen.removeObject(objectThing.name) + self.dirList = [] + # Adds the current (And possibly new) display + def displayCurrentLevel(self): + print("======++DISPLAY CURRENT++======") + for objectThing in self.dirList: + windowScreen.addObject(objectThing.name,objectThing) + windowScreen.blitScreen(); + + def getXYNode(self,node): + XY = [] + XY.append(node.x-self.WFXOffset) + XY.append(node.y-self.WFYOffset) + return XY + + + #Level function draws the level display and the level click functions + def level(self,levelName,directClick): + print("start") + print(directoriesNames) + #print("Hit level", levelName) + if(directClick==1): + self.currentLevelName = levelName + + + didHit = 0 + goUpDirtime = 0 + self.undisplayCurrentLevel(); + ### Front: CMD/MS buttons + if(levelName=="root"): + didHit = 1 + self.addToLevel(0.25,0.4,0.5,0.1,0,"choosemode",yellowBG,"Please choose mode!") + self.addToLevel(0.25,0.5,0.25,0.1,0,"root_local",grayBG,"Local") + self.addToLevel(0.5,0.5,0.25,0.1,0,"root_docker",grayBG,"Docker") + + if(levelName=="root_local"): + didHit = 1 + self.addToLevel(0.25,0.4,0.5,0.1,0,"wfList",yellowBG,"Workflows") + self.addToLevel(0.25,0.5,0.5,0.1,0,"msList",yellowBG,"Microservices") + self.addToLevel(0.25,0.6,0.5,0.1,0,"bbList",yellowBG,"Blackboxes") + + + # List all of the workflow types + if(levelName=="wfList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.25,0.10,0,"wfListHeader:",yellowBG,"Workflows:") + self.addToLevel(0.50,0.25,0.25,0.10,0,"newWF:",yellowBG,"New") + for WF in workflowList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"wfThing:"+str(countServices),grayBG,WF.name) + countServices+= 1 + + + ## bbList has not been implemented in the GUI yet + if(levelName=="bbList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.50,0.10,0,"bbListHeader:",yellowBG,"Microservices:") + for MS in microServiceList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"bbListThing"+str(countServices),grayBG,MS.name) + countServices+= 1 + + if "bbListThing" in levelName: + didHit = 1 + getMSID = int(filter(str.isdigit, levelName)) + msName = microServiceList[getMSID].name + self.addToLevel(0.25,0.25,0.50,0.10,0,"MicroservicesNameThing:"+str(getMSID),yellowBG,"MS: " + msName) + countServices = 0 + for server in microServiceList[getMSID].serverList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"bbinstance" + str(getMSID) + "bbinstance"+str(countServices),grayBG,server.name) + countServices+= 1 + + if "bbinstance" in levelName: + didHit = 1 + print("BELOW THIS LINE") + idList = levelName.split("bbinstance") + msID = int(idList[1]) + boxID = int(idList[2]) + print(levelName.split("bbinstance")) + box = microServiceList[msID].serverList[boxID] + msName = microServiceList[msID].name + boxName = box.name + + self.addToLevel(0.25,0.25,0.25,0.10,0,"MS:" + msName,yellowBG,"MS: " + msName) + self.addToLevel(0.50,0.25,0.25,0.10,0,"BoxNameBB:",yellowBG,"Box: " + boxName) + self.addToLevel(0.25,0.35,0.50,0.10,0,"StartServer:" +str(msID) + "StartServer:" + str(boxID),yellowBG,"Start!") + + if "StartServer:" in levelName: + idList = levelName.split("StartServer:") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + status = db.start_server(microServiceList[msID].dbData['_id'], box.dbData['id']) + self.refreshCurrent() + + + #List a specific workflow + if "wfThing:" in levelName: + didHit = 1 + getWFID = int(filter(str.isdigit, levelName)) + wfName = workflowList[getWFID].name + self.addToLevel(0.25,0.25,0.35,0.10,0,"WorkflowNameThing:"+str(getWFID),yellowBG,"WF: " + wfName) + self.addToLevel(0.60,0.25,0.25,0.10,0,"wfListThing"+str(getWFID),yellowBG,"State Graph") + + + inputTypes = ', '.join(workflowList[getWFID].dbData['input']) + + self.addToLevel(0.25,0.35,0.60,0.10,0,"wfType:"+str(getWFID),yellowBG,"Type:" + inputTypes) + self.addToLevel(0.25,0.45,0.60,0.25,0,"classPath:"+str(getWFID),yellowBG,"ClassPath:" + workflowList[getWFID].dbData['classifier']) + self.addToLevel(0.25,0.70,0.60,0.10,0,"DeleteWorkflow"+str(getWFID),dead,"Delete "+wfName) + + + + + # This is the actual state graph + if "wfListThing" in levelName: + didHit = 1 + countServices = 0; + getWFID = int(filter(str.isdigit, levelName)) + WF = workflowList[getWFID] + print(WF.nodeList) + + self.addToLevel(0.0,0.0,1.0,1.0,0,"wfBG:",bgIMG) # This gives the background of the state graph + + countServices = 0; + for node in WF.nodeList: + XY = self.getXYNode(node) + + + self.addToLevel(XY[0],XY[1],0.10,0.10,1,"node:"+str(countServices),getNodeImage(node),node.name) + + + #Display the links from this node to another + countLinks = 0 + for link in node.linkList: + toNode = WF.nodeList[link.toNode] + XY2 = self.getXYNode(toNode) + lineThickness = 0.01; + self.addToLevelLine(XY[0],XY[1],2,"IAmLine:"+str(countServices)+"IAmLine:"+str(countLinks),XY2[0],XY2[1]) + + + countLinks += 1 + + countServices += 1 + + + + + + #List microservices + if(levelName=="msList"): + didHit = 1 + countServices = 0; + + self.addToLevel(0.25,0.25,0.25,0.10,0,"msListHeader:",yellowBG,"Microservices:") + self.addToLevel(0.50,0.25,0.25,0.10,0,"newMicroService",yellowBG,"New") + for MS in microServiceList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.35+XY[1],0.10,0.10,0,"msListThing"+str(countServices),grayBG,MS.name) + countServices+= 1 + + #Display a specific microservice + if "msListThing" in levelName: + didHit = 1 + getMSID = int(filter(str.isdigit, levelName)) + msName = microServiceList[getMSID].name + self.addToLevel(0.25,0.25,0.25,0.10,0,"MicroservicesNameThing:"+str(getMSID),yellowBG,"MS: " + msName) + self.addToLevel(0.50,0.25,0.25,0.10,0,"newServer"+str(getMSID),yellowBG,"New") + self.addToLevel(0.25,0.35,0.50,0.05,0,"LearnType:"+str(getMSID),yellowBG,"LearnType: " + microServiceList[getMSID].dbData['learn']) + self.addToLevel(0.25,0.40,0.50,0.05,0,"InputType:"+str(getMSID),yellowBG,"InputType: " + microServiceList[getMSID].dbData['input']) + self.addToLevel(0.75,0.75,0.20,0.10,0,"DeleteService"+str(getMSID),dead,"Delete "+msName) + + countServices = 0 + for server in microServiceList[getMSID].serverList: + XY = self.getXYPositionEntry(countServices) + self.addToLevel(0.25+XY[0],0.45+XY[1],0.10,0.10,0,"serverBB" + str(getMSID) + "serverBB"+str(countServices),grayBG,server.name) + countServices+= 1 + + # An instance of the service + if "serverBB" in levelName: + didHit = 1 + print("BELOW THIS LINE") + idList = levelName.split("serverBB") + msID = int(idList[1]) + boxID = int(idList[2]) + print(levelName.split("serverBB")) + box = microServiceList[msID].serverList[boxID] + msName = microServiceList[msID].name + boxName = box.name + + self.addToLevel(0.25,0.25,0.25,0.10,0,"MS:" + msName,yellowBG,"MS: " + msName) + self.addToLevel(0.50,0.25,0.25,0.10,0,"BoxNameSet:" +str(msID) + "BoxNameSet:" + str(boxID),yellowBG,"Box: " + boxName) + self.addToLevel(0.25,0.35,0.50,0.05,0,"IP:PORT"+str(msID)+"IP:PORT"+str(boxID),yellowBG,"IP:PORT:" + str(box.IP) + ":" + str(box.port)) + self.addToLevel(0.25,0.40,0.50,0.35,0,"LOCATION"+str(msID)+"LOCATION"+str(boxID),yellowBG,"Location:"+str(box.location)) + self.addToLevel(0.25,0.75,0.50,0.10,0,"DeleteInstance"+str(msID)+"DeleteInstance"+str(boxID),dead,"Delete "+boxName) + + if "LOCATION" in levelName: + idList = levelName.split("LOCATION") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + returnText = getUserInputText() + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "location", returnText) + generateMicroServiceList() + self.refreshCurrent() + + if "DeleteInstance" in levelName: + didHit = 1 + goUpDirtime=2 + idList = levelName.split("DeleteInstance") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + status = db.delete_instance(microServiceList[msID].dbData['_id'], box.dbData['id']) + generateMicroServiceList() + + if "DeleteService" in levelName: + didHit = 1 + goUpDirtime=2 + getMSID = int(filter(str.isdigit, levelName)) + status = db.delete_service(microServiceList[getMSID].dbData['_id']) + generateMicroServiceList() + + if "DeleteWorkflow" in levelName: + didHit = 1 + goUpDirtime=2 + getWFID = int(filter(str.isdigit, levelName)) + status = db.delete_workflow(workflowList[getWFID].dbData['_id']) + generateWorkflowList() + + + #Change the name of a microservice + if "MicroservicesNameThing:" in levelName: + idList = levelName.split("MicroservicesNameThing:") + msID = int(idList[1]) + returnText = getUserInputText() + status = db.update_service( microServiceList[msID].dbData['_id'], "name", returnText) + status = db.update_service( microServiceList[msID].dbData['_id'], "acronym", returnText) + generateMicroServiceList() + self.refreshCurrent() + + # Change the name of a classifier path + if "classPath:" in levelName: + idList = levelName.split("classPath:") + wfID = int(idList[1]) + returnText = getUserInputText() + status = db.update_workflow( workflowList[wfID].dbData['_id'], "classifier", returnText) + generateWorkflowList() + self.refreshCurrent() + + + #Change the name of a workflow + if "WorkflowNameThing:" in levelName: + idList = levelName.split("WorkflowNameThing:") + wfID = int(idList[1]) + returnText = getUserInputText() + status = db.update_workflow( workflowList[wfID].dbData['_id'], "name", returnText) + generateWorkflowList() + workflowData = compileWorkflow(wfID) + db.update_workflow(workflowList[wfID].dbData['_id'], "code",workflowData); + self.refreshCurrent() + + + + #Change the IP:port of a microservice + if "IP:PORT" in levelName: + returnText = getUserInputText() + idList = levelName.split("IP:PORT") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + ipportinfo = returnText.split(":") + box.IP = ipportinfo[0] + box.port = int(ipportinfo[1]) + print(microServiceList[msID].dbData['_id'] , "SPACE" , box.dbData['id'] , "SPACE" + box.IP) + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "host", box.IP) + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "port", box.port) + + generateMicroServiceList() + self.refreshCurrent() + + #Name an instance + if "BoxNameSet:" in levelName: + returnText = getUserInputText() + idList = levelName.split("BoxNameSet:") + msID = int(idList[1]) + boxID = int(idList[2]) + box = microServiceList[msID].serverList[boxID] + status = db.update_instance(microServiceList[msID].dbData['_id'], box.dbData['id'], "name", returnText) + generateMicroServiceList() + self.refreshCurrent() + + + #Toggles a service between learn types + if "LearnType:" in levelName: + didHit=1 + goUpDirtime=1 + getMSID = int(filter(str.isdigit, levelName)) + if(microServiceList[getMSID].dbData['learn']=='text'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "image") + if(microServiceList[getMSID].dbData['learn']=='image'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "none") + if(microServiceList[getMSID].dbData['learn']=='none'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "learn", "text") + generateMicroServiceList() + + if "InputType:" in levelName: + didHit=1 + goUpDirtime=1 + getMSID = int(filter(str.isdigit, levelName)) + if(microServiceList[getMSID].dbData['input']=='text'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "input", "image") + if(microServiceList[getMSID].dbData['input']=='image'): + status = db.update_service( microServiceList[getMSID].dbData['_id'], "input", "text") + generateMicroServiceList() + + #Toggles a workflow between input types + if "wfType:" in levelName: + didHit=1 + goUpDirtime=1 + getWFID = int(filter(str.isdigit, levelName)) + + if(set(workflowList[getWFID].dbData['input'])==set(['text', 'image', 'text_image'])): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['image']) + elif(workflowList[getWFID].dbData['input']==['image']): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text']) + elif(workflowList[getWFID].dbData['input']==['text']): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text_image']) + elif(workflowList[getWFID].dbData['input']==['text_image']): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text','text_image']) + elif(set(workflowList[getWFID].dbData['input'])==set(['text', 'text_image'])): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['image','text_image']) + elif(set(workflowList[getWFID].dbData['input'])==set(['image', 'text_image'])): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text','image']) + elif(set(workflowList[getWFID].dbData['input'])==set(['text', 'image'])): + status = db.update_workflow( workflowList[getWFID].dbData['_id'], "input", ['text','image','text_image']) + + + + generateWorkflowList() + + + + + #New blackbox service registry + if "newServer" in levelName: + didHit=1 + goUpDirtime=1 + microserviceNameOfServerID = int(levelName.split("newServer",1)[1]) + print(microServiceList[microserviceNameOfServerID].dbData['_id']) + db.add_instance(microServiceList[microserviceNameOfServerID].dbData['_id']) + generateMicroServiceList() + + #A new workflow + if "newWF:" in levelName: + didHit=1 + goUpDirtime=1 + status,idWF = db.add_workflow() + + db.update_workflow(idWF, "name", "WFTaco"); + db.update_workflow(idWF, "input", ["text"]); + db.update_workflow(idWF, "classifier", "/home/masonhill/lucidas/lucidaapi/lucida-api/lucida/commandcenter/data/class_QAWF.txt"); + + generateWorkflowList() + + #Add a new micro service + if(levelName=="newMicroService"): + didHit=1 + goUpDirtime=1 + returnVal = 1; + newAppend = 0; + + while(returnVal!=0): + returnVal, serviceID = db.add_service() + newAppend+= 1 + + generateMicroServiceList() + + + + if(didHit==1): + countArr = len(directoriesNames) + if countArr>0: + previousDirName = directoriesNames.pop() + directoriesNames.append(previousDirName) + if(previousDirName!=levelName): + directoriesNames.append(levelName) + if(countArr==0): + directoriesNames.append(levelName) + + print("mid") + print(directoriesNames) + #directoriesNames.append(levelName) + self.displayCurrentLevel(); + + if goUpDirtime>=1: + self.goUpDir(goUpDirtime) + + + #HIt nothing, so need to refresh previous state + if(didHit==0): + upDirName = directoriesNames.pop() + lucida.level(upDirName,0) + + if(not "wfListThing" in directoriesNames[-1]): + currentActiveNode = -1 + + print("end") + print(directoriesNames) + + + + + + +class server(object): + def __init__(self,name): + self.name = name + self.IP = "127.0.0.1" + self.port = 0 + self.location = "" + + + +class microService(object): + def __init__(self,name): + self.name = name + self.serverList = [] + + + def addServer(self,name,ip="",port=0,location=""): + serverThing = server(name) + self.serverList.append(serverThing) + serverThing.IP = ip + serverThing.port = port + serverThing.location = location + +class linkObj(object): + def __init__(self,name,toNode): + self.name = name + self.toNode = toNode + self.code = "if(condArgs['pug']=='25 years'):\r\n\tself.pause = False\r\n\treturn True;" + + +class node(object): + def __init__(self,name): + self.name = name + self.linkList = [] # Contains a list of node links + self.x = 0.0; + self.y = 0.0 + self.active = 0 + self.code = "exeMS('pug',\"QA\",\"How old is Johann?\")\nEXE_BATCHED #THIS STATEMENT MUST EXIST AFTER ALL exeMS STATEMENTS. No exeMS is executed UNTIL AFTER THIS STATEMENT\ncondArgs['pug'] = batchedDataReturn['pug']\nself.ret = condArgs['pug']" + + def addLink(self,linkName,toNode): + problemHere = linkObj(linkName,toNode) + self.linkList.append(problemHere) + + +class workflow(object): + def __init__(self,name): + self.name = name + self.nodeList = [] + self.xOffset = 0.0 + self.yOffset = 0.0 + + def addNode(self,node): + self.nodeList.append(node) + +#A workflow needs to convert from stategraph to actual python code +def compileWorkflow(workflowID): + + workflowCompiled = "" + + # Each link needs its own function + + + + workflowName = workflowList[workflowID].name; + + + workflowCompiled += "\nclass "+workflowName+"(workFlow):" + + + + nodeCount = 0 + for node in workflowList[workflowID].nodeList: + + linkCount = 0 + for linkObj in node.linkList: + + + workflowCompiled += "\n\tdef branchCheck"+workflowName+str(nodeCount)+"_"+str(linkCount)+"(self,condArgs,passArgs):"; + + + + + stri = StringIO(linkObj.code) + while True: + nl = stri.readline() + if nl == '': break + workflowCompiled += "\n\t\t"+nl + + + + + linkCount +=1 + + nodeCount +=1 + + + workflowCompiled += "\n\r\n\tdef processCurrentState(self,batchingBit,batchedDataReturn,passArgs,inputModifierText,inputModifierImage):" + + workflowCompiled += "\n\t\tcondArgs = dict()" + nodeCount = 0 + + for node in workflowList[workflowID].nodeList: + workflowCompiled += "\n\t\tif(self.currentState==" + str(nodeCount) +"):" + + workflowCompiled += "\n\t\t\tif(batchingBit==1): self.batchedData = []" + + + stri = StringIO(node.code) + while True: + nl = stri.readline() + appendExeMSString = "" + if nl == '': break + #Need to check the line if it contains a batched request entry. This is [varname] = exeMS( format + currentCount = 0; + for c in nl: + + ''' + ''' + + + nl = nl.replace("exeMS(", "if(batchingBit==1): self.batchedData = appendServiceRequest(self.batchedData,") + nl = nl.replace("EXE_BATCHED", "if(batchingBit==1): return 2") + workflowCompiled += "\n\t\t\t"+nl + + workflowCompiled += "\n\t\t\tif(1==0): QUANTUM_PHYSICS()" + + linkCount = 0 + for linkObj in node.linkList: + + workflowCompiled += "\n\t\t\telif(self.branchCheck"+workflowName+str(nodeCount)+"_"+str(linkCount)+"(condArgs,passArgs)):" + workflowCompiled += "\n\t\t\t\tself.currentState = " + str(linkObj.toNode) + + + linkCount +=1 + workflowCompiled += "\n\t\t\telse: self.isEnd = True" + + workflowCompiled += "\n\t\t\treturn self.pause, self.ret" + nodeCount +=1 + + return workflowCompiled + + + + + +directoriesNames = [] +workflowList = [] +microServiceList = [] + + + + +# OBJ->DILL->DISK WRITE->DISK READ->BINARY BLOB->HEX64 +#This converts any object to base64. +def objToBase64(workflowThing): + #OBJ->DILL->DISK WRITE + with open('wf.tmp', 'wb') as f: + dill.dump(workflowThing, f) + + #DISK READ->BINARY BLOB->HEX64 + file = open('wf.tmp', 'rb') + file_content = file.read() + base64_two = base64.b64encode(file_content) + return base64_two + + + +#This converts base64 to obj +def base64ToObj(hexData): + #UN-HEX64->BINARY BLOB->DISK WRITE + writeData = hexData.decode('base64'); + + text_file = open('wf2.tmp', 'wb') + text_file.write(writeData) + text_file.close() + #DISK READ->DILL->OBJ + with open('wf2.tmp', 'rb') as f: + test = dill.load( f) + return test + + +def generateMicroServiceList(): + + global microServiceList + microServiceList = [] + serviceList = db.get_services(); + count = 0 + for service in serviceList: + #Create the microservice + if(service['name']==''): service['name'] = "NULL" + microServiceList.append(microService(service['name'])) + microServiceList[count].dbData = service + #And add a list of the valid microservice instances + countInst = 0 + for instance in service['instance']: + if(instance['name']==""): instance['name'] = "NULL" + microServiceList[count].addServer(instance['name'],instance['host'],instance['port'],instance['location']) + microServiceList[count].serverList[countInst].dbData = instance + countInst+=1 + count+= 1 + + + +def generateWorkflowList(): + + + + print("^^^^^^^^^^^^^^^^^^^WORKFLOW^^^^^^^^^^^^^^^^^^^^^") + global workflowList + workflowList = [] + workflows = db.get_workflows(); + count = 0 + for workflowI in workflows: + #Create the microservice + if( workflowI['name']==''): workflowI['name'] = "NULL" + workflowList.append(workflow( workflowI['name'])) + workflowList[count].dbData = workflowI + if( workflowI['stategraph']!=""): + workflowList[count].nodeList = base64ToObj( workflowI['stategraph']) + count+= 1 + + + + +generateMicroServiceList() +generateWorkflowList(); + + + + + + + +windowScreen = windowInformation(512,512) + +myfont = pygame.font.SysFont("arial", 14) +# render text +valueInc = 0 + + +lucida = lucidaGUI() +lucida.level("root",0) + +debounceKey = [] + + + + +#Create the debounce key stack +loop255 = 0; +while loop255!=255: + debounceKey.append(0) + loop255+=1 + + + +debounceMouse = [] +debounceMouse.append(0) +debounceMouse.append(0) +debounceMouse.append(0) + +oldMousePosition = [] +oldMousePosition.append(-1) +oldMousePosition.append(-1) +currentActiveNode = -1 + + +draggingNode = 0 +draggingWF = -1 +draggingNodeID = -1 +draggingX = 0.0 +draggingY = 0.0 + +draggingScreen = 0 +draggingScreenWF = -1 +draggingScreenX = 0.0 +draggingScreenY = 0.0 + + +debounceHm = 0; + +while True: + + + + sleep(0.03) + p = pygame.mouse.get_pos(); + keysPressed = pygame.key.get_pressed() + + #Check if backspace is hit + if(keysPressed[8]==1 and debounceKey[8]==0): + if(currentActiveNode!=-1): + nodeData = WF.nodeList[currentActiveNode] + currentActiveNode = -1 + lucida.updateObjectImage(lucida.currentLevelName,nodeIMG) + nodeData.active = 0 + lucida.goUpDir(1) + + + + + mousePress = pygame.mouse.get_pressed() + + #Get old mouse position (For moving screen around) + if(mousePress[0]==1 and debounceMouse[0]==1): + if(oldMousePosition[0]==-1): + oldMousePosition[0] = p[0] + oldMousePosition[1] = p[1] + + + #This creates a new node in the WF + if(keysPressed[110]==1 and debounceKey[110]==0 and "wfListThing" in directoriesNames[-1]): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + WF.addNode(node("some state")) + lucida.level(directoriesNames[-1],0); + + + + #This checks if we are moving a node around. + if(mousePress[0]==1 and debounceMouse[0]==1 and "wfListThing" in directoriesNames[-1] and "node:" in lucida.currentLevelName): + draggingNode = 1 + idList = lucida.currentLevelName.split("node:") + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeID = int(idList[1]) + draggingWF = getWFID + draggingNodeID = nodeID + + #Should be able to update now. + nodeData = WF.nodeList[nodeID] + xVector = ((float(p[0]-oldMousePosition[0]))/windowScreen.width) + yVector = ((float(p[1]-oldMousePosition[1]))/windowScreen.height) + + draggingX = xVector+nodeData.x + draggingY = yVector+nodeData.y + + xPlaceNew = xVector+nodeData.x-lucida.WFXOffset + yPlaceNew = yVector+nodeData.y-lucida.WFYOffset + lucida.updateObjectPosition(lucida.currentLevelName,xPlaceNew,yPlaceNew) + lucida.updateObjectPosition("wfListText:"+str(nodeID),xPlaceNew+textOffsetX,yPlaceNew+textOffsetY) + + linkCount = 0 + #This updates the lines at node. + for link in nodeData.linkList: + lucida.updateObjectPosition("IAmLine:" + str(nodeID) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount += 1 + + #But if for instance S0->S1, S0 is updating the line as it owns the node. But if S1 moves, it is not being updated as S1 does not own the node + #So must go through every single node, and find all toNodes and see if match + + fromNodeID = 0 + for nodeCheck in WF.nodeList: + + linkCount = 0; + for link in nodeCheck.linkList: + toNodeID = link.toNode + if(toNodeID==nodeID): # This means eg. S0->S1. We are moving S1. This means S0 has a node to S1, so the line from S0 needs to update. + #So update S0 to repoint to S1. + lucida.updateObjectPosition2("IAmLine:" + str(fromNodeID) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount+=1 + fromNodeID+=1 + + + windowScreen.blitScreen() + + + ## Update the state graph in DB. + if(draggingNode==1 and mousePress[0] == 0): + draggingNode=0 + WF = workflowList[draggingWF] + nodeData = WF.nodeList[draggingNodeID] + nodeData.x =draggingX + nodeData.y = draggingY + base64content = objToBase64(WF.nodeList) + db.update_workflow(workflowList[draggingWF].dbData['_id'], "stategraph", base64content); + + + + # With initial mouse point, now drag around + if(mousePress[0]==1 and debounceMouse[0]==1 and "wfListThing" in directoriesNames[-1] and "wfBG:" in lucida.currentLevelName): + draggingScreen = 1 + differencePoint = [(float(p[0]-oldMousePosition[0]))/windowScreen.width,(float((p[1]-oldMousePosition[1]))/windowScreen.height)] + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + + countServices = 0; + for nodeData in WF.nodeList: + draggingScreenX = -differencePoint[0] + draggingScreenY = -differencePoint[1] + + xPlaceNew = nodeData.x+differencePoint[0]-lucida.WFXOffset + yPlaceNew = nodeData.y+differencePoint[1]-lucida.WFYOffset + lucida.updateObjectPosition("node:"+str(countServices),xPlaceNew,yPlaceNew) + lucida.updateObjectPosition("wfListText:"+str(countServices),xPlaceNew+textOffsetX,yPlaceNew+textOffsetY) + + linkCount = 0 + for link in nodeData.linkList: + + lucida.updateObjectPosition("IAmLine:" + str(countServices) + "IAmLine:" + str(linkCount),xPlaceNew,yPlaceNew) + linkCount += 1 + + # S0->S1. So S0 gets hit. It has link to S1. So S0 updates its from. What is its to? Reference S1. + + linkCount = 0; + for link in nodeData.linkList: + toNodeID = link.toNode + nodeDataTo = WF.nodeList[toNodeID]; + xPlaceNew2 = nodeDataTo.x+differencePoint[0]-lucida.WFXOffset + yPlaceNew2 = nodeDataTo.y+differencePoint[1]-lucida.WFYOffset + + lucida.updateObjectPosition2("IAmLine:" + str(countServices) + "IAmLine:" + str(linkCount),xPlaceNew2,yPlaceNew2) + linkCount += 1 + + countServices += 1 + + + + + + windowScreen.blitScreen() + + + if(draggingScreen==1 and mousePress[0] == 0): + draggingScreen=0 + WF = workflowList[draggingScreenWF] + lucida.WFXOffset += draggingScreenX + lucida.WFYOffset += draggingScreenY + + if(mousePress[0]==1 and debounceMouse[0]==0): + lucida.clickLevel(p[0],p[1]) + + + #Modify a workflow logic + if(currentActiveNode!=-1 and "wfListThing" in directoriesNames[-1] and keysPressed[109]==1 and debounceKey[109]==0): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeThing = WF.nodeList[currentActiveNode] + writeFileForUseMod(nodeThing.code) + nodeThing.code = input_impl() #User code + workflowData = compileWorkflow(getWFID) + db.update_workflow(WF.dbData['_id'], "code",workflowData); + + + + ######## Lets check if I pressed on a line. + if(mousePress[0]==1 and currentActiveNode==-1 and debounceMouse[0]==0 and "wfListThing" in directoriesNames[-1] and "node:" not in lucida.currentLevelName): + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + #So go through every single line and see if within certain distance + # And to do that I must go through every single node. + countServices = 0; + floatP = [] + floatP.append(float(p[0])/windowScreen.width) + floatP.append(float(p[1])/windowScreen.height) + print(floatP) + for nodeData in WF.nodeList: + xStart = nodeData.x + yStart = nodeData.y; + + countServices+=1; + linkCount = 0; + for link in nodeData.linkList: + toNodeID = link.toNode + nodeDataTo = WF.nodeList[toNodeID]; + xEnd = nodeDataTo.x + yEnd = nodeDataTo.y + + #no division by 0 + if(xStart==xEnd): + xStart = 1; + m = (yStart-yEnd)/(xStart-xEnd); + b = yStart-m*xStart + d = abs((m*floatP[0])+(floatP[1]*-1) + b)/math.sqrt(m*m+1); + # This line is being pressed. + if(d<0.01 and keysPressed[109]==1 ): + writeFileForUseMod(link.code) + link.code = input_impl() + workflowData = compileWorkflow(getWFID) + db.update_workflow(WF.dbData['_id'], "code",workflowData); + linkCount+=1 + pass + + + # Unselect node, select node, and make link + if(mousePress[0]==1 and debounceMouse[0]==0 and "wfListThing" in directoriesNames[-1] and "node:" in lucida.currentLevelName): + idList = lucida.currentLevelName.split("node:") + getWFID = int(filter(str.isdigit, directoriesNames[-1])) + WF = workflowList[getWFID] + nodeID = int(idList[1]) + nodeData = WF.nodeList[nodeID] + + if(currentActiveNode==nodeID): # Click again to disable? + if(keysPressed[108]==1): + nodeDataOriginal = WF.nodeList[currentActiveNode] + isLinked = 0 + #Check for if already linked. If no link, link, if link, unlink. + for link in nodeDataOriginal.linkList: + if(link.toNode==nodeID): + isLinked = 1 + + #Link them + if(isLinked==0): + nodeDataOriginal.addLink("linkName",nodeID); + lucida.level(directoriesNames[-1],1); + pass + elif (isLinked==1): #TODO: Unlink + pass + + base64content = objToBase64(WF.nodeList) + db.update_workflow(workflowList[getWFID].dbData['_id'], "stategraph", base64content); + else: + print("Unselect node") + currentActiveNode = -1 + lucida.updateObjectImage(lucida.currentLevelName,nodeIMG) + nodeData.active = 0 + elif(currentActiveNode==-1): # No previous selected, so select + print("Select Node:", nodeID) + currentActiveNode = nodeID + lucida.updateObjectImage(lucida.currentLevelName,nodeActiveIMG) + nodeData.active = 1 + else: # There is a selected, and it is not the selected. So make link? + print("Make link", currentActiveNode, nodeID) + #Check if "l" is pressed when attempting to link two nodes. + nodeDataOriginal = WF.nodeList[currentActiveNode] + + #If "l" is pressed, AND not already linked, make link. + if(keysPressed[108]==1): + isLinked = 0 + #Check for if already linked. If no link, link, if link, unlink. + for link in nodeDataOriginal.linkList: + if(link.toNode==nodeID): + isLinked = 1 + + #Link them + if(isLinked==0): + nodeDataOriginal.addLink("linkName",nodeID); + lucida.level(directoriesNames[-1],1); + pass + elif (isLinked==1): #TODO: Unlink + pass + + base64content = objToBase64(WF.nodeList) + db.update_workflow(workflowList[getWFID].dbData['_id'], "stategraph", base64content); + + + + + + #If "l" is not pressed, this means selecting a new node. + if(keysPressed[108]==0): + + lucida.updateObjectImage("node:"+str(currentActiveNode),nodeIMG) + nodeDataOriginal.active = 0 + currentActiveNode = nodeID + lucida.updateObjectImage(lucida.currentLevelName,nodeActiveIMG) + nodeData.active = 1 + + + + + + + + + + + if(mousePress[0]==0): + oldMousePosition = [] + oldMousePosition.append(-1) + oldMousePosition.append(-1) + + + + + #Debounce the key stack + loop255 = 0; + while loop255!=255: + debounceKey[loop255] = keysPressed[loop255] + loop255+=1 + + #Debounce the mouse + debounceMouse[0] = mousePress[0] + + #Makes the program terminate if the x on the window is pressed + for event in pygame.event.get(): + if event.type == pygame.QUIT: + crashTheProgram() + pass + +pygame.quit() \ No newline at end of file diff --git a/tools/lucida-gui/gui_backend_cloud.py b/tools/lucida-gui/gui_backend_cloud.py new file mode 100644 index 000000000..76229c82e --- /dev/null +++ b/tools/lucida-gui/gui_backend_cloud.py @@ -0,0 +1,332 @@ +import os +import sys +import requests +import json +from pymongo import * +from bson.objectid import ObjectId + +url_pfx = 'http://127.0.0.1:3000' # url that the index page is hosted + +class MongoDB(object): + def __init__(self): + mongodb_addr = os.environ.get('MONGO_PORT_27017_TCP_ADDR') + if mongodb_addr: + self.db = MongoClient(mongodb_addr, 27017).lucida + else: + self.db = MongoClient().lucida + + def get_services(self): + dictReturn = [] + + url = url_pfx + '/api/v1/service' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['service_list'] + + return dictReturn + + def add_service(self): + """ + return code: + 0: success + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + # "location": location # location of service in local + } + + url = url_pfx + '/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_service(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: service name not found + 2: name already used + 3: acronym already used + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = url_pfx + '/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + elif error == 'Updated name already used': + return 2 + elif error == 'Updated acronym already used': + return 3 + else: + return -1 + + def delete_service(self, _id): + """ + return code: + 0: success + 1: service not exist + """ + + post = { + "option": "delete", + "_id": _id + } + + url = url_pfx + '/api/v1/service' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1 + else: + return -1 + + def get_workflows(self): + dictReturn = [] + + url = url_pfx + '/api/v1/workflow' + r = requests.get(url) + ret_JSON = r.json() + dictReturn = ret_JSON['workflow_list'] + + return dictReturn + + def add_workflow(self): + """ + return code: + 0: success + """ + + # list the attributes for the interface + post = { + "option": "add_empty" + } + + url = url_pfx + '/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['_id'] + else: + return -1, '' + + def update_workflow(self, _id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: workflow name not found + 2: updated name already used + """ + + post = { + "option": "update", + "_id": _id, + "op": op, + "value": value + } + + url = url_pfx + '/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + elif error == 'Updated name already used': + return 2 + else: + return -1 + + def delete_workflow(self, _id): + """ + return code: + 0: success + 1: workflow not exists + """ + + post = { + "option": "delete", + "_id": _id + } + + url = url_pfx + '/api/v1/workflow' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Workflow not exists': + return 1 + else: + return -1 + + def add_instance(self, _id): + """ + return code: + 0: success + 1: service not valid + """ + + # list the attributes for the interface + post = { + "option": "add_empty", + "_id": _id + } + + url = url_pfx + '/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0, ret_JSON['instance_id'] + else: + error = ret_JSON['error'] + if error == 'Service not exists': + return 1, '' + else: + return -1, '' + + def update_instance(self, _id, instance_id, op, value): + """ + op: field of what you want to update + value: update value for the field + return code: + 0: success + 1: instance not found + 2: host/port not valid + 3: host/port already used + """ + + post = { + "option": "update", + "_id": _id, + "instance_id": instance_id, + "op": op, + "value": value + } + + url = url_pfx + '/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + elif error == 'Host/port pair is not valid': + return 2 + elif error == 'Updated host/port has already been used': + return 3 + else: + return -1 + + def delete_instance(self, _id, instance_id): + """ + return code: + 0: success + 1: instance not exist + """ + + post = { + "option": "delete", + "_id": _id, + "instance_id": instance_id + } + + url = url_pfx + '/api/v1/instance' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + else: + return -1 + + def start_server(self, _id, instance_id): + post = { + "option": "start", + "_id": _id, + "instance_id": instance_id + } + + url = url_pfx + '/api/v1/blackbox' + headers = {'Content-type':'application/json', 'Accept': 'text/plain'} + r = requests.post(url, data=json.dumps(post), headers=headers) + ret_JSON = r.json() + ret_status = r.status_code + if ret_status == 200: + return 0 + else: + error = ret_JSON['error'] + if error == 'Instance not exists': + return 1 + else: + return -1 + +def validate_ip_port(s, p): + """ + Check if ip/port is valid with ipv4 + """ + + a = s.split('.') + if len(a) != 4: + return False + for x in a: + if not x.isdigit(): + return False + i = int(x) + if i < 0 or i > 255: + return False + if p < 0 or p > 65535: + return False + return True + +db = MongoDB() \ No newline at end of file diff --git a/tools/lucida-gui/images/alive.png b/tools/lucida-gui/images/alive.png new file mode 100644 index 000000000..35e40192c Binary files /dev/null and b/tools/lucida-gui/images/alive.png differ diff --git a/tools/lucida-gui/images/bg.png b/tools/lucida-gui/images/bg.png new file mode 100644 index 000000000..55e414e5d Binary files /dev/null and b/tools/lucida-gui/images/bg.png differ diff --git a/tools/lucida-gui/images/dead.png b/tools/lucida-gui/images/dead.png new file mode 100644 index 000000000..1cc68c236 Binary files /dev/null and b/tools/lucida-gui/images/dead.png differ diff --git a/tools/lucida-gui/images/grayBG.png b/tools/lucida-gui/images/grayBG.png new file mode 100644 index 000000000..87f6a2eaf Binary files /dev/null and b/tools/lucida-gui/images/grayBG.png differ diff --git a/tools/lucida-gui/images/node.png b/tools/lucida-gui/images/node.png new file mode 100644 index 000000000..020cbef31 Binary files /dev/null and b/tools/lucida-gui/images/node.png differ diff --git a/tools/lucida-gui/images/nodeActive.png b/tools/lucida-gui/images/nodeActive.png new file mode 100644 index 000000000..365639db7 Binary files /dev/null and b/tools/lucida-gui/images/nodeActive.png differ diff --git a/tools/lucida-gui/images/serviceListButton.png b/tools/lucida-gui/images/serviceListButton.png new file mode 100644 index 000000000..06a082209 Binary files /dev/null and b/tools/lucida-gui/images/serviceListButton.png differ diff --git a/tools/lucida-gui/images/yellowBG.png b/tools/lucida-gui/images/yellowBG.png new file mode 100644 index 000000000..8c3b1b7cd Binary files /dev/null and b/tools/lucida-gui/images/yellowBG.png differ diff --git a/tools/lucida-gui/textrect.py b/tools/lucida-gui/textrect.py new file mode 100755 index 000000000..264b18af2 --- /dev/null +++ b/tools/lucida-gui/textrect.py @@ -0,0 +1,115 @@ +#! /usr/bin/env python + +class TextRectException: + def __init__(self, message = None): + self.message = message + def __str__(self): + return self.message + +def render_textrect(string, font, rect, text_color, background_color, justification=0): + """Returns a surface containing the passed text string, reformatted + to fit within the given rect, word-wrapping as necessary. The text + will be anti-aliased. + + Takes the following arguments: + + string - the text you wish to render. \n begins a new line. + font - a Font object + rect - a rectstyle giving the size of the surface requested. + text_color - a three-byte tuple of the rgb value of the + text color. ex (0, 0, 0) = BLACK + background_color - a three-byte tuple of the rgb value of the surface. + justification - 0 (default) left-justified + 1 horizontally centered + 2 right-justified + + Returns the following values: + + Success - a surface object with the text rendered onto it. + Failure - raises a TextRectException if the text won't fit onto the surface. + """ + + import pygame + + final_lines = [] + + requested_lines = string.splitlines() + + # Create a series of lines that will fit on the provided + # rectangle. + + for requested_line in requested_lines: + if font.size(requested_line)[0] > rect.width: + words = list(requested_line) + # if any of our words are too long to fit, return. + for word in words: + if font.size(word)[0] >= rect.width: + raise TextRectException, "The word " + word + " is too long to fit in the rect passed." + # Start a new line + accumulated_line = "" + for word in words: + test_line = accumulated_line + word + "" + # Build the line while the words fit. + if font.size(test_line)[0] < rect.width: + accumulated_line = test_line + else: + final_lines.append(accumulated_line) + accumulated_line = word + " " + final_lines.append(accumulated_line) + else: + final_lines.append(requested_line) + + # Let's try to write the text out on the surface. + + #surface = pygame.Surface(rect.size) + surface = pygame.Surface([640,480], pygame.SRCALPHA, 32) + surface = surface.convert_alpha() + + # surface.fill(background_color) + + accumulated_height = 0 + for line in final_lines: + if accumulated_height + font.size(line)[1] >= rect.height: + raise TextRectException, "Once word-wrapped, the text string was too tall to fit in the rect." + if line != "": + tempsurface = font.render(line, 1, text_color) + if justification == 0: + surface.blit(tempsurface, (0, accumulated_height)) + elif justification == 1: + surface.blit(tempsurface, ((rect.width - tempsurface.get_width()) / 2, accumulated_height)) + elif justification == 2: + surface.blit(tempsurface, (rect.width - tempsurface.get_width(), accumulated_height)) + else: + raise TextRectException, "Invalid justification argument: " + str(justification) + accumulated_height += font.size(line)[1] + + return surface + + +if __name__ == '__main__': + import pygame + import pygame.font + from pygame.locals import * + + pygame.init() + + display = pygame.display.set_mode((400, 400)) + + my_font = pygame.font.Font(None, 22) + + my_string = "Hi there! I'm a nice bit of wordwrapped text. Won't you be my friend? Honestly, wordwrapping is easy, with David's fancy new render_textrect() function.\nThis is a new line.\n\nThis is another one.\n\n\nAnother line, you lucky dog." + + my_rect = pygame.Rect((40, 40, 300, 300)) + + rendered_text = render_textrect(my_string, my_font, my_rect, (216, 216, 216), (48, 48, 48), 1) + + if rendered_text: + display.blit(rendered_text, my_rect.topleft) + + pygame.display.update() + + while not pygame.event.wait().type in (QUIT, KEYDOWN): + pass + + + diff --git a/tools/python_requirements.txt b/tools/python_requirements.txt index a5b4c34c8..c055dc2b1 100644 --- a/tools/python_requirements.txt +++ b/tools/python_requirements.txt @@ -44,7 +44,7 @@ singledispatch==3.4.0.3 six==1.10.0 supervisor==3.3.1 texttable==0.8.8 -thrift==0.10.0 +thrift==0.9.3 tornado==4.4.3 torthrift==0.1.9 urllib3==1.20 diff --git a/tools/start_all_tmux.sh b/tools/start_all_tmux.sh deleted file mode 100755 index 723627df1..000000000 --- a/tools/start_all_tmux.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# This script is used to start all of the microservices -# for the Lucida project by creating a tmux window for each -# server in the background. -# Example calls: -# $./start_all_tmux.sh secure -# $./start_all_tmux.sh -# To attach to the tmux session -# use the following command: -# $tmux a -t lucida - -# source the port properties -. ../lucida/config.properties - -if [ "$1" == "test" ]; then - SESSION_NAME="lu-test" -else - SESSION_NAME="lucida" -fi - -# Check if session already exists -tmux has-session -t ${SESSION_NAME} -if [ $? -eq 0 ]; then - echo "Session ${SESSION_NAME} already exists." - exit 0; -elif [ -n "$TMUX" ]; then - echo "Already in a tmux session" - exit 0; -else - echo "Session ${SESSION_NAME} does not exit. Creating a ${SESSION_NAME} session." -fi - -# Check to see if we should run on http/ws (non-secure) or https/wss (secure) -if [ "$1" == "secure" ]; then - echo "Enabling secure host" - # Getting the host IP address - export ASR_ADDR_PORT="wss://$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'):$CMD_PORT" - export SECURE_HOST=true - - # Generate self-signed certificates - cd $(pwd)/../lucida/commandcenter/ - chmod +x gen_certs.sh - ./gen_certs.sh - cd $(pwd)/../../tools -else - echo "Enabling non-secure host" - export ASR_ADDR_PORT="ws://localhost:$CMD_PORT" -fi - -declare -a commandcenter=("CMD" "$(pwd)/../lucida/commandcenter/") -declare -a questionanswering=("QA" "$(pwd)/../lucida/questionanswering/OpenEphyra/") -declare -a imagematching=("IMM" "$(pwd)/../lucida/imagematching/opencv_imm/") -declare -a calendar=("CA" "$(pwd)/../lucida/calendar/") -declare -a speechrecognition=("ASR" "$(pwd)/../lucida/speechrecognition/kaldi_gstreamer_asr/") -declare -a imageclassification=("IMC" "$(pwd)/../lucida/djinntonic/imc/") -declare -a digitrecognition=("DIG" "$(pwd)/../lucida/djinntonic/dig/") -declare -a facerecognition=("FACE" "$(pwd)/../lucida/djinntonic/face") -declare -a weather=("WE" "$(pwd)/../lucida/weather") -declare -a botframework=("BFI" "$(pwd)/../lucida/botframework-interface") -declare -a musicservice=("MS" "$(pwd)/../lucida/musicservice") - -if [ "$1" == "test" ]; then - declare -a services=( - ) -else - declare -a services=( - commandcenter - speechrecognition) -fi - -services+=( - questionanswering - imagematching - calendar - imageclassification - digitrecognition - facerecognition - weather - botframework - musicservice) - -# Create the session -tmux new-session -s ${SESSION_NAME} -d - -# Create the service windows -TMUX_WIN=0 -for i in "${services[@]}" -do - NAME=$i[0] - SERV_PATH=$i[1] - if [ $TMUX_WIN == 0 ]; then - tmux rename-window -t ${SESSION_NAME}:$TMUX_WIN ${!NAME} - else - tmux new-window -n ${!NAME} -t ${SESSION_NAME} - fi - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "cd ${!SERV_PATH}" C-m - if [ "$1" == "test" ]; then - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "make start_test" C-m - else - tmux send-keys -t ${SESSION_NAME}:$TMUX_WIN "make start_server" C-m - fi - ((TMUX_WIN++)) -done - -# Start out on the first window when we attach -tmux select-window -t ${SESSION_NAME}:0 - diff --git a/tools/template.sh b/tools/template.sh new file mode 100644 index 000000000..eee344d83 --- /dev/null +++ b/tools/template.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +echo "=============================================" +echo " Lucida Micro Service Registry Tools" +echo " made in May 31, 2017" +echo "=============================================" +echo "" + +create_folder () { + cd ../lucida + cp -rf template/$1 . + mv $1 $2 + cd $2 + find ./ -type f -exec sed -i "s/template/$2/g" {} \; + find ./ -type f -exec sed -i "s/TPL/$2/g" {} \; + find . -depth -name '*template*' -execdir rename "s/template/$2/g" {} \; + find . -depth -name '*TPL*' -execdir rename "s/TPL/$2/g" {} \; + cd .. + echo "[Info] Template folder is ready!" +} + +OP="" +if [ "$1" = "add" ]; then + OP="add" +elif [ "$1" = "delete" ]; then + OP="delete" +else + echo "### Specify what you want to do (add or delete)" + printf "### Enter you want to do: " + read OP + echo "" +fi + +if [ "$OP" = "add" ]; then + NAME_VALID=1 + while [ $NAME_VALID -ne 0 ]; do + echo "### Specify your service name (e.g. musicservice)." + printf "### Enter your service name: " + read NAME + if [ "$NAME" = "" ]; then + echo "[Error] Service name cannot be empty! Please try another one!" + else + NAME_VALID=0 + fi + done + + echo "" + echo "### Specify the programming language you want to you in your programming. If C++/Java/Python, then template will be provided." + printf "### Enter the programming language: " + read LAN + LAN="$(tr [A-Z] [a-z] <<< "$LAN")" + if [ "$LAN" = "c++" ]; then + LAN="cpp" + fi + + if [ "$LAN" = "cpp" -o "$LAN" = "java" -o "$LAN" = "python" ]; then + # do copy template folder of cpp to lucida + if [ -d $NAME ]; then + echo "[Error] service already exists!" + exit 1 + else + create_folder $LAN $NAME + fi + else + # create an empty folder + if [ -d $NAME ]; then + echo "[Error] service already exists!" + exit 1 + else + mkdir $NAME + echo "[Info] Template folder is ready!" + fi + fi +fi \ No newline at end of file