diff --git a/apps/CardinalBanditsPureExploration/tests/test_api.py b/apps/CardinalBanditsPureExploration/tests/test_api.py index d31f93df..987d204c 100644 --- a/apps/CardinalBanditsPureExploration/tests/test_api.py +++ b/apps/CardinalBanditsPureExploration/tests/test_api.py @@ -14,12 +14,11 @@ try: import next.apps.test_utils as test_utils except: - sys.path.append('../../../next/apps') + sys.path.append('../../next/apps') import test_utils -def test_api(assert_200=True, num_arms=5, - num_experiments=1, num_clients=10, total_pulls=10): +def test_api(assert_200=True, num_arms=5, num_experiments=1, num_clients=10, total_pulls=10): app_id = 'CardinalBanditsPureExploration' true_means = numpy.array(range(num_arms)[::-1])/float(num_arms) diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 057dab2d..f3184de8 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -1,5 +1,5 @@ -# Each piece within the #'s could theoretically be its own machine. But it could also operate just on one big machine. -# Right now things are linked through docker. If we dploy with something else that doesn't respect links, thing will have to be refactored. +# Each piece within the #'s could theoretically be its own machine. But it could also operate just on one big machine. +# Right now things are linked through docker. If we dploy with something else that doesn't respect links, thing will have to be refactored. # Things could be factored as Minions being indivdiual machines and everything else goes on a master machine. ######## CADVISOR ######### @@ -16,37 +16,47 @@ cadvisor: ######## MINION MACHINE ######### -# minionredis: -# image: redis -# command: redis-server /database/next/database/CacheStore/redis.conf -# volumes: -# - next_backend:/database + +minionredis: + image: redis + command: redis-server /next_backend/next/database_client/CacheStore/redis.conf + volumes: + - /Users/jain/Repositories/Development/NEXT-dev:/next_backend + expose: + - "6379" minionworker: - build: /home/ubuntu/NEXT/next/base_docker_image + build: /Users/jain/Repositories/Development/NEXT-dev/next/base_docker_image command: ./next/broker/next_worker_startup.sh volumes: - - /home/ubuntu/NEXT:/next_backend + - /Users/jain/Repositories/Development/NEXT-dev:/next_backend working_dir: /next_backend links: - - mongodb:MONGODB + - mongodb:MONGODB - rabbitmq:RABBIT - rabbitmqredis:RABBITREDIS + - minionredis:MINIONREDIS environment: - PYTHONUNBUFFERED=TRUE - CELERY_SYNC_WORKER_COUNT=6 - - CELERY_ASYNC_WORKER_COUNT=2 + - CELERY_ASYNC_WORKER_COUNT=4 - CELERY_THREADS_PER_ASYNC_WORKER=1 + # Should be max(1,int(.25*master_num_cpus)) + - CELERY_ASYNC_WORKER_PREFETCH=4 + - CELERY_DASHBOARD_WORKER_COUNT=1 + - CELERY_THREADS_PER_DASHBOARD_WORKER=2 + - CELERY_DASHBOARD_WORKER_PREFETCH=1 + - GIT_HASH={{GIT_HASH}} - PYTHONPATH=:/next_backend #################################### ######### API ######### nextbackenddocker: - build: /home/ubuntu/NEXT/next/base_docker_image + build: /Users/jain/Repositories/Development/NEXT-dev/next/base_docker_image command: gunicorn --worker-class=gevent -R --log-level=debug --logger-class=simple --reload -b 0.0.0.0:8000 -w 1 next.api.api:app volumes: - - /home/ubuntu/NEXT:/next_backend + - /Users/jain/Repositories/Development/NEXT-dev:/next_backend working_dir: /next_backend ports: - "0.0.0.0:8000:8000" # gunicorn @@ -55,41 +65,43 @@ nextbackenddocker: - rabbitmq:RABBIT - rabbitmqredis:RABBITREDIS - mongodb:MONGODB + - minionworker:MINIONWORKER environment: - PYTHONUNBUFFERED=TRUE - - PYTHONPATH=:/next_backend + - PYTHONPATH=:/next_backend # global url for dashboards on backend - NEXT_BACKEND_GLOBAL_HOST - NEXT_BACKEND_GLOBAL_PORT - TERM=xterm + - CELERY_ON=False #################################### ######### PERSISTENT STORE ######### mongodb: - build: /home/ubuntu/NEXT/next/base_docker_image + build: /Users/jain/Repositories/Development/NEXT-dev/next/base_docker_image command: /usr/bin/mongod --smallfiles --rest volumes: - - /home/ubuntu/NEXT:/next_backend + - /Users/jain/Repositories/Development/NEXT-dev:/next_backend - /dump - /data/db working_dir: /next_backend expose: - - "27017" + - "27017" ports: - "0.0.0.0:28017:28017" # mongo http monitor mongodbbackup: - build: /home/ubuntu/NEXT/next/base_docker_image + build: /Users/jain/Repositories/Development/NEXT-dev/next/base_docker_image command: python ./next/database/daemon_database_backup.py volumes: - - /home/ubuntu/NEXT:/next_backend + - /Users/jain/Repositories/Development/NEXT-dev:/next_backend volumes_from: - mongodb working_dir: /next_backend links: - - mongodb:MONGODB + - mongodb:MONGODB environment: - NEXT_BACKEND_GLOBAL_HOST - PYTHONPATH=:/next_backend @@ -109,18 +121,6 @@ rabbitmq: rabbitmqredis: image: redis -rabbitmqmonitor: - build: /home/ubuntu/NEXT/next/base_docker_image - command: python ./next/broker/cluster_monitor.py - volumes: - - /home/ubuntu/NEXT:/next_backend - working_dir: /next_backend - links: - - rabbitmq:RABBIT - - rabbitmqredis:RABBITREDIS - environment: - - PYTHONPATH=:/next_backend - - PYTHONUNBUFFERED=TRUE ########################### diff --git a/next/api/resources/experiment.py b/next/api/resources/experiment.py index 217800df..197cfd78 100644 --- a/next/api/resources/experiment.py +++ b/next/api/resources/experiment.py @@ -54,26 +54,22 @@ def get(self, exp_uid): return attach_meta(experiment, meta_success), 200 def post(self): - utils.debug_print('experiment:58',request.data) post_parser = exp_parser.copy() post_parser.add_argument('app_id', type=str, required=True) post_parser.add_argument('args', type=dict, required=True) # Validate args with post_parser args_data = post_parser.parse_args() - utils.debug_print(args_data) app_id = args_data['app_id'] - utils.debug_print(app_id) # Create and set exp_uid exp_uid = '%030x' % random.randrange(16**30) # Args from dict to json type args_json = json.dumps(args_data) - print('experiment:69') # Execute initExp through the broker response_json,didSucceed,message = broker.applyAsync(app_id, exp_uid, 'initExp', json.dumps(args_data)) - + if not didSucceed: return attach_meta({}, meta_error['InitExpError'], backend_error=message), 400 diff --git a/next/dashboard/dashboard.py b/next/dashboard/dashboard.py index 031aecd4..4a4525c6 100644 --- a/next/dashboard/dashboard.py +++ b/next/dashboard/dashboard.py @@ -59,13 +59,11 @@ def experiment_list(): print e pass - host_url = 'http://{}:{}'.format(constants.NEXT_BACKEND_GLOBAL_HOST, - constants.NEXT_BACKEND_GLOBAL_PORT) if constants.SITE_KEY: - dashboard_url='{}/dashboard/{}'.format(host_url, constants.SITE_KEY) + dashboard_url='/dashboard/{}'.format(host_url, constants.SITE_KEY) else: - dashboard_url='{}/dashboard'.format(host_url) - + dashboard_url='/dashboard'.format(host_url) + return render_template('experiment_list.html', dashboard_url=dashboard_url, experiments = reversed(experiments)) @@ -75,7 +73,7 @@ def get_stats(): args_dict = request.json exp_uid = args_dict['exp_uid'] app_id = rm.get_app_id(exp_uid) - + response_json,didSucceed,message = broker.dashboardAsync(app_id,exp_uid,args_dict) response_dict = json.loads(response_json,parse_float=lambda o:round(float(o),4)) response_json = json.dumps(response_dict) @@ -124,13 +122,13 @@ def experiment_dashboard(exp_uid, app_id): 'alg_label_clean':'_'.join(alg['alg_label'].split())} for alg in alg_label_list] - host_url = 'http://{}:{}'.format(constants.NEXT_BACKEND_GLOBAL_HOST, - constants.NEXT_BACKEND_GLOBAL_PORT) + host_url = ''# 'http://{}:{}'.format(constants.NEXT_BACKEND_GLOBAL_HOST, + # constants.NEXT_BACKEND_GLOBAL_PORT) if constants.SITE_KEY: - dashboard_url='{}/dashboard/{}'.format(host_url, constants.SITE_KEY) + dashboard_url='/dashboard/{}'.format(constants.SITE_KEY) else: - dashboard_url='{}/dashboard'.format(host_url) - + dashboard_url='/dashboard' + env = Environment(loader=ChoiceLoader([PackageLoader('apps.{}'.format(app_id), 'dashboard'), PackageLoader('next.dashboard', @@ -139,16 +137,15 @@ def experiment_dashboard(exp_uid, app_id): return template.render(app_id=app_id, exp_uid=exp_uid, alg_list=alg_list, - host_url=host_url, dashboard_url=dashboard_url, - exceptions_present=exceptions_present(exp_uid, host_url), + exceptions_present=False,#exceptions_present(exp_uid), url_for=url_for, simple_flag=int(simple_flag), force_recompute=int(force_recompute)) -def exceptions_present(exp_uid, host_url): - url = '{}/api/experiment/{}/logs/APP-EXCEPTION'.format(host_url, exp_uid) +def exceptions_present(exp_uid): + url = '/api/experiment/{}/logs/APP-EXCEPTION'.format(exp_uid) r = requests.get(url) logs = yaml.load(r.content)['log_data'] return True if len(logs) > 0 else False diff --git a/next/dashboard/templates/basic.html b/next/dashboard/templates/basic.html index 84cea90a..f56b3436 100644 --- a/next/dashboard/templates/basic.html +++ b/next/dashboard/templates/basic.html @@ -55,7 +55,7 @@ 'exp_uid': "{{ exp_uid }}", 'args':{ 'stat_id' : 'api_activity_histogram', - 'params':{}, + 'params':{}, 'force_recompute': {{ force_recompute }} } }),