From 14517240256a9a1a05c1e60a79914b410e22c33e Mon Sep 17 00:00:00 2001 From: scherniavsky Date: Thu, 29 Nov 2018 17:49:49 +0100 Subject: [PATCH 1/5] probe for master readiness and number of connected slaves --- example/simple.py | 6 +- example/simple_post.py | 5 +- src/app.py | 161 ++++++++++++++++++++---------------- src/tests/test_bootstrap.py | 42 ++++++---- 4 files changed, 123 insertions(+), 91 deletions(-) diff --git a/example/simple.py b/example/simple.py index e1c4bc4..bd2d07e 100644 --- a/example/simple.py +++ b/example/simple.py @@ -1,14 +1,12 @@ from locust import HttpLocust from locust import TaskSet from locust import task -from locust.web import app +# For HTML reporting +from locust.web import app from src import report - -# For reporting app.add_url_rule('/htmlreport', 'htmlreport', report.download_report) - class SimpleBehavior(TaskSet): @task diff --git a/example/simple_post.py b/example/simple_post.py index faad60e..3b41e47 100644 --- a/example/simple_post.py +++ b/example/simple_post.py @@ -5,11 +5,10 @@ from locust import HttpLocust from locust import TaskSet from locust import task -from locust.web import app +# For HTML reporting +from locust.web import app from src import report - -# For reporting app.add_url_rule('/htmlreport', 'htmlreport', report.download_report) # Read json file diff --git a/src/app.py b/src/app.py index aa47832..eca9a53 100755 --- a/src/app.py +++ b/src/app.py @@ -5,11 +5,11 @@ import multiprocessing import os +import requests import signal import subprocess import sys - -import requests +import time processes = [] logging.basicConfig() @@ -50,6 +50,9 @@ def bootstrap(_return=0): logger.info('target host: {target}, locust file: {file}, master: {master}, multiplier: {multiplier}'.format( target=target_host, file=locust_file, master=master_host, multiplier=multiplier)) + + wait_for_master() + for _ in range(multiplier): logger.info('Started Process') s = subprocess.Popen([ @@ -72,8 +75,8 @@ def bootstrap(_return=0): os.getenv('SLAVE_MUL', multiprocessing.cpu_count())) # Default time duration to wait all slaves to be connected is 1 minutes / 60 seconds slaves_check_timeout = float(os.getenv('SLAVES_CHECK_TIMEOUT', 60)) - # Default sleep time interval is 10 seconds - slaves_check_interval = float(os.getenv('SLAVES_CHECK_INTERVAL', 5)) + # Default sleep time interval is 3 seconds + slaves_check_interval = float(os.getenv('SLAVES_CHECK_INTERVAL', 3)) users = int(get_or_raise('USERS')) hatch_rate = int(get_or_raise('HATCH_RATE')) duration = int(get_or_raise('DURATION')) @@ -81,72 +84,67 @@ def bootstrap(_return=0): 'master url: {url}, users: {users}, hatch_rate: {rate}, duration: {duration}'.format( url=master_url, users=users, rate=hatch_rate, duration=duration)) - for _ in range(0, 5): - import time - time.sleep(3) - - res = requests.get(url=master_url) - if res.ok: - timeout = time.time() + slaves_check_timeout - connected_slaves = 0 - while time.time() < timeout: - try: - logger.info('Checking if all slave(s) are connected.') - stats_url = '/'.join([master_url, 'stats/requests']) - res = requests.get(url=stats_url) - connected_slaves = res.json().get('slave_count') - - if connected_slaves >= total_slaves: - break - else: - logger.info('Currently connected slaves: {con}'.format(con=connected_slaves)) - time.sleep(slaves_check_interval) - except ValueError as v_err: - logger.error(v_err.message) - else: - logger.warning('Connected slaves:{con} != defined slaves:{dfn}'.format( - con=connected_slaves, dfn=total_slaves)) - - logger.info('All slaves are succesfully connected! ' - 'Start load test automatically for {duration} seconds.'.format(duration=duration)) - payload = {'locust_count': users, 'hatch_rate': hatch_rate} - res = requests.post(url=master_url + '/swarm', data=payload) - - if res.ok: - time.sleep(duration) - requests.get(url=master_url + '/stop') - logger.info('Load test is stopped.') - - time.sleep(4) - - logging.info('Creating report folder.') - report_path = os.path.join(os.getcwd(), 'reports') - if not os.path.exists(report_path): - os.makedirs(report_path) - - logger.info('Creating reports...') - for _url in ['requests', 'distribution']: - res = requests.get(url=master_url + '/stats/' + _url + '/csv') - with open(os.path.join(report_path, _url + '.csv'), "wb") as file: - file.write(res.content) - - if _url == 'distribution': - continue - res = requests.get(url=master_url + '/stats/' + _url) - with open(os.path.join(report_path, _url + '.json'), "wb") as file: - file.write(res.content) - - res = requests.get(url=master_url + '/htmlreport') - with open(os.path.join(report_path, 'reports.html'), "wb") as file: - file.write(res.content) - logger.info('Reports have been successfully created.') + wait_for_master() + + timeout = time.time() + slaves_check_timeout + connected_slaves = 0 + while time.time() < timeout: + try: + logger.info('Checking if all slave(s) are connected.') + stats_url = '/'.join([master_url, 'stats/requests']) + res = requests.get(url=stats_url) + connected_slaves = res.json().get('slave_count') + + if connected_slaves >= total_slaves: + break else: - logger.error('Locust cannot be started. Please check logs!') + logger.info('Currently connected slaves: {con}'.format(con=connected_slaves)) + + except ValueError as v_err: + logger.error(v_err.message) + + time.sleep(slaves_check_interval) + else: + logger.error('Connected slaves:{con} < defined slaves:{dfn}'.format( + con=connected_slaves, dfn=total_slaves)) + raise RuntimeError('The Slaves did not connect in time.') + + logger.info('All slaves are succesfully connected! ' + 'Start load test automatically for {duration} seconds.'.format(duration=duration)) + payload = {'locust_count': users, 'hatch_rate': hatch_rate} + res = requests.post(url=master_url + '/swarm', data=payload) + + if res.ok: + time.sleep(duration) + requests.get(url=master_url + '/stop') + logger.info('Load test is stopped.') + + time.sleep(4) + + logging.info('Creating reports folder.') + report_path = os.path.join(os.getcwd(), 'reports') + if not os.path.exists(report_path): + os.makedirs(report_path) + + logger.info('Creating reports...') + for _url in ['requests', 'distribution']: + res = requests.get(url=master_url + '/stats/' + _url + '/csv') + with open(os.path.join(report_path, _url + '.csv'), "wb") as file: + file.write(res.content) + + if _url == 'distribution': + continue + res = requests.get(url=master_url + '/stats/' + _url) + with open(os.path.join(report_path, _url + '.json'), "wb") as file: + file.write(res.content) + + res = requests.get(url=master_url + '/htmlreport') + with open(os.path.join(report_path, 'reports.html'), "wb") as file: + file.write(res.content) + logger.info('Reports have been successfully created.') + else: + logger.error('Locust cannot be started. Please check logs!') - break - else: - logger.error('Attempt: {attempt}. Locust master might not ready yet.' - 'Status code: {status}'.format(attempt=_, status=res.status_code)) except ValueError as v_err: logger.error(v_err) @@ -164,7 +162,7 @@ def bootstrap(_return=0): sys.exit(0) else: - raise RuntimeError('Invalid ROLE value. Valid Options: master, slave, controller.') + raise RuntimeError('Invalid ROLE value. Valid Options: master, slave, controller, standalone.') if _return: return @@ -352,6 +350,31 @@ def kill(signal, frame): s.kill(s) +def wait_for_master(): + master_host = get_or_raise('MASTER_HOST') + master_url = 'http://{master}:8089'.format(master=master_host) + + # Wait for the master to come online during SLAVES_CHECK_TIMEOUT + master_check_timeout = float(os.getenv('MASTER_CHECK_TIMEOUT', 60)) + # Default sleep time interval is 3 seconds + master_check_interval = float(os.getenv('MASTER_CHECK_INTERVAL', 3)) + + timeout = time.time() + master_check_timeout + cnt = 1 + while time.time() < timeout: + try: + res = requests.get(url=master_url, timeout=1) + if res.ok: + logger.info('Locust master is ready.') + return + except requests.exceptions.ConnectionError: + pass + logger.warning('Attempt: {attempt}. Locust master is not ready yet.'.format(attempt=cnt)) + cnt += 1 + time.sleep(master_check_interval) + raise RuntimeError('The master did not start in time.') + + if __name__ == '__main__': logger.setLevel(logging.INFO) logger.info('Started main') diff --git a/src/tests/test_bootstrap.py b/src/tests/test_bootstrap.py index 07fbd34..7c9dfb7 100644 --- a/src/tests/test_bootstrap.py +++ b/src/tests/test_bootstrap.py @@ -25,12 +25,34 @@ def test_valid_master(self, popen, mocked_send_usage): self.assertTrue(mocked_send_usage.called) @mock.patch('subprocess.Popen') - def test_valid_slave(self, mocked_popen): + def test_master_not_ready_in_time(self, popen): + os.environ['ROLE'] = 'slave' + os.environ['TARGET_HOST'] = 'https://test.com' + os.environ['MASTER_HOST'] = '127.0.0.1' + os.environ['SLAVE_MUL'] = '1' + os.environ['MASTER_CHECK_TIMEOUT'] = '0.3' + os.environ['MASTER_CHECK_INTERVAL'] = '0.1' + + with mock.patch('src.app.get_locust_file') as file: + with self.assertRaises(RuntimeError) as e: + bootstrap() + self.assertFalse(file.called) + self.assertFalse(popen.called) + self.assertEqual('The Master did not start in time.', str(e.exception)) + + + @mock.patch('subprocess.Popen') + @requests_mock.Mocker() + def test_valid_slave(self, mocked_popen, mocked_request): os.environ['ROLE'] = 'slave' os.environ['TARGET_HOST'] = 'https://test.com' os.environ['MASTER_HOST'] = '127.0.0.1' os.environ['SLAVE_MUL'] = '3' + os.environ['SLAVES_CHECK_TIMEOUT'] = '0.3' + os.environ['SLAVES_CHECK_INTERVAL'] = '0.1' + MASTER_URL = 'http://127.0.0.1:8089' + mocked_request.get(url=MASTER_URL, text='ok') with mock.patch('src.app.get_locust_file') as file: bootstrap() self.assertTrue(file.called) @@ -81,14 +103,9 @@ def test_valid_controller_automatic(self, mocked_timeout, mocked_dir, mocked_ope for endpoint in ['stop', 'stats/requests/csv', 'stats/distribution/csv', 'htmlreport']: mocked_request.get(url='/'.join([MASTER_URL, endpoint]), text='ok') - self.assertFalse(mocked_timeout.called) - self.assertFalse(mocked_request.called) - #self.assertFalse(mocked_dir.called) - self.assertFalse(mocked_open.called) bootstrap() self.assertTrue(mocked_timeout.called) self.assertTrue(mocked_request.called) - #self.assertTrue(mocked_dir.called) self.assertTrue(mocked_open.called) @mock.patch('time.sleep') @@ -113,15 +130,10 @@ def test_slaves_not_fully_connected(self, mocked_timeout, mocked_dir, mocked_ope for endpoint in ['stop', 'stats/requests/csv', 'stats/distribution/csv', 'htmlreport']: mocked_request.get(url='/'.join([MASTER_URL, endpoint]), text='ok') - self.assertFalse(mocked_timeout.called) - self.assertFalse(mocked_request.called) - #self.assertFalse(mocked_dir.called) - self.assertFalse(mocked_open.called) - bootstrap() - self.assertTrue(mocked_timeout.called) - self.assertTrue(mocked_request.called) - #self.assertTrue(mocked_dir.called) - self.assertTrue(mocked_open.called) + with self.assertRaises(RuntimeError): + bootstrap() + self.assertFalse(mocked_request.called) + self.assertFalse(mocked_open.called) def test_invalid_role(self): os.environ['ROLE'] = 'unknown' From a3e503b3ae70acab3a2d45c7aef21d0e80417fe6 Mon Sep 17 00:00:00 2001 From: scherniavsky Date: Fri, 30 Nov 2018 07:24:17 +0100 Subject: [PATCH 2/5] typo --- src/tests/test_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_bootstrap.py b/src/tests/test_bootstrap.py index 7c9dfb7..c20bf4f 100644 --- a/src/tests/test_bootstrap.py +++ b/src/tests/test_bootstrap.py @@ -38,7 +38,7 @@ def test_master_not_ready_in_time(self, popen): bootstrap() self.assertFalse(file.called) self.assertFalse(popen.called) - self.assertEqual('The Master did not start in time.', str(e.exception)) + self.assertEqual('The master did not start in time.', str(e.exception)) @mock.patch('subprocess.Popen') From ca11cd8a77ef4f183a6c45a61151e35b3e812634 Mon Sep 17 00:00:00 2001 From: scherniavsky Date: Fri, 30 Nov 2018 12:38:52 +0100 Subject: [PATCH 3/5] documented TOTAL_SLAVES usage --- README.md | 26 ++++++++++++++++---------- src/.#app.py | 1 + src/app.py | 4 ++-- 3 files changed, 19 insertions(+), 12 deletions(-) create mode 120000 src/.#app.py diff --git a/README.md b/README.md index 2c7c866..2d9ccdb 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,22 @@ The purpose of this project is to provide a ready and easy-to-use version of [lo Architecture ------------ -Docker-Locust consist of 3 different roles: - -- Master: Instance that will run Locust's web interface where you start and stop the load test and see live statistics. -- Slave: Instance that will simulate users and attack the target url based on user parameters. -- Controller: Instance that will be run for automatic mode and will download the HTML report at the end of load test. - -This architecture support following type of deployment: - -- single container (standalone mode): If user have only one single machine. -- multiple containers (normal mode): If user have more than one machine and want to create bigger load. This type of deployment might be used in docker-swarm or kubernetes case. An example for deployment in different containers can be seen in [docker-compose]. +Docker-Locust container can be started in 4 different roles: + +- `master`: Runs Locust's web interface where you start and stop the load test and see live statistics. +- `slave`: Simulates users and attacks the target url based on user parameters. +- `controller`: Orchestrates Master in automatic mode and downloads reports when the test is over. +- `standalone`: Automatically starts the above components locally. + +There are 2 supported run types: +- Manual: when a user manually starts and stops a test via a Locust Master UI. +- Automatic: when a test is started by the Controller and runs for a specified time interval. F + +And there are 2 ways to deploy it: +- Local deployment (using `standalone` mode or [docker-compose]): when a singe machine can generate enough traffic. +- Distributed deployment: when multiple machines are required to generate a bigger load. This type of deployment might be used in AWS or Kubernetes. +An example deployment with different container roles can be found in [docker-compose]. +Using Automatic mode together with Distributed deployment type requires `TOTAL_SLAVES` variable to be set on the `controller` side. Key advantages -------------- diff --git a/src/.#app.py b/src/.#app.py new file mode 120000 index 0000000..e60cf99 --- /dev/null +++ b/src/.#app.py @@ -0,0 +1 @@ +scherniavsky@shanti.4948 \ No newline at end of file diff --git a/src/app.py b/src/app.py index eca9a53..1935f48 100755 --- a/src/app.py +++ b/src/app.py @@ -107,7 +107,7 @@ def bootstrap(_return=0): else: logger.error('Connected slaves:{con} < defined slaves:{dfn}'.format( con=connected_slaves, dfn=total_slaves)) - raise RuntimeError('The Slaves did not connect in time.') + raise RuntimeError('The slaves did not connect in time.') logger.info('All slaves are succesfully connected! ' 'Start load test automatically for {duration} seconds.'.format(duration=duration)) @@ -143,7 +143,7 @@ def bootstrap(_return=0): file.write(res.content) logger.info('Reports have been successfully created.') else: - logger.error('Locust cannot be started. Please check logs!') + logger.error('Locust cannot be started. Please check the logs!') except ValueError as v_err: logger.error(v_err) From e0360b6f20623c2456b1a07032a27fb49b295e5f Mon Sep 17 00:00:00 2001 From: scherniavsky Date: Fri, 30 Nov 2018 15:38:25 +0100 Subject: [PATCH 4/5] typo fix --- README.md | 2 +- src/.#app.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 src/.#app.py diff --git a/README.md b/README.md index 2d9ccdb..88499de 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Docker-Locust container can be started in 4 different roles: There are 2 supported run types: - Manual: when a user manually starts and stops a test via a Locust Master UI. -- Automatic: when a test is started by the Controller and runs for a specified time interval. F +- Automatic: when a test is started by the Controller and runs for a specified time interval. And there are 2 ways to deploy it: - Local deployment (using `standalone` mode or [docker-compose]): when a singe machine can generate enough traffic. diff --git a/src/.#app.py b/src/.#app.py deleted file mode 120000 index e60cf99..0000000 --- a/src/.#app.py +++ /dev/null @@ -1 +0,0 @@ -scherniavsky@shanti.4948 \ No newline at end of file From 12aa623b57d603570912393cfee0e778db069463 Mon Sep 17 00:00:00 2001 From: scherniavsky Date: Fri, 30 Nov 2018 16:03:14 +0100 Subject: [PATCH 5/5] typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 88499de..a0bd99d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ And there are 2 ways to deploy it: - Local deployment (using `standalone` mode or [docker-compose]): when a singe machine can generate enough traffic. - Distributed deployment: when multiple machines are required to generate a bigger load. This type of deployment might be used in AWS or Kubernetes. An example deployment with different container roles can be found in [docker-compose]. -Using Automatic mode together with Distributed deployment type requires `TOTAL_SLAVES` variable to be set on the `controller` side. +Using Automatic mode together with Distributed deployment requires the `TOTAL_SLAVES` variable to be set on the `controller` side. Key advantages --------------