From e239f6ec475c4bbd15c5bdb08afe68b0e58585d6 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 20 May 2016 12:38:21 -0400 Subject: [PATCH 1/3] no more grep in logtail, actually fix duplicates --- scripts/logfetch/entrypoint.py | 1 - scripts/logfetch/s3_logs.py | 8 +++++++- scripts/logfetch/tail.py | 19 +------------------ 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/scripts/logfetch/entrypoint.py b/scripts/logfetch/entrypoint.py index 0c53150e3d..b4238ce129 100644 --- a/scripts/logfetch/entrypoint.py +++ b/scripts/logfetch/entrypoint.py @@ -324,7 +324,6 @@ def tail(): parser.add_argument("-r", "--request-id", dest="requestId", help="RequestId of request to fetch logs for (can be a glob)") parser.add_argument("-d", "--deploy-id", dest="deployId", help="DeployId of tasks to fetch logs for (can be a glob)") parser.add_argument("-u", "--singularity-uri-base", dest="singularity_uri_base", help="The base for singularity (eg. http://localhost:8080/singularity/v1)") - parser.add_argument("-g", "--grep", dest="grep", help="String to grep for") parser.add_argument("-l", "--logfile", dest="logfile", help="Logfile path/name to tail (ie 'logs/access.log')") parser.add_argument("-V", "--verbose", dest="verbose", help="more verbose output", action='store_true') parser.add_argument("--silent", dest="silent", help="No stderr (progress, file names, etc) output", action='store_true') diff --git a/scripts/logfetch/s3_logs.py b/scripts/logfetch/s3_logs.py index bdb426cd96..faaaa135dc 100644 --- a/scripts/logfetch/s3_logs.py +++ b/scripts/logfetch/s3_logs.py @@ -68,7 +68,13 @@ def logs_for_all_requests(args): for request in logfetch_base.all_requests(args): s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params) logs = logs + s3_logs if s3_logs else logs - return [dict(t) for t in set(tuple(l.items()) for l in logs)] # remove any duplicates + found_logs = [] + keys = [] + for log in logs: + if not log['key'] in keys: + found_logs.append(log) + keys.append(log['key']) + return found_logs def s3_task_logs_uri(args, idString): return S3LOGS_URI_FORMAT.format(logfetch_base.base_uri(args), TASK_FORMAT.format(idString)) diff --git a/scripts/logfetch/tail.py b/scripts/logfetch/tail.py index 8a29ef9315..7ce1a50034 100644 --- a/scripts/logfetch/tail.py +++ b/scripts/logfetch/tail.py @@ -104,28 +104,11 @@ def fetch_new_log_data(self, uri, path, offset, args, task): response = requests.get(uri, params=params, headers=args.headers).json() prefix = '({0}) =>\n'.format(task) if args.verbose else '' if len(response['data'].encode('utf-8')) > 0: - if args.grep: - filename = '{0}/.grep{1}'.format(args.dest, self.Task) - self.create_grep_file(args, filename, response['data']) - output = os.popen(grep_command(args, filename)).read() - sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), output)) - self.remove_grep_file(filename) - else: - sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), response['data'].encode('utf-8'))) + sys.stdout.write('{0}{1}'.format(colored(prefix, 'cyan'), response['data'].encode('utf-8'))) return offset + len(response['data'].encode('utf-8')) else: return offset - def create_grep_file(self, args, filename, content): - grep_file = open(filename, 'wb') - grep_file.write(content.encode('utf-8')) - grep_file.close() - - - def remove_grep_file(self, grep_file): - if os.path.isfile(grep_file): - os.remove(grep_file) - def show_available_files(self, args, task): sys.stderr.write(colored('Available files (-l arguments):\n', 'cyan')) try: From a298a05b8f5bc513bdeb2c0fdb5034e467e01f81 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 20 May 2016 12:46:04 -0400 Subject: [PATCH 2/3] bump version --- scripts/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup.py b/scripts/setup.py index 12a7e9cc77..01c3614a23 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -11,7 +11,7 @@ setup( name='singularity-logfetch', - version='0.24.3', + version='0.25.0', description='Singularity log fetching and searching', author="HubSpot", author_email='singularity-users@googlegroups.com', From 0675c445c191dd96c777b69b50f1dbed49669ede Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 20 May 2016 12:48:35 -0400 Subject: [PATCH 3/3] update docs --- scripts/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index c89a36346d..ff909dad18 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -98,7 +98,7 @@ For example, to tail the `service.log` file for all tasks for a request named `M - The path for the log file is relative to the base path for that task's sandbox. For example, to tail a file in `(sandbox path)/logs/access.log`, the argument to -l would be `logs/access.log` -You can also provide the `-g` option which will provide the grep string to the singularity API and search the results. This can be a string to match on or a full grep command as above. +As of `0.25.0` a grep option is no longer supported in `logtail`. it more efficient/usable, and therefore recommended, to pipe output to grep for this type of functionality. ##Options |Flags|Description|Default| @@ -109,7 +109,6 @@ You can also provide the `-g` option which will provide the grep string to the s |-r , --request-id|Request Id to fetch logs for|| |-d , --deploy-id|Deploy Id to fetch logs for (Must also specify requestId when using this option)|| |-u, --singularity-uri-base|Base url for singularity (e.g. `localhost:8080/singularity/v2/api`)|Must be set!| -|-g, --grep|Grep string or full command for searching output|| |-l, --logfile|Log file path to tail (ie logs/access.log)|Must be set!| |-V, --verbose|Extra output about the task id associated with logs in the output|False| |--silent|No output except for log content, overrides -V|false|