diff --git a/tablechop b/tablechop index 89ef574..0ed5052 100755 --- a/tablechop +++ b/tablechop @@ -51,8 +51,16 @@ def clean_backups(args, log): bucket = get_bucket(args) + if args.prefix: + key_prefix = '%s%s%s%s' % (args.prefix, args.keyname_separator, args.name, args.path) + else: + if args.keyname_separator == '/' and args.path.startswith('/'): + key_prefix = '%s%s' % (args.name, args.path) + else: + key_prefix = '%s%s%s' % (args.name, args.keyname_separator, args.path) + try: - key_list = bucket.list("%s:%s" % (args.name, args.path)) + key_list = bucket.list(key_prefix) except boto.exception.BotoServerError, e: log.error('Problem getting keys from S3 bucket: %s', e) sys.exit(1) @@ -125,7 +133,7 @@ def get_file_list(args, bucket, index_file): if len(index.values()) != 1: raise SystemError('-listdir.json file should have a single key/value pair!') dirname = index.keys()[0] - return ["%s:%s" % (args.name, os.path.join(dirname, filename)) for filename in index.values()[0]] + return ["%s%s%s" % (args.name, args.keyname_separator, os.path.join(dirname, filename)) for filename in index.values()[0]] def main(log): @@ -171,6 +179,12 @@ def main(log): default='us-east-1', choices=[region.name for region in boto.s3.regions()], help='AWS region to connect to.') + parser.add_argument('--prefix', + default='', + help='Set a string prefix for uploaded files in S3') + parser.add_argument('--keyname-separator', + default=':', + help='Separator for the keyname between name and path.') parser.add_argument( '-n', '--name', diff --git a/tableslurp b/tableslurp index 77a2c9c..3e44e50 100755 --- a/tableslurp +++ b/tableslurp @@ -364,6 +364,10 @@ def main(): default='us-east-1', choices=[region.name for region in boto.s3.regions()], help='AWS region to connect to.') + ap.add_argument('--prefix', default='', + help='Set a string prefix for uploaded files in S3') + ap.add_argument('--keyname-separator', default=':', + help='Separator for the keyname between name and path.') ap.add_argument('-p', '--preserve', default=False, action='store_true', help='Preserve the permissions (if they exist) from the source. ' 'This overrides -o and -g') @@ -398,8 +402,13 @@ def main(): ap.add_argument('target', nargs=1, help='Path in the local FS where files should be downloaded to') args = ap.parse_args() - - prefix='%s:%s' % (args.name, args.origin[0]) + if args.prefix: + prefix = '%s%s%s%s' % (args.prefix, args.keyname_separator, args.name, args.origin[0]) + else: + if args.keyname_separator == '/' and args.origin[0].startswith('/'): + prefix='%s%s' % (args.name, args.origin[0]) + else: + prefix='%s%s%s' % (args.name, args.keyname_separator, args.origin[0]) bucket=get_bucket(args.aws_region, args.aws_key, args.aws_secret, args.token, args.bucket[0]) log.info('Building fileset') if args.recursive: @@ -409,7 +418,14 @@ def main(): prefix=os.path.join(prefix, table), fileset=fileset) dh.run() elif args.commitlogs: - clog_prefix = '%s:%s' % (args.name, args.commitlogs) + if args.prefix: + clog_prefix = '%s%s%s%s' % (args.prefix, args.keyname_separator, args.name, args.commitlogs) + else: + if args.keyname_separator == '/' and args.commitlogs.startswith('/'): + clog_prefix = '%s%s' % (args.name, args.commitlogs) + else: + clog_prefix = '%s%s%s' % (args.name, args.keyname_separator, args.commitlogs) + fileset = build_clog_fileset(bucket, prefix, clog_prefix) dh = DownloadHandler(args, target=os.path.join(args.target[0]), prefix=clog_prefix, fileset=fileset)