From fecfeb980ff5d20fa4aaeba4c8ceda9217aa76be Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 18 Aug 2021 18:03:36 -0700 Subject: [PATCH 1/4] Feat : Handle path to sync single file --- plugins/modules/s3_sync.py | 86 ++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/plugins/modules/s3_sync.py b/plugins/modules/s3_sync.py index 1e7d01680f1..b5b9687f19b 100644 --- a/plugins/modules/s3_sync.py +++ b/plugins/modules/s3_sync.py @@ -322,41 +322,57 @@ def calculate_multipart_etag(source_path, chunk_size=DEFAULT_CHUNK_SIZE): def gather_files(fileroot, include=None, exclude=None): ret = [] - for (dirpath, dirnames, filenames) in os.walk(fileroot): - for fn in filenames: - fullpath = os.path.join(dirpath, fn) - # include/exclude - if include: - found = False - for x in include.split(','): - if fnmatch.fnmatch(fn, x): - found = True - if not found: - # not on the include list, so we don't want it. - continue - - if exclude: - found = False - for x in exclude.split(','): - if fnmatch.fnmatch(fn, x): - found = True - if found: - # skip it, even if previously included. - continue - - chopped_path = os.path.relpath(fullpath, start=fileroot) - fstat = os.stat(fullpath) - f_size = fstat[osstat.ST_SIZE] - f_modified_epoch = fstat[osstat.ST_MTIME] - ret.append({ - 'fullpath': fullpath, - 'chopped_path': chopped_path, - 'modified_epoch': f_modified_epoch, - 'bytes': f_size, - }) - # dirpath = path *to* the directory - # dirnames = subdirs *in* our directory - # filenames + + if os.path.isfile(fileroot): + fullpath = fileroot + fstat = os.stat(fullpath) + path_array = fileroot.split('/') + chopped_path = path_array[-1] + f_size = fstat[osstat.ST_SIZE] + f_modified_epoch = fstat[osstat.ST_MTIME] + ret.append({ + 'fullpath': fullpath, + 'chopped_path': chopped_path, + 'modified_epoch': f_modified_epoch, + 'bytes': f_size, + }) + + else: + for (dirpath, dirnames, filenames) in os.walk(fileroot): + for fn in filenames: + fullpath = os.path.join(dirpath, fn) + # include/exclude + if include: + found = False + for x in include.split(','): + if fnmatch.fnmatch(fn, x): + found = True + if not found: + # not on the include list, so we don't want it. + continue + + if exclude: + found = False + for x in exclude.split(','): + if fnmatch.fnmatch(fn, x): + found = True + if found: + # skip it, even if previously included. + continue + + chopped_path = os.path.relpath(fullpath, start=fileroot) + fstat = os.stat(fullpath) + f_size = fstat[osstat.ST_SIZE] + f_modified_epoch = fstat[osstat.ST_MTIME] + ret.append({ + 'fullpath': fullpath, + 'chopped_path': chopped_path, + 'modified_epoch': f_modified_epoch, + 'bytes': f_size, + }) + # dirpath = path *to* the directory + # dirnames = subdirs *in* our directory + # filenames return ret From 59253c5c32db69338a7818be6c67b43ff19067d7 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 18 Aug 2021 22:28:30 -0700 Subject: [PATCH 2/4] Adding integration test --- .../targets/s3_sync/defaults/main.yml | 1 + .../targets/s3_sync/tasks/main.yml | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/integration/targets/s3_sync/defaults/main.yml b/tests/integration/targets/s3_sync/defaults/main.yml index c46e675d53c..b48f03b7849 100644 --- a/tests/integration/targets/s3_sync/defaults/main.yml +++ b/tests/integration/targets/s3_sync/defaults/main.yml @@ -1,2 +1,3 @@ test_bucket: "{{ tiny_prefix }}-testbucket-ansible" test_bucket_2: "{{ tiny_prefix }}-testbucket-ansible-2" +test_bucket_3: "{{ tiny_prefix }}-testbucket-ansible-3" diff --git a/tests/integration/targets/s3_sync/tasks/main.yml b/tests/integration/targets/s3_sync/tasks/main.yml index eb72de27251..c2dc113d4ed 100644 --- a/tests/integration/targets/s3_sync/tasks/main.yml +++ b/tests/integration/targets/s3_sync/tasks/main.yml @@ -77,6 +77,7 @@ - assert: that: - output is changed + # ============================================================ - name: Sync files already present @@ -99,6 +100,27 @@ that: - output is not changed + # ============================================================ + - name: Create a third s3_bucket + s3_bucket: + name: "{{ test_bucket_3 }}" + state: present + register: output + + - assert: + that: + - output.changed + - output.name == "{{ test_bucket_3 }}" + - not output.requester_pays + + - name: Sync individual file with remote bucket + s3_sync: + bucket: "{{ test_bucket_3 }}" + file_root: "{{ output_dir }}/s3_sync/test1.txt" + register: output + - assert: + that: + - output is changed # ============================================================ # DOCUMENTATION EXAMPLES @@ -132,3 +154,4 @@ with_items: - "{{ test_bucket }}" - "{{ test_bucket_2 }}" + - "{{ test_bucket_3 }}" From 25bc37d210f9c451e4aa9c94fecdeee0fe8ac0e9 Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 18 Aug 2021 22:43:30 -0700 Subject: [PATCH 3/4] Added example --- plugins/modules/s3_sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/modules/s3_sync.py b/plugins/modules/s3_sync.py index b5b9687f19b..c9021c3dbf9 100644 --- a/plugins/modules/s3_sync.py +++ b/plugins/modules/s3_sync.py @@ -148,6 +148,11 @@ file_root: roles/s3/files/ storage_class: GLACIER +- name: basic individual file upload + community.aws.s3_sync: + bucket: tedder + file_root: roles/s3/files/file_name + - name: all the options community.aws.s3_sync: bucket: tedder From 6295318bf6836b147e26487372b7514f7e8e7afa Mon Sep 17 00:00:00 2001 From: Mandar Kulkarni Date: Wed, 18 Aug 2021 22:52:42 -0700 Subject: [PATCH 4/4] Add changelogs fragment --- changelogs/fragments/692-s3_sync-individual-file-path.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/692-s3_sync-individual-file-path.yml diff --git a/changelogs/fragments/692-s3_sync-individual-file-path.yml b/changelogs/fragments/692-s3_sync-individual-file-path.yml new file mode 100644 index 00000000000..77b7ac80522 --- /dev/null +++ b/changelogs/fragments/692-s3_sync-individual-file-path.yml @@ -0,0 +1,2 @@ +bugfixes: +- s3_sync - fix handling individual file path to upload a individual file to s3 bucket (https://github.com/ansible-collections/community.aws/pull/692).