From d8b024ef9e380ff7ee6f23c75f6b7e7b73a16a44 Mon Sep 17 00:00:00 2001 From: naveenpaul1 Date: Wed, 9 Oct 2024 10:03:55 +0530 Subject: [PATCH] NSFS | Improve list objects performance on top of NS FS Signed-off-by: naveenpaul1 --- config.js | 3 + src/api/object_api.js | 6 + src/endpoint/s3/ops/s3_get_bucket.js | 1 + src/native/fs/fs_napi.cpp | 2 +- src/sdk/keymarker_fs.js | 50 ++ src/sdk/namespace_fs.js | 319 ++++++++--- src/test/system_tests/test_utils.js | 22 + .../jest_tests/test_list_object.test.js | 175 ------ .../test_unsort_list_object.test.js | 541 ++++++++++++++++++ src/test/unit_tests/test_namespace_fs.js | 29 +- src/test/unit_tests/test_namespace_fs_mpu.js | 21 +- 11 files changed, 857 insertions(+), 312 deletions(-) create mode 100644 src/sdk/keymarker_fs.js delete mode 100644 src/test/unit_tests/jest_tests/test_list_object.test.js create mode 100644 src/test/unit_tests/jest_tests/test_unsort_list_object.test.js diff --git a/config.js b/config.js index d121ff9e84..3ac75b8adb 100644 --- a/config.js +++ b/config.js @@ -843,6 +843,9 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000; // of `manage_nsfs glacier restore` config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000; +// enable/disable unsorted listing application level +config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; + // NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies // when NooBaa should allow running glacier expiry process // NOTE: This will also be in the same timezone as specified in diff --git a/src/api/object_api.js b/src/api/object_api.js index 25b5beae87..7842813a9d 100644 --- a/src/api/object_api.js +++ b/src/api/object_api.js @@ -723,6 +723,9 @@ module.exports = { limit: { type: 'integer' }, + list_type: { + type: 'string', + }, } }, reply: { @@ -777,6 +780,9 @@ module.exports = { limit: { type: 'integer' }, + list_type: { + type: 'string', + }, } }, reply: { diff --git a/src/endpoint/s3/ops/s3_get_bucket.js b/src/endpoint/s3/ops/s3_get_bucket.js index 8c9a5cc797..bde3e7c61a 100644 --- a/src/endpoint/s3/ops/s3_get_bucket.js +++ b/src/endpoint/s3/ops/s3_get_bucket.js @@ -41,6 +41,7 @@ async function get_bucket(req) { bucket: req.params.bucket, prefix: req.query.prefix, delimiter: req.query.delimiter, + list_type: list_type, limit: Math.min(max_keys_received, 1000), key_marker: list_type === '2' ? (s3_utils.cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker, diff --git a/src/native/fs/fs_napi.cpp b/src/native/fs/fs_napi.cpp index b5195eb5b8..7222ca03a1 100644 --- a/src/native/fs/fs_napi.cpp +++ b/src/native/fs/fs_napi.cpp @@ -2105,7 +2105,7 @@ struct TellDir : public FSWrapWorker } virtual void OnOK() { - DBG0("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res)); + DBG1("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res)); Napi::Env env = Env(); auto res = Napi::BigInt::New(env, _tell_res); _deferred.Resolve(res); diff --git a/src/sdk/keymarker_fs.js b/src/sdk/keymarker_fs.js new file mode 100644 index 0000000000..8eb8fc1e9f --- /dev/null +++ b/src/sdk/keymarker_fs.js @@ -0,0 +1,50 @@ +/* Copyright (C) 2020 NooBaa */ +'use strict'; + +class KeyMarkerFS { + constructor({ marker, marker_pos, pre_dir, pre_dir_pos }, is_unsorted = false) { + this.marker = marker; + this.marker_pos = marker_pos.toString(); + this.pre_dir = pre_dir; + this.pre_dir_pos = pre_dir_pos; + this.key_marker_value = marker; + this.current_dir = ''; + this.is_unsorted = is_unsorted; + this.last_pre_dir = ''; + this.last_pre_dir_position = ''; + if (is_unsorted) { + this.current_dir = pre_dir.length > 0 && marker.includes('/') ? + marker.substring(0, marker.lastIndexOf('/') + 1) : ''; + } + } + + async update_key_marker(marker, marker_pos) { + this.marker = marker; + this.marker_pos = marker_pos; + this.key_marker_value = marker; + } + + async get_previour_dir_length() { + return this.pre_dir.length; + } + + async get_previour_dir_info() { + return { + pre_dir_path: this.pre_dir.pop(), + pre_dir_position: this.pre_dir_pos.pop(), + }; + } + + async add_previour_dir(pre_dir, pre_dir_pos) { + this.pre_dir.push(pre_dir); + this.pre_dir_pos.push(pre_dir_pos.toString()); + } + + async update_last_previour_dir(last_pre_dir, last_pre_dir_position) { + this.last_pre_dir = last_pre_dir; + this.last_pre_dir_position = last_pre_dir_position; + } +} + +// EXPORTS +module.exports = KeyMarkerFS; diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index fabe042ed3..c7df813acc 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -27,6 +27,7 @@ const { S3Error } = require('../endpoint/s3/s3_errors'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; const { PersistentLogger } = require('../util/persistent_logger'); const { GlacierBackend } = require('./nsfs_glacier_backend/backend'); +const KeyMarkerFS = require('./keymarker_fs'); const multi_buffer_pool = new buffer_utils.MultiSizeBuffersPool({ sorted_buf_sizes: [ @@ -593,6 +594,7 @@ class NamespaceFS { * prefix?: string, * delimiter?: string, * key_marker?: string, + * list_type?: string, * limit?: number, * }} ListParams */ @@ -611,6 +613,7 @@ class NamespaceFS { * prefix?: string, * delimiter?: string, * key_marker?: string, + * list_type?: string, * version_id_marker?: string, * limit?: number, * }} ListVersionsParams @@ -633,8 +636,8 @@ class NamespaceFS { prefix = '', version_id_marker = '', key_marker = '', + list_type = '', } = params; - if (delimiter && delimiter !== '/') { throw new Error('NamespaceFS: Invalid delimiter ' + delimiter); } @@ -645,7 +648,19 @@ class NamespaceFS { if (!limit) return { is_truncated: false, objects: [], common_prefixes: [] }; let is_truncated = false; - + let skip_list = false; + let keymarker; + if (typeof(key_marker) === 'object') { + keymarker = new KeyMarkerFS(key_marker, true); + } else { + keymarker = new KeyMarkerFS({ + marker: key_marker, + marker_pos: '', + pre_dir: [], + pre_dir_pos: [], + }); + } + dbg.log1('list object bucket :', bucket, ' key_marker :', keymarker.key_marker_value, 'list_type : ', list_type); /** * @typedef {{ * key: string, @@ -677,8 +692,8 @@ class NamespaceFS { // dbg.log0(`prefix dir does not match so no keys in this dir can apply: dir_key=${dir_key} prefix_dir=${prefix_dir}`); return; } - const marker_dir = key_marker.slice(0, dir_key.length); - const marker_ent = key_marker.slice(dir_key.length); + const marker_dir = keymarker.key_marker_value.slice(0, dir_key.length); + const marker_ent = keymarker.key_marker_value.slice(dir_key.length); // marker is after dir so no keys in this dir can apply if (dir_key < marker_dir) { // dbg.log0(`marker is after dir so no keys in this dir can apply: dir_key=${dir_key} marker_dir=${marker_dir}`); @@ -694,7 +709,7 @@ class NamespaceFS { * common_prefix: boolean * }} */ - const insert_entry_to_results_arr = async r => { + const insert_entry_to_results_arr = async r => { let pos; // Since versions are arranged next to latest object in the latest first order, // no need to find the sorted last index. Push the ".versions/#VERSION_OBJECT" as @@ -705,7 +720,6 @@ class NamespaceFS { } else { pos = results.length; } - if (pos >= limit) { is_truncated = true; return; // not added @@ -713,6 +727,9 @@ class NamespaceFS { if (!delimiter && r.common_prefix) { await process_dir(r.key); } else { + if (keymarker.key_marker_value === r.key) { + return; + } if (pos < results.length) { results.splice(pos, 0, r); } else { @@ -723,37 +740,130 @@ class NamespaceFS { is_truncated = true; } } + }; + + /** + * @typedef {{ + * key: string, + * common_prefix: boolean + * }} + */ + const insert_unsort_entry_to_results_arr = async r => { + // Since versions are arranged next to latest object in the latest first order, + // no need to find the sorted last index. Push the ".versions/#VERSION_OBJECT" as + // they are in order + const pos = results.length; + if (pos >= limit) { + if (keymarker.last_pre_dir) keymarker.add_previour_dir(keymarker.last_pre_dir, keymarker.last_pre_dir_position); + is_truncated = true; + return; // not added + } + if (!delimiter && r.common_prefix) { + if (r.marker_pos && !keymarker.pre_dir.includes(r.pre_dir)) { + keymarker.add_previour_dir(r.pre_dir, r.marker_pos); + } + await process_dir(r.key); + } else { + if (keymarker.key_marker_value === r.key) { + return; + } + results.push(r); + if (results.length > limit) { + results.length = limit; + is_truncated = true; + } + keymarker.update_last_previour_dir('', ''); + } + }; + + const push_dir_entries = async (marker_index, sorted_entries) => { + if (marker_index) { + const prev_dir = sorted_entries[marker_index - 1]; + const prev_dir_name = prev_dir.name; + if (marker_curr.startsWith(prev_dir_name) && dir_key !== prev_dir.name) { + if (!delimiter) { + const isDir = await is_directory_or_symlink_to_directory( + prev_dir, fs_context, path.join(dir_path, prev_dir_name, '/')); + if (isDir) { + await process_dir(path.join(dir_key, prev_dir_name, '/')); + } + } + } + } }; /** * @param {fs.Dirent} ent + * @param {string} pos */ - const process_entry = async ent => { - // dbg.log0('process_entry', dir_key, ent.name); + const process_entry = async (ent, pos = '') => { if ((!ent.name.startsWith(prefix_ent) || - ent.name < marker_curr || + (pos === '' && ent.name < marker_curr.split('/')[0]) || ent.name === this.get_bucket_tmpdir_name() || ent.name === config.NSFS_FOLDER_OBJECT_NAME) || this._is_hidden_version_path(ent.name)) { return; } const isDir = await is_directory_or_symlink_to_directory(ent, fs_context, path.join(dir_path, ent.name)); - let r; if (list_versions && _is_version_or_null_in_file_name(ent.name)) { r = { key: this._get_version_entry_key(dir_key, ent), common_prefix: isDir, - is_latest: false + is_latest: false, + marker_pos: pos, + pre_dir: dir_key ? dir_key : '/', }; } else { r = { key: this._get_entry_key(dir_key, ent, isDir), common_prefix: isDir, - is_latest: true + is_latest: true, + marker_pos: pos, + pre_dir: dir_key ? dir_key : '/', }; } - await insert_entry_to_results_arr(r); + if (config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED && list_type === '2') { + await insert_unsort_entry_to_results_arr(r); + } else { + await insert_entry_to_results_arr(r); + } + }; + + /** + * Process unsorted list objects + */ + const process_unsort_entry = async () => { + for (;;) { + if (is_truncated) break; + const dir_entry = await dir_handle.read(fs_context); + // After listing the last item from sub dir, check for parent dirs and parent directory position + // and go back to that position. + if (!dir_entry) { + // Skip item listing in bucket root path when list flow coming from sub dir to bucket root dir, + // if do not skip items in bucket root path will list two times, + // first in normal flow, and second when return back from sub dir. + if (this.bucket_path + '/' === dir_path) { + skip_list = true; + } + // After iterating the last element in subdir flow will go back to the parent folder, + // to avoid listing items again from start use previous dir path and position from + // previous_dirs and previous_dir_positions arrays respectively. + if (keymarker.pre_dir.length > 0) { + keymarker.last_pre_dir = keymarker.pre_dir.pop(); + keymarker.last_pre_dir_position = keymarker.pre_dir_pos.pop(); + // Next dir process will use the previous dir path and position to iterate from + // the previously left parentt dir position. + keymarker.update_key_marker(keymarker.last_pre_dir, keymarker.last_pre_dir_position); + await process_dir(keymarker.last_pre_dir); + } + break; + } + if ((dir_entry.name === config.NSFS_FOLDER_OBJECT_NAME && dir_key === marker_dir) || skip_list) { + continue; + } + await process_entry(dir_entry, params.key_marker && !keymarker.is_unsorted ? '' : dir_entry.off); + } }; if (!(await this.check_access(fs_context, dir_path))) return; @@ -774,12 +884,16 @@ class NamespaceFS { // insert dir object to objects list if its key is lexicographicly bigger than the key marker && // no delimiter OR prefix is the current directory entry const is_dir_content = cached_dir.stat.xattr && cached_dir.stat.xattr[XATTR_DIR_CONTENT]; - if (is_dir_content && dir_key > key_marker && (!delimiter || dir_key === prefix)) { + if (is_dir_content && dir_key > keymarker.key_marker_value && (!delimiter || dir_key === prefix)) { const r = { key: dir_key, common_prefix: false }; - await insert_entry_to_results_arr(r); + if (config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED && list_type === '2') { + await insert_unsort_entry_to_results_arr(r); + } else { + await insert_entry_to_results_arr(r); + } } - if (cached_dir.sorted_entries) { + if (!config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED && cached_dir.sorted_entries) { const sorted_entries = cached_dir.sorted_entries; let marker_index; // Two ways followed here to find the index. @@ -807,19 +921,7 @@ class NamespaceFS { // handling a scenario in which key_marker points to an object inside a directory // since there can be entries inside the directory that will need to be pushed // to results array - if (marker_index) { - const prev_dir = sorted_entries[marker_index - 1]; - const prev_dir_name = prev_dir.name; - if (marker_curr.startsWith(prev_dir_name) && dir_key !== prev_dir.name) { - if (!delimiter) { - const isDir = await is_directory_or_symlink_to_directory( - prev_dir, fs_context, path.join(dir_path, prev_dir_name, '/')); - if (isDir) { - await process_dir(path.join(dir_key, prev_dir_name, '/')); - } - } - } - } + await push_dir_entries(marker_index, sorted_entries); for (let i = marker_index; i < sorted_entries.length; ++i) { const ent = sorted_entries[i]; // when entry is NSFS_FOLDER_OBJECT_NAME=.folder file, @@ -837,14 +939,25 @@ class NamespaceFS { // for large dirs we cannot keep all entries in memory // so we have to stream the entries one by one while filtering only the needed ones. try { - dbg.warn('NamespaceFS: open dir streaming', dir_path, 'size', cached_dir.stat.size); - dir_handle = await nb_native().fs.opendir(fs_context, dir_path); //, { bufferSize: 128 }); - for (;;) { - const dir_entry = await dir_handle.read(fs_context); - if (!dir_entry) break; - await process_entry(dir_entry); - // since we dir entries streaming order is not sorted, - // we have to keep scanning all the keys before we can stop. + if (list_type === '2') { + // For unsorted listing dir position is used to when pagination split the items. + dbg.warn('NamespaceFS: open unsorted dir streaming', dir_path, 'size', cached_dir.stat.size, 'key_marker', keymarker); + dir_handle = await nb_native().fs.opendir(fs_context, dir_path); //, { bufferSize: 128 }); + if (keymarker.marker_pos) { + await dir_handle.seekdir(fs_context, BigInt(keymarker.marker_pos)); + keymarker.marker_pos = undefined; + } + await process_unsort_entry(); + } else { + dbg.warn('NamespaceFS: open dir streaming', dir_path, 'size', cached_dir.stat.size); + dir_handle = await nb_native().fs.opendir(fs_context, dir_path); //, { bufferSize: 128 }); + for (;;) { + const dir_entry = await dir_handle.read(fs_context); + if (!dir_entry) break; + await process_entry(dir_entry); + // since we dir entries streaming order is not sorted, + // we have to keep scanning all the keys before we can stop. + } } await dir_handle.close(fs_context); dir_handle = null; @@ -861,22 +974,9 @@ class NamespaceFS { } }; - let previous_key; - /** - * delete markers are always in the .versions folder, so we need to have special case to determine - * if they are delete markers. since the result list is ordered by latest entries first, the first - * entry of every key is the latest - * TODO need different way to check for isLatest in case of unordered list object versions - * @param {Object} obj_info - */ - const set_latest_delete_marker = obj_info => { - if (obj_info.delete_marker && previous_key !== obj_info.key) { - obj_info.is_latest = true; - } - }; - const prefix_dir_key = prefix.slice(0, prefix.lastIndexOf('/') + 1); - await process_dir(prefix_dir_key); + // current_dir added for unsorted listing + await process_dir(prefix_dir_key + keymarker.current_dir); await Promise.all(results.map(async r => { if (r.common_prefix) return; const entry_path = path.join(this.bucket_path, r.key); @@ -884,44 +984,85 @@ class NamespaceFS { const use_lstat = !(await this._is_path_in_bucket_boundaries(fs_context, entry_path)); r.stat = await nb_native().fs.stat(fs_context, entry_path, { use_lstat }); })); - const res = { - objects: [], - common_prefixes: [], - is_truncated, - next_marker: undefined, - next_version_id_marker: undefined, - }; - for (const r of results) { - let obj_info; - if (r.common_prefix) { - res.common_prefixes.push(r.key); - } else { - obj_info = this._get_object_info(bucket, r.key, r.stat, false, r.is_latest); - if (!list_versions && obj_info.delete_marker) { - continue; - } - if (this._is_hidden_version_path(obj_info.key)) { - obj_info.key = path.normalize(obj_info.key.replace(HIDDEN_VERSIONS_PATH + '/', '')); - obj_info.key = _get_filename(obj_info.key); - set_latest_delete_marker(obj_info); - } - res.objects.push(obj_info); - previous_key = obj_info.key; + return await this.prepare_result(bucket, is_truncated, results, list_versions, keymarker, list_type); + } catch (err) { + throw native_fs_utils.translate_error_codes(err, native_fs_utils.entity_enum.OBJECT); + } + } + + /** + * Prepare result for list_type 1 and 2, + * For list_type 1 : return simply `next_marke`, it cant hold complex objects, Because of that next marker dosnt + * hold and position for file system, + * For list_type 2 : Return object that contains `marker`, `marker_pos`, parent dir structure with positions for + * back tracking when the child dir list all files. + * @param {string} bucket + * @param {boolean} is_truncated + * @param {object[]} results + * @param {boolean} list_versions + * @param {Object} keymarker + * @param {string} list_type + */ + async prepare_result(bucket, is_truncated, results, list_versions, keymarker, list_type) { + const res = { + objects: [], + common_prefixes: [], + is_truncated, + next_marker: undefined, + next_version_id_marker: undefined, + }; + + let previous_key; + /** + * delete markers are always in the .versions folder, so we need to have special case to determine + * if they are delete markers. since the result list is ordered by latest entries first, the first + * entry of every key is the latest + * TODO need different way to check for isLatest in case of unordered list object versions + * @param {Object} obj_info + */ + const set_latest_delete_marker = obj_info => { + if (obj_info.delete_marker && previous_key !== obj_info.key) { + obj_info.is_latest = true; + } + }; + for (const r of results) { + let obj_info; + if (r.common_prefix) { + res.common_prefixes.push(r.key); + } else { + obj_info = this._get_object_info(bucket, r.key, r.stat, false, r.is_latest); + if (!list_versions && obj_info.delete_marker) { + continue; } - if (res.is_truncated) { - if (list_versions && _is_version_object(r.key)) { - const next_version_id_marker = r.key.substring(r.key.lastIndexOf('/') + 1); - res.next_version_id_marker = next_version_id_marker; - res.next_marker = _get_filename(next_version_id_marker); - } else { - res.next_marker = r.key; - } + if (this._is_hidden_version_path(obj_info.key)) { + obj_info.key = path.normalize(obj_info.key.replace(HIDDEN_VERSIONS_PATH + '/', '')); + obj_info.key = _get_filename(obj_info.key); + set_latest_delete_marker(obj_info); + } + res.objects.push(obj_info); + previous_key = obj_info.key; + } + if (res.is_truncated) { + if (list_versions && _is_version_object(r.key)) { + const next_version_id_marker = r.key.substring(r.key.lastIndexOf('/') + 1); + res.next_version_id_marker = next_version_id_marker; + res.next_marker = list_type === '2' ? { + marker: _get_filename(next_version_id_marker), + marker_pos: r.marker_pos ? r.marker_pos.toString() : '', + pre_dir: keymarker.pre_dir, + pre_dir_pos: keymarker.pre_dir_pos, + } : _get_filename(next_version_id_marker); + } else { + res.next_marker = list_type === '2' ? { + marker: r.key, + marker_pos: r.marker_pos ? r.marker_pos.toString() : '', + pre_dir: keymarker.pre_dir, + pre_dir_pos: keymarker.pre_dir_pos, + } : r.key; } } - return res; - } catch (err) { - throw native_fs_utils.translate_error_codes(err, native_fs_utils.entity_enum.OBJECT); } + return res; } ///////////////// @@ -3045,14 +3186,13 @@ class NamespaceFS { dbg.log1('Namespace_fs._delete_latest_version:', latest_ver_info); if (latest_ver_info) { if (is_gpfs) { - gpfs_options = await this._open_files_gpfs(fs_context, latest_ver_path, undefined, undefined, undefined, - undefined, true); + gpfs_options = await this._open_files_gpfs(fs_context, latest_ver_path, undefined, undefined, undefined, + undefined, true); const latest_fd = gpfs_options?.delete_version?.src_file; latest_ver_info = latest_fd && await this._get_version_info(fs_context, undefined, latest_fd); if (!latest_ver_info) break; } const versioned_path = this._get_version_path(params.key, latest_ver_info.version_id_str); - const suspended_and_latest_is_not_null = this._is_versioning_suspended() && latest_ver_info.version_id_str !== NULL_VERSION_ID; const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); @@ -3393,4 +3533,3 @@ NamespaceFS._restore_wal = null; module.exports = NamespaceFS; module.exports.multi_buffer_pool = multi_buffer_pool; - diff --git a/src/test/system_tests/test_utils.js b/src/test/system_tests/test_utils.js index 7a6b097b5c..d9c51800c1 100644 --- a/src/test/system_tests/test_utils.js +++ b/src/test/system_tests/test_utils.js @@ -491,6 +491,26 @@ function get_new_buckets_path_by_test_env(new_buckets_full_path, new_buckets_dir return is_nc_coretest ? path.join(new_buckets_full_path, new_buckets_dir) : new_buckets_dir; } + +/** + * common dummy SDK for testing + */ +function make_dummy_object_sdk() { + return { + requesting_account: { + force_md5_etag: false, + nsfs_account_config: { + uid: process.getuid(), + gid: process.getgid(), + } + }, + abort_controller: new AbortController(), + throw_if_aborted() { + if (this.abort_controller.signal.aborted) throw new Error('request aborted signal'); + } + }; +} + /** * write_manual_config_file writes config file directly to the file system without using config FS * used for creating backward compatibility tests, invalid config files etc @@ -758,6 +778,7 @@ exports.create_identity_dir_if_missing = create_identity_dir_if_missing; exports.symlink_account_name = symlink_account_name; exports.symlink_account_access_keys = symlink_account_access_keys; exports.create_file = create_file; +exports.make_dummy_object_sdk = make_dummy_object_sdk; exports.create_redirect_file = create_redirect_file; exports.delete_redirect_file = delete_redirect_file; exports.create_system_json = create_system_json; @@ -765,3 +786,4 @@ exports.update_system_json = update_system_json; exports.fail_test_if_default_config_dir_exists = fail_test_if_default_config_dir_exists; exports.create_config_dir = create_config_dir; exports.clean_config_dir = clean_config_dir; + diff --git a/src/test/unit_tests/jest_tests/test_list_object.test.js b/src/test/unit_tests/jest_tests/test_list_object.test.js deleted file mode 100644 index ea1fabdee8..0000000000 --- a/src/test/unit_tests/jest_tests/test_list_object.test.js +++ /dev/null @@ -1,175 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - - -const fs = require('fs'); -const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const {TMP_PATH} = require('../../system_tests/test_utils'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); - -const tmp_fs_path = path.join(TMP_PATH, 'test_list_object'); -const DEFAULT_FS_CONFIG = get_process_fs_context(); - -// eslint-disable-next-line max-lines-per-function -describe('manage list objct flow', () => { - describe('Telldir and Seekdir implementation', () => { - const list_dir_root = path.join(tmp_fs_path, 'list_dir_root'); - const list_dir_1_1 = path.join(list_dir_root, 'list_dir_1_1'); - const total_files = 4; - - beforeAll(async () => { - await fs_utils.create_fresh_path(list_dir_root); - await fs_utils.create_fresh_path(list_dir_1_1); - for (let i = 0; i < total_files; i++) { - create_temp_file(list_dir_root, `test_${i}.json`, {test: test}); - } - }); - - afterAll(async () => { - await fs_utils.folder_delete(`${list_dir_root}`); - await fs_utils.folder_delete(`${list_dir_1_1}`); - }); - - it('telldir returns bigint', async () => { - const dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); - const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); - expect(typeof tell_dir).toStrictEqual('bigint'); - }); - - it('seekdir expects bigint', async () => { - const big_int = 2n ** 32n; - const dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); - const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); - expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, Number(tell_dir))).toThrow(); - expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, 2n ** 32n ** 32n)).toThrow(); - expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, -(2n ** 32n ** 32n))).toThrow(); - // valid scenario - expect(await dir_handle.seekdir(DEFAULT_FS_CONFIG, big_int)).toBeUndefined(); - - }); - - it('list dir files - telldir and seekdir.', async () => { - let tell_dir; - let dir_marker; - let total_dir_entries = 0; - let dir_entry; - let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); - // reak first read after 3 entries. - for (let i = 0; i <= 2; i++) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); - dir_marker = { - dir_path: list_dir_root, - pos: tell_dir, - }; - total_dir_entries += 1; - } - // Continue the read using dir location fetch from telldir - try { - dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); - await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); - for (;;) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - total_dir_entries += 1; - } - await dir_handle.close(DEFAULT_FS_CONFIG); - dir_handle = null; - } catch (err) { - console.log("Error :", err); - } - //total number of dir and files inside list_dir_root is 5 - expect(total_dir_entries).toBe(total_files + 1); - }); - - it('list dir files - Dir.read() and seekdir()', async () => { - let dir_marker; - let total_dir_entries = 0; - let dir_entry; - let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); - // reak first read after 3 entries. - for (let i = 0; i <= 2; i++) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); - //verify tell_dir and dir_entry.off return same value - expect(tell_dir).toBe(dir_entry.off); - dir_marker = { - dir_path: list_dir_root, - pos: dir_entry.off, - }; - total_dir_entries += 1; - } - // Continue the read using dir location fetch from Dir.read() - try { - dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); - await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); - for (;;) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - total_dir_entries += 1; - } - await dir_handle.close(DEFAULT_FS_CONFIG); - dir_handle = null; - } catch (err) { - console.log("Error :", err); - } - //total number of dir and files inside list_dir_root is 5 - expect(total_dir_entries).toBe(total_files + 1); - }); - - it('list 10000 dir files - telldir and seekdir', async () => { - for (let i = total_files; i < total_files + 9995; i++) { - create_temp_file(list_dir_root, `test_${i}.json`, {test: test}); - } - let tell_dir; - let dir_marker; - let total_dir_entries = 0; - let dir_entry; - let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); - // reak first read after 3 entries. - for (let i = 0; i <= 500; i++) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); - dir_marker = { - dir_path: list_dir_root, - pos: tell_dir, - }; - total_dir_entries += 1; - } - // Continue the read using dir location fetch from telldir - try { - dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); - await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); - for (;;) { - dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); - if (!dir_entry) break; - total_dir_entries += 1; - } - await dir_handle.close(DEFAULT_FS_CONFIG); - dir_handle = null; - } catch (err) { - console.log("Error :", err); - } - //total number of dir and files inside list_dir_root is 5 - expect(total_dir_entries).toBe(10000); - }, 10000); - }); -}); - -/** - * create_temp_file would create a file with the data - * @param {string} path_to_dir - * @param {string} file_name - * @param {object} data - */ -async function create_temp_file(path_to_dir, file_name, data) { - const path_to_temp_file_name = path.join(path_to_dir, file_name); - const content = JSON.stringify(data); - await fs.promises.writeFile(path_to_temp_file_name, content); - return path_to_temp_file_name; -} diff --git a/src/test/unit_tests/jest_tests/test_unsort_list_object.test.js b/src/test/unit_tests/jest_tests/test_unsort_list_object.test.js new file mode 100644 index 0000000000..388749f4e7 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_unsort_list_object.test.js @@ -0,0 +1,541 @@ +/* Copyright (C) 2016 NooBaa */ +/* eslint-disable no-undef */ +'use strict'; + + +const fs = require('fs'); +const path = require('path'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const { TMP_PATH, make_dummy_object_sdk } = require('../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const buffer_utils = require('../../../util/buffer_utils'); +const crypto = require('crypto'); +const config = require('../../../../config'); + +const DEFAULT_FS_CONFIG = get_process_fs_context(); +config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; +const tmp_nsfs_path = path.join(TMP_PATH, 'test_unsort_list_objects'); +const upload_bkt = 'test_ns_uploads_object'; +const src_bkt = 'src'; +const timeout = 50000; +const ns_tmp_bucket_path = `${tmp_nsfs_path}/${src_bkt}`; + +const tmp_ns_nsfs_path = path.join(TMP_PATH, 'test_nsfs_unsort_list'); +const nsfs_src_bkt = 'nsfs_src'; +const ns_nsfs_tmp_bucket_path = `${tmp_ns_nsfs_path}/${nsfs_src_bkt}`; +const list_bkt = 'test_ns_list_object'; + +const files_without_folders_to_upload = make_keys(264, i => `file_without_folder${i}`); +const folders_to_upload = make_keys(264, i => `folder${i}/`); +const files_in_folders_to_upload = make_keys(264, i => `folder1/file${i}`); +const files_in_utf_diff_delimiter = make_keys(264, i => `תיקיה#קובץ${i}`); +const files_in_inner_folders_to_upload_post = make_keys(264, i => `folder1/inner_folder/file${i}`); +const files_in_inner_folders_to_upload_pre = make_keys(264, i => `folder1/ainner_folder/file${i}`); +const dummy_object_sdk = make_dummy_object_sdk(); +const ns_tmp = new NamespaceFS({ bucket_path: ns_tmp_bucket_path, bucket_id: '2', namespace_resource_id: undefined }); +const ns_nsfs_tmp = new NamespaceFS({ bucket_path: ns_nsfs_tmp_bucket_path, bucket_id: '3', namespace_resource_id: undefined }); + +// eslint-disable-next-line max-lines-per-function +describe('manage unsorted list objcts flow', () => { + const keys_objects = make_keys(999, i => `max_keys_test${i}`); + describe('Unsorted List objects ', () => { + const data = crypto.randomBytes(100); + + beforeAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + await fs_utils.create_fresh_path(ns_tmp_bucket_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(`${ns_tmp_bucket_path}`); + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; + }); + + it('List object unsorted with one object upload', async () => { + const upload_key_2 = 'my_data'; + await ns_tmp.upload_object({ + bucket: upload_bkt, + key: upload_key_2, + source_stream: buffer_utils.buffer_to_read_stream(data), + }, dummy_object_sdk); + const ls_obj_res = await ns_tmp.list_objects({ + bucket: upload_bkt, + delimiter: '/', + }, dummy_object_sdk); + expect(ls_obj_res.objects.map(obj => obj.key)).toStrictEqual([upload_key_2]); + }); + + it('List object unsorted with multiple object upload', async () => { + await create_keys(upload_bkt, ns_tmp, keys_objects); + const ls_obj_res = await ns_tmp.list_objects({ + bucket: upload_bkt, + delimiter: '/', + }, dummy_object_sdk); + expect(ls_obj_res.objects.map(it => it.key).length).toStrictEqual(keys_objects.length + 1); + }); + }); + + describe('Telldir and Seekdir implementation', () => { + const tmp_fs_path = path.join(TMP_PATH, 'test_list_object'); + const list_dir_root = path.join(tmp_fs_path, 'list_dir_root'); + const list_dir_1_1 = path.join(list_dir_root, 'list_dir_1_1'); + const total_files = 4; + beforeAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + await fs_utils.create_fresh_path(list_dir_root); + await fs_utils.create_fresh_path(list_dir_1_1); + for (let i = 0; i < total_files; i++) { + create_temp_file(list_dir_root, `test_${i}.json`, {test: test}); + } + }); + + afterAll(async () => { + await fs_utils.folder_delete(`${list_dir_root}`); + await fs_utils.folder_delete(`${list_dir_1_1}`); + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; + }); + + it('telldir returns bigint', async () => { + const dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); + const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); + expect(typeof tell_dir).toStrictEqual('bigint'); + }); + + it('seekdir expects bigint', async () => { + const big_int = 2n ** 32n; + const dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); + const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); + expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, Number(tell_dir))).toThrow(); + expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, 2n ** 32n ** 32n)).toThrow(); + expect(() => dir_handle.seekdir(DEFAULT_FS_CONFIG, -(2n ** 32n ** 32n))).toThrow(); + // valid scenario + expect(await dir_handle.seekdir(DEFAULT_FS_CONFIG, big_int)).toBeUndefined(); + + }); + + it('list dir files - telldir and seekdir.', async () => { + let tell_dir; + let dir_marker; + let total_dir_entries = 0; + let dir_entry; + let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); + // reak first read after 3 entries. + for (let i = 0; i <= 2; i++) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); + dir_marker = { + dir_path: list_dir_root, + pos: tell_dir, + }; + total_dir_entries += 1; + } + // Continue the read using dir location fetch from telldir + try { + dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); + await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); + for (;;) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + total_dir_entries += 1; + } + await dir_handle.close(DEFAULT_FS_CONFIG); + dir_handle = null; + } catch (err) { + console.log("Error :", err); + } + //total number of dir and files inside list_dir_root is 5 + expect(total_dir_entries).toBe(total_files + 1); + }); + + it('list dir files - Dir.read() and seekdir()', async () => { + let dir_marker; + let total_dir_entries = 0; + let dir_entry; + let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); + // reak first read after 3 entries. + for (let i = 0; i <= 2; i++) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + const tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); + //verify tell_dir and dir_entry.off return same value + expect(tell_dir).toBe(dir_entry.off); + dir_marker = { + dir_path: list_dir_root, + pos: dir_entry.off, + }; + total_dir_entries += 1; + } + // Continue the read using dir location fetch from Dir.read() + try { + dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); + await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); + for (;;) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + total_dir_entries += 1; + } + await dir_handle.close(DEFAULT_FS_CONFIG); + dir_handle = null; + } catch (err) { + console.log("Error :", err); + } + //total number of dir and files inside list_dir_root is 5 + expect(total_dir_entries).toBe(total_files + 1); + }); + + it('list 10000 dir files - telldir and seekdir', async () => { + for (let i = total_files; i < total_files + 9995; i++) { + create_temp_file(list_dir_root, `test_${i}.json`, {test: test}); + } + let tell_dir; + let dir_marker; + let total_dir_entries = 0; + let dir_entry; + let dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, list_dir_root); + // Seak first read after 3 entries. + for (let i = 0; i <= 500; i++) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + tell_dir = await dir_handle.telldir(DEFAULT_FS_CONFIG); + dir_marker = { + dir_path: list_dir_root, + pos: tell_dir, + }; + total_dir_entries += 1; + } + // Continue the read using dir location fetch from telldir + try { + dir_handle = await nb_native().fs.opendir(DEFAULT_FS_CONFIG, dir_marker.dir_path); + await dir_handle.seekdir(DEFAULT_FS_CONFIG, dir_marker.pos); + for (;;) { + dir_entry = await dir_handle.read(DEFAULT_FS_CONFIG); + if (!dir_entry) break; + total_dir_entries += 1; + } + await dir_handle.close(DEFAULT_FS_CONFIG); + dir_handle = null; + } catch (err) { + console.log("Error :", err); + } + //total number of dir and files inside list_dir_root is 5 + expect(total_dir_entries).toBe(10000); + }); + }); + + + describe('list objects - dirs', () => { + beforeAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + await fs_utils.create_fresh_path(ns_tmp_bucket_path); + await create_keys(upload_bkt, ns_tmp, [ + ...folders_to_upload, + ...files_in_folders_to_upload, + ...files_without_folders_to_upload, + ...files_in_utf_diff_delimiter, + ...files_in_inner_folders_to_upload_pre, + ...files_in_inner_folders_to_upload_post + ]); + }); + afterAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; + await delete_keys(upload_bkt, ns_tmp, [ + ...folders_to_upload, + ...files_in_folders_to_upload, + ...files_without_folders_to_upload, + ...files_in_utf_diff_delimiter, + ...files_in_inner_folders_to_upload_pre, + ...files_in_inner_folders_to_upload_post + ]); + await fs_utils.folder_delete(`${ns_tmp_bucket_path}`); + }, timeout); + it('key_marker=folder229/', async () => { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: "2", + key_marker: 'folder229/' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(false); + expect(r.common_prefixes).toStrictEqual([]); + const fd = folders_to_upload.filter(folder => folder > 'folder229/'); + expect(r.objects.map(it => it.key).sort()).toStrictEqual([...fd, ...files_in_utf_diff_delimiter].sort()); + }); + + it('key_marker=folder229', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: "2", + key_marker: 'folder229' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(false); + expect(r.common_prefixes).toStrictEqual([]); + const fd = folders_to_upload.filter(folder => folder >= 'folder229/'); + expect(r.objects.map(it => it.key).sort()).toEqual([...fd, ...files_in_utf_diff_delimiter].sort()); + }); + + it('key_marker=folder1/', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: "2", + key_marker: 'folder1/' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(true); + expect(r.common_prefixes).toStrictEqual([]); + expect(r.objects.length).toEqual(1000); + expect(r.objects.map(it => it.key)).not.toContain("folder0/"); + }); + + it('key_marker=folder1/file57', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: "2", + key_marker: 'folder1/file57' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(false); + expect(r.common_prefixes).toStrictEqual([]); + const fd = folders_to_upload.filter(folder => folder > 'folder1/'); + const fd1 = files_in_folders_to_upload.filter(folder => folder > 'folder1/file57'); + expect(r.objects.map(it => it.key).sort()).toEqual([...fd1, ...files_in_inner_folders_to_upload_post, + ...fd, ...files_in_utf_diff_delimiter].sort()); + }); + it('key_marker=folder1/inner_folder/file40', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: "2", + key_marker: 'folder1/inner_folder/file40' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(false); + expect(r.common_prefixes).toStrictEqual([]); + const fd1 = files_in_inner_folders_to_upload_post.filter(file => file > 'folder1/inner_folder/file40'); + const fd = folders_to_upload.filter(folder => folder > 'folder1/'); + expect(r.objects.map(it => it.key).sort()).toEqual([...fd1, ...fd, ...files_in_utf_diff_delimiter].sort()); + }); + + it('key_marker=folder1/inner_folder/', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: '2', + key_marker: 'folder1/inner_folder/' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(false); + expect(r.common_prefixes).toStrictEqual([]); + const fd1 = files_in_inner_folders_to_upload_post.filter(file => file > 'folder1/inner_folder/'); + const fd = folders_to_upload.filter(folder => folder > 'folder1/inner_folder/'); + expect(r.objects.map(it => it.key).sort()).toEqual([...fd1, ...fd, ...files_in_utf_diff_delimiter].sort()); + }); + + it('key_marker=folder1/ainner_folder/', async function() { + const r = await ns_tmp.list_objects({ + bucket: upload_bkt, + list_type: '2', + key_marker: 'folder1/ainner_folder/file50' + }, dummy_object_sdk); + expect(r.is_truncated).toStrictEqual(true); + expect(r.common_prefixes).toStrictEqual([]); + expect(r.objects.length).toEqual(1000); + expect(r.objects.map(it => it.key)).not.toContain("folder1/ainner_folder/file50"); + expect(r.objects.map(it => it.key)).not.toContain("folder1/ainner_folder/file49"); + }); + }); + + describe('list objects - pagination', () => { + beforeAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + await fs_utils.create_fresh_path(ns_nsfs_tmp_bucket_path); + await create_keys(list_bkt, ns_nsfs_tmp, [ + ...files_without_folders_to_upload, + ...folders_to_upload, + ...files_in_folders_to_upload, + ...files_in_inner_folders_to_upload_post, + ...files_in_inner_folders_to_upload_pre, + ...files_in_utf_diff_delimiter, + ]); + }, timeout); + afterAll(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; + await delete_keys(list_bkt, ns_nsfs_tmp, [ + ...folders_to_upload, + ...files_in_folders_to_upload, + ...files_without_folders_to_upload, + ...files_in_utf_diff_delimiter, + ...files_in_inner_folders_to_upload_pre, + ...files_in_inner_folders_to_upload_post + ]); + await fs_utils.folder_delete(`${ns_nsfs_tmp_bucket_path}`); + }); + it('page=1000 and list_type 2', async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + list_type: "2", + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }, timeout); + it('page=500 and list_type 2', async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + list_type: "2", + limit: 500, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }, timeout); + it('page=250 and list_type 2', async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + list_type: "2", + limit: 250, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }, timeout); + it('page=100 and list_type 2', async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = true; + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + list_type: "2", + limit: 100, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }, timeout); + it('page=250 and list_type 1', async () => { + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + limit: 250, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }); + it('page=500 and list_type 1', async () => { + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + limit: 500, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }); + it('page=1000 and list_type 1', async () => { + let r; + let total_items = 0; + for (;;) { + r = await ns_nsfs_tmp.list_objects({ + bucket: list_bkt, + key_marker: r ? r.next_marker : "", + }, dummy_object_sdk); + total_items += r.objects.length; + await validat_pagination(r, total_items); + if (!r.next_marker) { + break; + } + } + }); + }); +}); + + +async function validat_pagination(r, total_items) { + if (r.next_marker) { + expect(r.is_truncated).toStrictEqual(true); + } else { + expect(total_items).toEqual(1584); + expect(r.is_truncated).toStrictEqual(false); + } +} +/** + * @param {number} count + * @param {(i:number)=>string} gen + * @returns {string[]} + */ +function make_keys(count, gen) { + const arr = new Array(count); + for (let i = 0; i < count; ++i) arr[i] = gen(i); + arr.sort(); + Object.freeze(arr); + return arr; +} + +async function delete_keys(bkt, nsfs_delete_tmp, keys) { + await nsfs_delete_tmp.delete_multiple_objects({ + bucket: bkt, + objects: keys.map(key => ({ key })), + }, dummy_object_sdk); +} + +async function create_keys(bkt, nsfs_create_tmp, keys) { + return Promise.all(keys.map(async key => { + await nsfs_create_tmp.upload_object({ + bucket: bkt, + key, + content_type: 'application/octet-stream', + source_stream: buffer_utils.buffer_to_read_stream(null), + size: 0 + }, dummy_object_sdk); + })); +} + +/** + * create_temp_file would create a file with the data + * @param {string} path_to_dir + * @param {string} file_name + * @param {object} data + */ +async function create_temp_file(path_to_dir, file_name, data) { + const path_to_temp_file_name = path.join(path_to_dir, file_name); + const content = JSON.stringify(data); + await fs.promises.writeFile(path_to_temp_file_name, content); + return path_to_temp_file_name; +} diff --git a/src/test/unit_tests/test_namespace_fs.js b/src/test/unit_tests/test_namespace_fs.js index 5ff9b6fc31..a461c88390 100644 --- a/src/test/unit_tests/test_namespace_fs.js +++ b/src/test/unit_tests/test_namespace_fs.js @@ -21,10 +21,9 @@ const s3_utils = require('../../endpoint/s3/s3_utils'); const buffer_utils = require('../../util/buffer_utils'); const { S3Error } = require('../../endpoint/s3/s3_errors'); const test_ns_list_objects = require('./test_ns_list_objects'); -const { TMP_PATH } = require('../system_tests/test_utils'); +const { TMP_PATH, make_dummy_object_sdk } = require('../system_tests/test_utils'); const { get_process_fs_context } = require('../../util/native_fs_utils'); const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); -const SensitiveString = require('../../util/sensitive_string'); const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); @@ -42,31 +41,6 @@ const DEFAULT_FS_CONFIG = get_process_fs_context(); const empty_data = crypto.randomBytes(0); const empty_stream = () => buffer_utils.buffer_to_read_stream(empty_data); -function make_dummy_object_sdk(config_root) { - return { - requesting_account: { - force_md5_etag: false, - nsfs_account_config: { - uid: process.getuid(), - gid: process.getgid(), - } - }, - abort_controller: new AbortController(), - throw_if_aborted() { - if (this.abort_controller.signal.aborted) throw new Error('request aborted signal'); - }, - - read_bucket_sdk_config_info(name) { - return { - bucket_owner: new SensitiveString('dummy-owner'), - owner_account: { - id: 'dummy-id-123', - } - }; - } - }; -} - mocha.describe('namespace_fs', function() { const src_bkt = 'src'; @@ -99,6 +73,7 @@ mocha.describe('namespace_fs', function() { }); mocha.before(async () => { + config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; await P.all(_.map([src_bkt, upload_bkt, mpu_bkt], async buck => fs_utils.create_fresh_path(`${tmp_fs_path}/${buck}`))); }); diff --git a/src/test/unit_tests/test_namespace_fs_mpu.js b/src/test/unit_tests/test_namespace_fs_mpu.js index 9f11327de9..712cdf6229 100644 --- a/src/test/unit_tests/test_namespace_fs_mpu.js +++ b/src/test/unit_tests/test_namespace_fs_mpu.js @@ -17,35 +17,18 @@ const time_utils = require('../../util/time_utils'); const NamespaceFS = require('../../sdk/namespace_fs'); const s3_utils = require('../../endpoint/s3/s3_utils'); const buffer_utils = require('../../util/buffer_utils'); -const { TMP_PATH } = require('../system_tests/test_utils'); +const { TMP_PATH, make_dummy_object_sdk } = require('../system_tests/test_utils'); const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); const XATTR_MD5_KEY = 'content_md5'; -const DUMMY_OBJECT_SDK = make_DUMMY_OBJECT_SDK(); +const DUMMY_OBJECT_SDK = make_dummy_object_sdk(); const src_bkt = 'src'; const tmp_fs_path = path.join(TMP_PATH, 'test_namespace_fs_mpu'); const ns_tmp_bucket_path = `${tmp_fs_path}/${src_bkt}`; -function make_DUMMY_OBJECT_SDK() { - return { - requesting_account: { - force_md5_etag: false, - nsfs_account_config: { - uid: process.getuid(), - gid: process.getgid(), - } - }, - abort_controller: new AbortController(), - throw_if_aborted() { - if (this.abort_controller.signal.aborted) throw new Error('request aborted signal'); - } - }; -} - - mocha.describe('namespace_fs mpu optimization tests', function() { const upload_bkt = 'test_ns_uploads_object';