Skip to content

Commit

Permalink
Merge pull request #8747 from shirady/nsfs-nc-list-separate
Browse files Browse the repository at this point in the history 
NC | NSFS | CLI | Separate Bucket and Account List Functions
  • Loading branch information
@shirady
shirady authored Feb 10, 2025
2 parents b89f7bc + b11aa5d commit a3bc7dd
Show file tree
Hide file tree
Showing 2 changed files with 176 additions and 99 deletions.
242 changes: 146 additions & 96 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations'
const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
const notifications_util = require('../util/notifications_util');

///////////////
//// GENERAL //
///////////////

let config_fs;

async function main(argv = minimist(process.argv.slice(2))) {
Expand Down Expand Up @@ -92,6 +96,10 @@ async function main(argv = minimist(process.argv.slice(2))) {
}
}

///////////////
//// BUCKETS //
///////////////

// in name and new_name we allow type number, hence convert it to string
async function fetch_bucket_data(action, user_input) {
let data = {
Expand Down Expand Up @@ -146,7 +154,6 @@ async function fetch_bucket_data(action, user_input) {
return data;
}


/**
* merge_new_and_existing_config_data returns the merged object of the existing bucket data and the user data
* @param {Object} user_input_bucket_data
Expand Down Expand Up @@ -259,6 +266,62 @@ async function delete_bucket(data, force) {
}
}


/**
* filter_bucket will return true or false based on whether a bucket meets the criteria defined by the supported flags.
* @param {object} bucket
* @param {object} [filters]
*/
function filter_bucket(bucket, filters) {
for (const [key, val] of Object.entries(filters)) {
if (bucket[key] !== val) { // We will never reach here if we will not add an appropriate field to the filter
return false;
}
}
return true;
}

/**
* list_bucket_config_files will list all the bucket config files (json) in a given config directory
* @param {boolean} [wide]
* @param {object} [filters]
*/
async function list_bucket_config_files(wide, filters = {}) {
let entry_names = [];
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;

const options = {
silent_if_missing: true
};

// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entry_names = [filters.name];
} else {
entry_names = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entry_names, async entry_name => {
if (wide || should_filter) {
const data = await config_fs.get_bucket_by_name(entry_name, options);
if (!data) return undefined;
if (should_filter && !filter_bucket(data, filters)) return undefined;
if (!wide) return { name: entry_name };
await set_bucker_owner(data);
return data;
} else {
return { name: entry_name };
}
});
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
config_files_list = config_files_list.filter(item => item);

return config_files_list;
}

/**
* bucket_management does the following -
* 1. fetches the bucket data if this is not a list operation
Expand Down Expand Up @@ -287,14 +350,18 @@ async function bucket_management(action, user_input) {
} else if (action === ACTIONS.LIST) {
const bucket_filters = _.pick(user_input, LIST_BUCKET_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
const buckets = await list_config_files(TYPES.BUCKET, wide, undefined, bucket_filters);
const buckets = await list_bucket_config_files(wide, bucket_filters);
response = { code: ManageCLIResponse.BucketList, detail: buckets };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);
}

////////////////
//// ACCOUNTS //
////////////////

/**
* set_access_keys will set the access keys either given or generated.
* @param {string} access_key
Expand Down Expand Up @@ -488,65 +555,10 @@ async function get_account_status(data, show_secrets) {
}
}

/**
* account_management does the following -
* 1. sets variables by the user input options
* 2. iniates nc_master_key_manager on UPDATE/ADD/show_secrets
* 2. validates account args - TODO - we should split it to validate_account_args
* and validations of the merged account (user_input + existing account config)
* 3. call account operation based on the action argument
* 4. write output to stdout
* @param {'add'|'update'|'delete'|'status'|'list'} action
* @param {Object} user_input
*/
async function account_management(action, user_input) {
const show_secrets = get_boolean_or_string_value(user_input.show_secrets);
const is_flag_iam_operate_on_root_account = get_boolean_or_string_value(user_input.iam_operate_on_root_account);
const account_filters = _.pick(user_input, LIST_ACCOUNT_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
if (get_boolean_or_string_value(user_input.anonymous)) {
user_input.name = config.ANONYMOUS_ACCOUNT_NAME;
user_input.email = config.ANONYMOUS_ACCOUNT_NAME;
}
// init nc_mkm here to avoid concurrent initializations
// init if actions is add/update (require encryption) or show_secrets = true (require decryption)
if ([ACTIONS.ADD, ACTIONS.UPDATE].includes(action) || show_secrets) await nc_mkm.init();
const data = action === ACTIONS.LIST ? undefined : await fetch_account_data(action, user_input);
await manage_nsfs_validations.validate_account_args(config_fs, data, action, is_flag_iam_operate_on_root_account);

let response = {};
if (action === ACTIONS.ADD) {
response = await add_account(data);
} else if (action === ACTIONS.STATUS) {
response = await get_account_status(data, show_secrets);
} else if (action === ACTIONS.UPDATE) {
response = await update_account(data);
} else if (action === ACTIONS.DELETE) {
response = await delete_account(data);
} else if (action === ACTIONS.LIST) {
const accounts = await list_config_files(TYPES.ACCOUNT, wide, show_secrets, account_filters);
response = { code: ManageCLIResponse.AccountList, detail: accounts };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);

}

/**
* filter_list_item will return an answer of filter_account() or filter_bucket() based on the entity type
* @param {string} type
* @param {object} entity
* @param {string[]} [filters]
*/
function filter_list_item(type, entity, filters) {
return type === TYPES.ACCOUNT ? filter_account(entity, filters) : filter_bucket(entity, filters);
}

/**
* filter_account will return true or false based on whether an account meets the criteria defined by the supported flags.
* @param {object} account
* @param {string[]} [filters]
* @param {object} [filters]
*/
function filter_account(account, filters) {
for (const [key, val] of Object.entries(filters)) {
Expand All @@ -570,32 +582,17 @@ function filter_account(account, filters) {
}

/**
* filter_bucket will return true or false based on whether a bucket meets the criteria defined by the supported flags.
* currently not implemented
* @param {object} bucket
* @param {string[]} [filters]
*/
function filter_bucket(bucket, filters) {
for (const [key, val] of Object.entries(filters)) {
if (bucket[key] !== val) { // We will never reach here if we will not add an appropriate field to the filter
return false;
}
}
return true;
}
/**
* list_config_files will list all the config files (json) in a given config directory
* @param {string} type
* list_account_config_files will list all the account config files (json) in a given config directory
* @param {boolean} [wide]
* @param {boolean} [show_secrets]
* @param {object} [filters]
*/
async function list_config_files(type, wide, show_secrets, filters = {}) {
async function list_account_config_files(wide, show_secrets, filters = {}) {
let entry_names = [];
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;

// decryption causing mkm initalization
// decryption causing mkm initialization
// decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true)
const options = {
show_secrets: show_secrets || should_filter,
Expand All @@ -607,26 +604,18 @@ async function list_config_files(type, wide, show_secrets, filters = {}) {
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entry_names = [filters.name];
} else if (type === TYPES.ACCOUNT) {
} else {
entry_names = await config_fs.list_accounts();
} else if (type === TYPES.BUCKET) {
entry_names = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entry_names, async entry_name => {
if (wide || should_filter) {
const data = type === TYPES.ACCOUNT ?
await config_fs.get_account_by_name(entry_name, options) :
await config_fs.get_bucket_by_name(entry_name, options);
const data = await config_fs.get_account_by_name(entry_name, options);
if (!data) return undefined;
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
if (should_filter && !filter_account(data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
if (!wide) return { name: entry_name };
if (type === TYPES.ACCOUNT) return _.omit(data, show_secrets ? [] : ['access_keys']);
if (type === TYPES.BUCKET) {
await set_bucker_owner(data);
return data;
}
return _.omit(data, show_secrets ? [] : ['access_keys']);
} else {
return { name: entry_name };
}
Expand All @@ -639,16 +628,48 @@ async function list_config_files(type, wide, show_secrets, filters = {}) {
}

/**
* list_connections
* @returns An array with names of all connection files.
* account_management does the following -
* 1. sets variables by the user input options
* 2. iniates nc_master_key_manager on UPDATE/ADD/show_secrets
* 2. validates account args - TODO - we should split it to validate_account_args
* and validations of the merged account (user_input + existing account config)
* 3. call account operation based on the action argument
* 4. write output to stdout
* @param {'add'|'update'|'delete'|'status'|'list'} action
* @param {Object} user_input
*/
async function list_connections() {
let conns = await config_fs.list_connections();
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
conns = conns.filter(item => item);
async function account_management(action, user_input) {
const show_secrets = get_boolean_or_string_value(user_input.show_secrets);
const is_flag_iam_operate_on_root_account = get_boolean_or_string_value(user_input.iam_operate_on_root_account);
const account_filters = _.pick(user_input, LIST_ACCOUNT_FILTERS);
const wide = get_boolean_or_string_value(user_input.wide);
if (get_boolean_or_string_value(user_input.anonymous)) {
user_input.name = config.ANONYMOUS_ACCOUNT_NAME;
user_input.email = config.ANONYMOUS_ACCOUNT_NAME;
}
// init nc_mkm here to avoid concurrent initializations
// init if actions is add/update (require encryption) or show_secrets = true (require decryption)
if ([ACTIONS.ADD, ACTIONS.UPDATE].includes(action) || show_secrets) await nc_mkm.init();
const data = action === ACTIONS.LIST ? undefined : await fetch_account_data(action, user_input);
await manage_nsfs_validations.validate_account_args(config_fs, data, action, is_flag_iam_operate_on_root_account);

let response = {};
if (action === ACTIONS.ADD) {
response = await add_account(data);
} else if (action === ACTIONS.STATUS) {
response = await get_account_status(data, show_secrets);
} else if (action === ACTIONS.UPDATE) {
response = await update_account(data);
} else if (action === ACTIONS.DELETE) {
response = await delete_account(data);
} else if (action === ACTIONS.LIST) {
const accounts = await list_account_config_files(wide, show_secrets, account_filters);
response = { code: ManageCLIResponse.AccountList, detail: accounts };
} else {
throw_cli_error(ManageCLIError.InvalidAction);
}
write_stdout_response(response.code, response.detail, response.event_arg);

return conns;
}

/**
Expand Down Expand Up @@ -689,6 +710,10 @@ async function set_bucker_owner(bucket_data) {
bucket_data.bucket_owner = account_data?.name;
}

////////////////////
//// IP WHITELIST //
////////////////////

async function whitelist_ips_management(args) {
const ips = args.ips;
manage_nsfs_validations.validate_whitelist_arg(ips);
Expand All @@ -708,6 +733,10 @@ async function whitelist_ips_management(args) {
write_stdout_response(ManageCLIResponse.WhiteListIPUpdated, ips);
}

///////////////
//// GLACIER //
///////////////

async function glacier_management(argv) {
const action = argv._[1] || '';
await manage_glacier_operations(action, argv);
Expand All @@ -729,10 +758,18 @@ async function manage_glacier_operations(action, argv) {
}
}

//////////////////////
//// BUCKET LOGGING //
//////////////////////

async function logging_management() {
await manage_nsfs_logging.export_bucket_logging(config_fs);
}

/////////////////////
//// NOTIFICATIONS //
////////////////////

async function notification_management() {
await new notifications_util.Notificator({
fs_context: config_fs.fs_context,
Expand Down Expand Up @@ -780,5 +817,18 @@ async function connection_management(action, user_input) {
write_stdout_response(response.code, response.detail, response.event_arg);
}

/**
* list_connections
* @returns An array with names of all connection files.
*/
async function list_connections() {
let conns = await config_fs.list_connections();
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
// in case the entry was deleted during the list it also inserts undefined
conns = conns.filter(item => item);

return conns;
}

exports.main = main;
if (require.main === module) main();
Loading

0 comments on commit a3bc7dd

Please sign in to comment.