Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ADBDEV-4648: Limit diskquota hash table's size according initial request #28

Merged
merged 34 commits into from
Dec 29, 2023
Merged
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
d0b41d6
Merge remote-tracking branch 'origin/2.3.0-sync2' into 2.3.0-sync
red1452 Nov 23, 2023
f7b0ca1
Limit diskquota hash table's size according initial request
KnightMurloc Nov 27, 2023
705d2ca
using functions to work with time from gpdb instead of libc. add flag…
KnightMurloc Nov 28, 2023
2e1eed7
Revert "using functions to work with time from gpdb instead of libc. …
KnightMurloc Nov 29, 2023
645fd60
using functions to work with time from gpdb instead of libc
KnightMurloc Nov 29, 2023
1e3f233
unified logic for adding elements to hash tables in shared memory for…
KnightMurloc Nov 30, 2023
7158307
few fixes
KnightMurloc Nov 30, 2023
800adc1
remake shm_hash_enter
KnightMurloc Nov 30, 2023
7ff7717
use TimestampTz instead of time_t
KnightMurloc Nov 30, 2023
1d4a647
Merge branch 'gpdb' into ADBDEV-4648
KnightMurloc Dec 5, 2023
bcae9e1
fix formating
KnightMurloc Dec 5, 2023
9844e59
fix formating
KnightMurloc Dec 5, 2023
8af2777
make tests more stable
KnightMurloc Dec 5, 2023
edbda84
change guc description. fix warning message
KnightMurloc Dec 6, 2023
a94fd92
rework
KnightMurloc Dec 13, 2023
cdc9414
fix format
KnightMurloc Dec 13, 2023
db86b9b
change warning messages
KnightMurloc Dec 14, 2023
652c111
change macro names
KnightMurloc Dec 14, 2023
785fecd
use an external table to check warnings in the test.
KnightMurloc Dec 21, 2023
cb4e1bf
rename waribles. remove GUC value from warning messages.
KnightMurloc Dec 22, 2023
b8bbfc8
Add guc diskquota.max_reject_entries that control size of
KnightMurloc Dec 25, 2023
f650b93
fix format
KnightMurloc Dec 25, 2023
77e6caa
change default value of table_size_map_last_overflow_report from 0 to…
KnightMurloc Dec 25, 2023
a03e278
add comment.
KnightMurloc Dec 25, 2023
f65bddd
fix format
KnightMurloc Dec 25, 2023
58acb43
fix test
KnightMurloc Dec 26, 2023
a26964f
move quota_info_map_last_overflow_report size adding to diskquota_wor…
KnightMurloc Dec 26, 2023
a84787d
change warning messages. move size adding of quota_info_map to diskqu…
KnightMurloc Dec 26, 2023
2a2ed36
make tests more stable
KnightMurloc Dec 26, 2023
df6ef6f
add comment to tests. add new line at the end of new test
KnightMurloc Dec 27, 2023
e6e4033
Merge branch 'gpdb' into ADBDEV-4648
andr-sokolov Dec 28, 2023
4508501
change macro to const varibles.
KnightMurloc Dec 29, 2023
7f66cd5
make warning constats static
KnightMurloc Dec 29, 2023
7139176
add static to altered_reloid_cache_warning
KnightMurloc Dec 29, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
change macro names
  • Loading branch information
KnightMurloc committed Dec 14, 2023
commit 652c111629c73b24cf26af5f4746340efafe5479
2 changes: 2 additions & 0 deletions src/diskquota_utility.c
Original file line number Diff line number Diff line change
@@ -1699,12 +1699,14 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time
if (num_entries == max_size)
{
TimestampTz current_time = GetCurrentTimestamp();

if (TimestampDifferenceExceeds(*last_overflow_report, current_time,
diskquota_hashmap_overflow_report_timeout * 1000))
{
ereport(WARNING, (errmsg("%s %d", warning_message, guc_value)));
*last_overflow_report = current_time;
}
}

return HASH_FIND;
}
11 changes: 6 additions & 5 deletions src/gp_activetable.c
Original file line number Diff line number Diff line change
@@ -54,7 +54,7 @@ typedef struct DiskQuotaSetOFCache
HTAB *active_tables_map = NULL; // Set<DiskQuotaActiveTableFileEntry>
TimestampTz active_tables_last_overflow_report = 0;

#define ACTIVE_TABLE_WARNING \
#define ACTIVE_TABLES_MAP_WARNING \
"[diskquota] the number of active tables reached the limit, please increase " \
"the GUC value for diskquota.max_active_tables. Current " \
"diskquota.max_active_tables value:"
@@ -333,7 +333,7 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode)
item.tablespaceoid = relFileNode->node.spcNode;

LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE);
HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING,
HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING,
&active_tables_last_overflow_report, diskquota_max_active_tables);
entry = hash_search(active_tables_map, &item, action, &found);
if (entry && !found) *entry = item;
@@ -865,8 +865,9 @@ get_active_tables_oid(void)
hash_seq_init(&iter, local_active_table_file_map);
while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL)
{
HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING,
&active_tables_last_overflow_report, diskquota_max_active_tables);
HASHACTION action =
check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING,
&active_tables_last_overflow_report, diskquota_max_active_tables);
hash_search(active_tables_map, active_table_file_entry, action, NULL);
}
/* TODO: hash_seq_term(&iter); */
@@ -930,7 +931,7 @@ get_active_tables_oid(void)
while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL)
{
HASHACTION action =
check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING,
check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING,
&active_tables_last_overflow_report, diskquota_max_active_tables);
entry = hash_search(active_tables_map, active_table_file_entry, action, &found);
if (entry) *entry = *active_table_file_entry;
28 changes: 14 additions & 14 deletions src/quotamodel.c
Original file line number Diff line number Diff line change
@@ -145,7 +145,7 @@ Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = {
HTAB *quota_info_map;
TimestampTz quota_info_map_last_overflow_report = 0;

#define QUOTA_INFO_WARNING \
#define QUOTA_INFO_MAP_WARNING \
"[diskquota] the number of quota probe reached the limit, please " \
"increase the GUC value for diskquota.max_quota_probes. Current " \
"diskquota.max_quota_probes value:"
@@ -191,7 +191,7 @@ struct LocalRejectMapEntry
static HTAB *table_size_map = NULL;
static TimestampTz table_size_last_overflow_report = 0;

#define TABLE_SIZE_WARNING \
#define TABLE_SIZE_MAP_WARNING \
"[diskquota] the number of tables reached the limit, please increase " \
"the GUC value for diskquota.max_table_segments. Current " \
"diskquota.max_table_segments value:"
@@ -203,11 +203,11 @@ static HTAB *local_disk_quota_reject_map = NULL;
static TimestampTz disk_quota_reject_last_overflow_report = 0;
static TimestampTz local_disk_quota_reject_last_overflow_report = 0;

#define REJECT_MAP_WARNING \
#define DISK_QUOTA_REJECT_MAP_WARNING \
"[diskquota] Shared disk quota reject map size limit reached. " \
"Some out-of-limit schemas or roles will be lost in rejectmap. Current limit:"

#define LOCAL_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:"
#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:"

static shmem_startup_hook_type prev_shmem_startup_hook = NULL;

@@ -254,7 +254,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid)
memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid));
key.type = type;
key.segid = segid;
action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING,
action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING,
&quota_info_map_last_overflow_report, diskquota_max_quota_probes);
entry = hash_search(quota_info_map, &key, action, &found);
/* If the number of quota exceeds the limit, entry will be NULL */
@@ -281,7 +281,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys)
memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid));
key.type = type;
key.segid = i;
action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING,
action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING,
&quota_info_map_last_overflow_report, diskquota_max_quota_probes);
entry = hash_search(quota_info_map, &key, action, &found);
/* If the number of quota exceeds the limit, entry will be NULL */
@@ -312,9 +312,9 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se
keyitem.tablespaceoid = tablespaceoid;
keyitem.targettype = (uint32)type;
ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid)));
HASHACTION action =
check_hash_fullness(local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_REJECT_MAP_WARNING,
&local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES);
HASHACTION action = check_hash_fullness(
local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_DISK_QUOTA_REJECT_MAP_WARNING,
&local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES);
localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL);
if (localrejectentry)
{
@@ -990,7 +990,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map)
key.reloid = relOid;
key.id = TableSizeEntryId(cur_segid);

HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_WARNING,
HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_MAP_WARNING,
&table_size_last_overflow_report, diskquota_max_table_segments);
tsentry = hash_search(table_size_map, &key, action, &table_size_map_found);

@@ -1279,9 +1279,9 @@ flush_local_reject_map(void)
*/
if (localrejectentry->isexceeded)
{
HASHACTION action =
check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING,
&disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES);
HASHACTION action = check_hash_fullness(
disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING,
&disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES);
rejectentry = hash_search(disk_quota_reject_map, &localrejectentry->keyitem, action, &found);
if (rejectentry == NULL)
{
@@ -2119,7 +2119,7 @@ refresh_rejectmap(PG_FUNCTION_ARGS)
if (OidIsValid(rejectmapentry->keyitem.targetoid)) continue;

HASHACTION action =
check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING,
check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING,
&disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES);
new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, action, &found);
if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry));
Loading