Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

RuntimeError: pg 18.6a to be moved to osd.117 is misplaced with -198781.0<0 objects already transferred #38

Open
patrakov opened this issue Apr 14, 2024 · 1 comment
Open

RuntimeError: pg 18.6a to be moved to osd.117 is misplaced with -198781.0<0 objects already transferred #38

patrakov opened this issue Apr 14, 2024 · 1 comment

Comments

@patrakov
Copy link

I am using the script to watch the progress of backfills on a broken cluster. Yet, it shows an exception:

Traceback (most recent call last):
  File "/root/./placementoptimizer.py", line 5496, in <module>
    exit(main())
  File "/root/./placementoptimizer.py", line 5451, in main
    state.preprocess()
  File "/root/./placementoptimizer.py", line 2183, in preprocess
    raise RuntimeError(f"pg {pg_incoming} to be moved to osd.{osdid} is misplaced "
RuntimeError: pg 18.6a to be moved to osd.117 is misplaced with -198781.0<0 objects already transferred

I will send you the dump via email. Yes, I know that one PG is not recoverable without the manual export/import.
@tatuylonen
Copy link

I was having the same issue. 259 osds on 15 hosts, running reef (18.2.4). It looks like the problem pg belongs to a 2.4PB 8+2 erasure-coded pool and is active+undersized+degraded+remapped+backfill_toofull. It's number of objects is close but not identical to the negated value in the error message (could have been exactly the same at the time the error was reported). Note that "acting" osds has one missing, while it is present in "up" osds (see below).

The cluster is also in the middle of a major rebalancing due to increasing pg_num and has several osds overfull while there are other recently added osds that have plenty of space. Many pgs are backfill_toofull and 1.3% of objects are degraded. The cluster is currently doing 2-3GB/s (1.3k obj/s) recovery/backfill on the hdds with several more weeks to go.

The problem appeared immeaditely/shortly after using commands output by placementoptimizer.py; however, the problem pg was not one of the pgs output by it.

HOWEVER, since I started writing this report, the problem has gone away! I think we had the problem for a couple of hours. But apparently it is transient. I did a diff of "ceph pg 18.1b4 query" outputs from when I had the problem and after the problem disappeared, and the only changes are small changes in various counts/timestamps abnd last_update/last_complete fields. The "up" and "acting" values have not changed. The output below is from when I was still having the problem.

./placementoptimizer.py balance --osdfrom limiting
Traceback (most recent call last):
File "/root/ceph-balancer/./placementoptimizer.py", line 5497, in
exit(main())
File "/root/ceph-balancer/./placementoptimizer.py", line 5452, in main
state.preprocess()
File "/root/ceph-balancer/./placementoptimizer.py", line 2183, in preprocess
raise RuntimeError(f"pg {pg_incoming} to be moved to osd.{osdid} is misplaced "
RuntimeError: pg 18.1b4 to be moved to osd.32 is misplaced with -776870.0<0 objects already transferred

ceph pg 18.1b4 query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"epoch": 215672,
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"backfill_targets": [
"32(4)",
"122(0)"
],
"acting_recovery_backfill": [
"32(4)",
"54(6)",
"68(2)",
"70(8)",
"83(7)",
"122(0)",
"125(3)",
"132(0)",
"161(5)",
"170(1)",
"247(9)"
],
"info": {
"pgid": "18.1b4s0",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577702,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215671'4577702",
"reported_seq": 19740603,
"reported_epoch": 215672,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T20:25:11.909448+0000",
"last_change": "2024-10-13T20:25:11.909397+0000",
"last_active": "2024-10-13T20:25:11.909448+0000",
"last_peered": "2024-10-13T20:25:11.909448+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T20:01:11.801147+0000",
"last_became_peered": "2024-10-13T20:01:11.801147+0000",
"last_unstale": "2024-10-13T20:25:11.909448+0000",
"last_undegraded": "2024-10-13T20:01:11.422268+0000",
"last_fullsized": "2024-10-13T20:01:11.422115+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 564,
"log_dups_size": 0,
"ondisk_log_size": 564,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918716394516,
"num_objects": 777093,
"num_object_clones": 0,
"num_object_copies": 7770930,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776821,
"num_objects_misplaced": 739276,
"num_objects_unfound": 0,
"num_objects_dirty": 777093,
"num_whiteouts": 0,
"num_read": 803841,
"num_read_kb": 551576194,
"num_write": 5748289,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777093
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "32(4)",
"pgid": "18.1b4s4",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "18:2d8005fc:::1006742e866.000002c5:head",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 215612,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"log_dups_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 671809536,
"num_objects": 272,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 776821,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 272,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "54(6)",
"pgid": "18.1b4s6",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "68(2)",
"pgid": "18.1b4s2",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "69(4)",
"pgid": "18.1b4s4",
"last_update": "211188'4568978",
"last_complete": "211188'4568978",
"log_tail": "210915'4568438",
"last_user_version": 4568978,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215497,
"last_interval_started": 215496,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "211180'4568977",
"reported_seq": 19564165,
"reported_epoch": 211188,
"state": "active+remapped+backfill_toofull",
"last_fresh": "2024-10-06T16:46:49.755441+0000",
"last_change": "2024-10-06T16:46:30.115348+0000",
"last_active": "2024-10-06T16:46:49.755441+0000",
"last_peered": "2024-10-06T16:46:49.755441+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-06T10:25:58.339133+0000",
"last_became_peered": "2024-10-06T10:25:58.339133+0000",
"last_unstale": "2024-10-06T16:46:49.755441+0000",
"last_undegraded": "2024-10-06T16:46:49.755441+0000",
"last_fullsized": "2024-10-06T16:46:49.755441+0000",
"mapping_epoch": 215612,
"log_start": "210915'4568438",
"ondisk_log_start": "210915'4568438",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 539,
"log_dups_size": 0,
"ondisk_log_size": 539,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1923746176392,
"num_objects": 774830,
"num_object_clones": 0,
"num_object_copies": 7748290,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 2263,
"num_objects_degraded": 0,
"num_objects_misplaced": 1549658,
"num_objects_unfound": 0,
"num_objects_dirty": 774830,
"num_whiteouts": 0,
"num_read": 803596,
"num_read_kb": 551561398,
"num_write": 5735901,
"num_write_kb": 2826002453,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2836330,
"num_bytes_recovered": 6673862612392,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),69(4),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 774829
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 210747,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "70(8)",
"pgid": "18.1b4s8",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "83(7)",
"pgid": "18.1b4s7",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "110(4)",
"pgid": "18.1b4s4",
"last_update": "209518'4565236",
"last_complete": "209518'4565236",
"log_tail": "209385'4564736",
"last_user_version": 0,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215497,
"last_interval_started": 215496,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 215612,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"log_dups_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 777093,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 209521,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "122(0)",
"pgid": "18.1b4s0",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "18:2d831f56:::1003328e99c.00000000:head",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 215612,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"log_dups_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 94492426240,
"num_objects": 37817,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 739276,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 37817,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 4,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "125(3)",
"pgid": "18.1b4s3",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "161(5)",
"pgid": "18.1b4s5",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "170(1)",
"pgid": "18.1b4s1",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "247(9)",
"pgid": "18.1b4s9",
"last_update": "215671'4577702",
"last_complete": "215671'4577702",
"log_tail": "213021'4577138",
"last_user_version": 4577670,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1464,
"epoch_pool_created": 1464,
"last_epoch_started": 215613,
"last_interval_started": 215612,
"last_epoch_clean": 209341,
"last_interval_clean": 209339,
"last_epoch_split": 0,
"last_epoch_marked_full": 213776,
"same_up_since": 215611,
"same_interval_since": 215612,
"same_primary_since": 214396,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "215584'4577670",
"reported_seq": 19740230,
"reported_epoch": 215584,
"state": "active+undersized+degraded+remapped+backfill_toofull",
"last_fresh": "2024-10-13T19:59:37.801137+0000",
"last_change": "2024-10-13T19:59:12.028385+0000",
"last_active": "2024-10-13T19:59:37.801137+0000",
"last_peered": "2024-10-13T19:59:37.801137+0000",
"last_clean": "2024-10-05T21:19:48.957145+0000",
"last_became_active": "2024-10-13T19:47:11.881666+0000",
"last_became_peered": "2024-10-13T19:47:11.881666+0000",
"last_unstale": "2024-10-13T19:59:37.801137+0000",
"last_undegraded": "2024-10-13T19:47:11.556935+0000",
"last_fullsized": "2024-10-13T19:47:11.556783+0000",
"mapping_epoch": 215612,
"log_start": "213021'4577138",
"ondisk_log_start": "213021'4577138",
"created": 1464,
"last_epoch_clean": 209341,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "194136'4452286",
"last_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_deep_scrub": "194136'4452286",
"last_deep_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"last_clean_scrub_stamp": "2024-08-26T13:19:05.558024+0000",
"objects_scrubbed": 748927,
"log_size": 532,
"log_dups_size": 0,
"ondisk_log_size": 532,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 11288,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 11287.163211722,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 1918850612244,
"num_objects": 777125,
"num_object_clones": 0,
"num_object_copies": 7771250,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 776853,
"num_objects_misplaced": 739304,
"num_objects_unfound": 0,
"num_objects_dirty": 777125,
"num_whiteouts": 0,
"num_read": 803839,
"num_read_kb": 551576189,
"num_write": 5748257,
"num_write_kb": 2826988477,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2874181,
"num_bytes_recovered": 6767543386901,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
122,
170,
68,
125,
32,
161,
54,
83,
70,
247
],
"acting": [
132,
170,
68,
125,
2147483647,
161,
54,
83,
70,
247
],
"avail_no_missing": [
"132(0)",
"54(6)",
"68(2)",
"69(4)",
"70(8)",
"83(7)",
"125(3)",
"161(5)",
"170(1)",
"247(9)"
],
"object_location_counts": [
{
"shards": "54(6),68(2),70(8),83(7),125(3),132(0),161(5),170(1),247(9)",
"objects": 777125
}
],
"blocked_by": [],
"up_primary": 122,
"acting_primary": 132,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 215613,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2024-10-13T20:01:11.422177+0000",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"32(4)",
"122(0)"
],
"waiting_on_backfill": [],
"last_backfill_started": "18:2d8005fc:::1006742e866.000002c5:head",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
}
},
{
"name": "Started",
"enter_time": "2024-10-13T20:01:10.316297+0000"
}
],
"scrubber": {
"active": false,
"must_scrub": true,
"must_deep_scrub": true,
"must_repair": false,
"need_auto": false,
"scrub_reg_stamp": "1.000000",
"schedule": "queued for deep scrub"
},
"agent_state": {}
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@patrakov @tatuylonen