From 99c971f13868cc32f6fcab72f4e4196df1d18b70 Mon Sep 17 00:00:00 2001 From: Nick Putting Date: Wed, 2 Jun 2021 20:53:50 -0500 Subject: [PATCH 1/4] Adding progress percent based starting Adding progress percent based starting --- .gitignore | 1 + config.yaml.default | 5 +++++ plotmanager/library/utilities/jobs.py | 8 ++++++++ plotmanager/library/utilities/objects.py | 1 + 4 files changed, 15 insertions(+) diff --git a/.gitignore b/.gitignore index 0eebacd..cd1ea0b 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,4 @@ dmypy.json # Custom config.yaml *.bak +.idea diff --git a/config.yaml.default b/config.yaml.default index 3f75870..d376eec 100644 --- a/config.yaml.default +++ b/config.yaml.default @@ -137,6 +137,9 @@ jobs: # size: This refers to the k size of the plot. You would type in something like 32, 33, 34, 35... in here. # bitfield: This refers to whether you want to use bitfield or not in your plotting. Typically, you want to keep # this as true. + # [OPTIONAL] percent: Use progress percentage to control starting plots, this will aim to space plots 1/max_concurrent + # spacing apart dynamically. It's recommended to use a short stagger minutes with this as well as + # max_concurrent_with_start_early >= max_concurrent + 1 # threads: This is the number of threads that will be assigned to the plotter. Only phase 1 uses more than 1 thread. # buckets: The number of buckets to use. The default provided by Chia is 128. # memory_buffer: The amount of memory you want to allocate to the process. @@ -183,6 +186,7 @@ jobs: destination_directory: J:\Plots size: 32 bitfield: true + percent: false threads: 8 buckets: 128 memory_buffer: 4000 @@ -216,6 +220,7 @@ jobs: - K:\Plots size: 32 bitfield: true + percent: false threads: 8 buckets: 128 memory_buffer: 4000 diff --git a/plotmanager/library/utilities/jobs.py b/plotmanager/library/utilities/jobs.py index 07a7c97..8059387 100644 --- a/plotmanager/library/utilities/jobs.py +++ b/plotmanager/library/utilities/jobs.py @@ -120,6 +120,7 @@ def load_jobs(config_jobs): job.size = info['size'] job.bitfield = info['bitfield'] + job.percent = info['percent'] job.threads = info['threads'] job.buckets = info['buckets'] job.memory_buffer = info['memory_buffer'] @@ -204,10 +205,17 @@ def monitor_jobs_to_start(jobs, running_work, max_concurrent, max_for_phase_1, n f'Setting Max: {max_for_phase_1}') continue phase_1_count = 0 + percent = 0 for pid in job.running_work: + percent += float(running_work[pid].progress.strip("%")) if running_work[pid].current_phase > 1: continue phase_1_count += 1 + percent = round(percent / job.total_running, 2) + minimum_percent = (job.total_running + 1) / 2 / job.max_concurrent * 100 + if percent < minimum_percent and job.percent: + logging.info(f'Minimum percent not met, {percent} vs {minimum_percent}') + continue logging.info(f'Total jobs in phase 1: {phase_1_count}') if job.max_for_phase_1 and phase_1_count >= job.max_for_phase_1: logging.info(f'Job max for phase 1 met, skipping. Max: {job.max_for_phase_1}') diff --git a/plotmanager/library/utilities/objects.py b/plotmanager/library/utilities/objects.py index b9736cf..be85871 100644 --- a/plotmanager/library/utilities/objects.py +++ b/plotmanager/library/utilities/objects.py @@ -28,6 +28,7 @@ class Job: skip_full_destinations = None size = None bitfield = None + percent = None threads = None buckets = None memory_buffer = None From 5bedd68b58e7d9f05ee0232bd941238fffc664cf Mon Sep 17 00:00:00 2001 From: Nick Putting Date: Wed, 2 Jun 2021 21:24:04 -0500 Subject: [PATCH 2/4] Update jobs.py --- plotmanager/library/utilities/jobs.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plotmanager/library/utilities/jobs.py b/plotmanager/library/utilities/jobs.py index 8059387..a5953e4 100644 --- a/plotmanager/library/utilities/jobs.py +++ b/plotmanager/library/utilities/jobs.py @@ -156,6 +156,7 @@ def determine_job_size(k_size): if k_size > base_k_size: # Why 2.06? Just some quick math from my current plots. size *= pow(2.06, k_size-base_k_size) + size *= pow(2.06, k_size - base_k_size) return size @@ -211,11 +212,12 @@ def monitor_jobs_to_start(jobs, running_work, max_concurrent, max_for_phase_1, n if running_work[pid].current_phase > 1: continue phase_1_count += 1 - percent = round(percent / job.total_running, 2) - minimum_percent = (job.total_running + 1) / 2 / job.max_concurrent * 100 - if percent < minimum_percent and job.percent: - logging.info(f'Minimum percent not met, {percent} vs {minimum_percent}') - continue + if job.total_running != 0: + percent = round(percent / job.total_running, 2) + minimum_percent = (job.total_running + 1) / 2 / job.max_concurrent * 100 + if percent < minimum_percent and job.percent: + logging.info(f'Minimum percent not met, {percent} vs {minimum_percent}') + continue logging.info(f'Total jobs in phase 1: {phase_1_count}') if job.max_for_phase_1 and phase_1_count >= job.max_for_phase_1: logging.info(f'Job max for phase 1 met, skipping. Max: {job.max_for_phase_1}') From 78f56da101f9fd610fbd34f84e01e736ea4683e0 Mon Sep 17 00:00:00 2001 From: Nick Putting Date: Wed, 2 Jun 2021 21:25:43 -0500 Subject: [PATCH 3/4] Update jobs.py --- plotmanager/library/utilities/jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plotmanager/library/utilities/jobs.py b/plotmanager/library/utilities/jobs.py index a5953e4..ee95cc1 100644 --- a/plotmanager/library/utilities/jobs.py +++ b/plotmanager/library/utilities/jobs.py @@ -156,7 +156,7 @@ def determine_job_size(k_size): if k_size > base_k_size: # Why 2.06? Just some quick math from my current plots. size *= pow(2.06, k_size-base_k_size) - size *= pow(2.06, k_size - base_k_size) + size *= pow(2.06, k_size-base_k_size) return size From 8e1301189075b8ba47400008f0b7805dc275d7f1 Mon Sep 17 00:00:00 2001 From: Nick Putting Date: Wed, 2 Jun 2021 21:45:48 -0500 Subject: [PATCH 4/4] Remove duplicate line. --- plotmanager/library/utilities/jobs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plotmanager/library/utilities/jobs.py b/plotmanager/library/utilities/jobs.py index ee95cc1..88157b4 100644 --- a/plotmanager/library/utilities/jobs.py +++ b/plotmanager/library/utilities/jobs.py @@ -156,7 +156,6 @@ def determine_job_size(k_size): if k_size > base_k_size: # Why 2.06? Just some quick math from my current plots. size *= pow(2.06, k_size-base_k_size) - size *= pow(2.06, k_size-base_k_size) return size