diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 614df9e783..27385b7709 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -26,6 +26,7 @@ from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.futures import asyncio +from portage.util.path import first_existing from portage.util.SlotObject import SlotObject from portage.util._async.SchedulerInterface import SchedulerInterface from portage.package.ebuild.digestcheck import digestcheck @@ -64,7 +65,7 @@ class Scheduler(PollScheduler): - # max time between loadavg checks (seconds) + # max time between loadavg and tmpdir space checks (seconds) _loadavg_latency = 30 # max time between display status updates (seconds) @@ -229,6 +230,7 @@ def __init__( if max_jobs is None: max_jobs = 1 self._set_max_jobs(max_jobs) + self._jobs_tmpdir_space_threshold = myopts.get("--jobs-tmpdir-space-threshold") self._running_root = trees[trees._running_eroot]["root_config"] self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": @@ -1573,7 +1575,10 @@ def _main_loop(self): self._main_exit = self._event_loop.create_future() if ( - self._max_load is not None + ( + self._max_load is not None + or self._jobs_tmpdir_space_threshold is not None + ) and self._loadavg_latency is not None and (self._max_jobs is True or self._max_jobs > 1) ): @@ -1792,6 +1797,30 @@ def _is_work_scheduled(self): def _running_job_count(self): return self._jobs + def _can_add_job(self): + if not super()._can_add_job(): + return False + + if self._jobs_tmpdir_space_threshold is not None and hasattr(os, "statvfs"): + tmpdir = first_existing( + os.path.join(self.settings["PORTAGE_TMPDIR"], "portage") + ) + try: + vfs_stat = os.statvfs(tmpdir) + except OSError as e: + writemsg_level( + f"!!! statvfs('{tmpdir}'): {e}\n", + noiselevel=-1, + level=logging.ERROR, + ) + else: + if ( + (vfs_stat.f_blocks - vfs_stat.f_bavail) / vfs_stat.f_blocks + ) > self._jobs_tmpdir_space_threshold: + return False + + return True + def _schedule_tasks(self): while True: state_change = 0 diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 465e20163d..18ccd76f1b 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -1,4 +1,4 @@ -# Copyright 1999-2023 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import argparse @@ -165,6 +165,7 @@ def __contains__(self, s): "--getbinpkgonly": y_or_n, "--ignore-world": y_or_n, "--jobs": valid_integers, + "--jobs-tmpdir-space-threshold": valid_floats, "--keep-going": y_or_n, "--load-average": valid_floats, "--onlydeps-with-ideps": y_or_n, @@ -523,6 +524,10 @@ def parse_opts(tmpcmdline, silent=False): "help": "Specifies the number of packages to build " + "simultaneously.", "action": "store", }, + "--jobs-tmpdir-space-threshold": { + "help": "Specifies maximum used space ratio when starting a new job.", + "action": "store", + }, "--keep-going": { "help": "continue as much as possible after an error", "choices": true_y_or_n, @@ -1033,6 +1038,24 @@ def parse_opts(tmpcmdline, silent=False): myoptions.jobs = jobs + if myoptions.jobs_tmpdir_space_threshold == "True": + myoptions.jobs_tmpdir_space_threshold = None + + if myoptions.jobs_tmpdir_space_threshold: + try: + jobs_tmpdir_space_threshold = float(myoptions.jobs_tmpdir_space_threshold) + except ValueError: + jobs_tmpdir_space_threshold = 0.0 + + if jobs_tmpdir_space_threshold <= 0.0 or jobs_tmpdir_space_threshold > 1.0: + jobs_tmpdir_space_threshold = None + if not silent: + parser.error( + f"Invalid --jobs-tmpdir-space-threshold: '{myoptions.jobs_tmpdir_space_threshold}'\n" + ) + + myoptions.jobs_tmpdir_space_threshold = jobs_tmpdir_space_threshold + if myoptions.load_average == "True": myoptions.load_average = None @@ -1304,6 +1327,16 @@ def emerge_main(args: Optional[list[str]] = None): emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts( tmpcmdline ) + if ( + "--jobs-tmpdir-space-threshold" in emerge_config.opts + and "keep-work" in emerge_config.running_config.settings.features + ): + writemsg_level( + "--jobs-tmpdir-space-threshold conflicts with FEATURES=keep-work\n", + level=logging.ERROR, + noiselevel=-1, + ) + return 1 try: return run_action(emerge_config) diff --git a/man/emerge.1 b/man/emerge.1 index e30f5f813e..cee38f8ce4 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,4 +1,4 @@ -.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage" +.TH "EMERGE" "1" "Jun 2024" "Portage @VERSION@" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" @@ -693,6 +693,14 @@ Note that interactive packages currently force a setting of \fI\-\-jobs=1\fR. This issue can be temporarily avoided by specifying \fI\-\-accept\-properties=\-interactive\fR. .TP +.BR \-\-jobs\-tmpdir\-space\-threshold[=RATIO] +Specifies the maximum ratio of used space allowed (a floating\-point +number) in \fBPORTAGE_TMPDIR\fR when starting a new job. With no +argument, removes a previous space ratio threshold. For example, +use a ratio of \fI0.85\fR to stop starting new jobs when the space +usage in \fBPORTAGE_TMPDIR\fR exceeds \fI85%\fR. This option conflicts +with \fBFEATURES="keep\-work"\fR. +.TP .BR "\-\-keep\-going [ y | n ]" Continue as much as possible after an error. When an error occurs, dependencies are recalculated for remaining packages and any with