diff --git a/.github/workflows/getdeps_linux.yml b/.github/workflows/getdeps_linux.yml index b36fc7ce30b..807485d82c8 100644 --- a/.github/workflows/getdeps_linux.yml +++ b/.github/workflows/getdeps_linux.yml @@ -15,6 +15,12 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + - name: Show disk space at start + run: df -h + - name: Free up disk space + run: sudo rm -rf /usr/local/lib/android + - name: Show disk space after freeing up + run: df -h - name: Fetch boost run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost - name: Fetch ninja @@ -50,39 +56,39 @@ jobs: - name: Fetch xz run: python3 build/fbcode_builder/getdeps.py fetch --no-tests xz - name: Build boost - run: python3 build/fbcode_builder/getdeps.py build --no-tests boost + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests boost - name: Build ninja - run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests ninja - name: Build cmake - run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests cmake - name: Build double-conversion - run: python3 build/fbcode_builder/getdeps.py build --no-tests double-conversion + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests double-conversion - name: Build fmt - run: python3 build/fbcode_builder/getdeps.py build --no-tests fmt + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests fmt - name: Build gflags - run: python3 build/fbcode_builder/getdeps.py build --no-tests gflags + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests gflags - name: Build glog - run: python3 build/fbcode_builder/getdeps.py build --no-tests glog + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests glog - name: Build googletest - run: python3 build/fbcode_builder/getdeps.py build --no-tests googletest + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests googletest - name: Build libevent - run: python3 build/fbcode_builder/getdeps.py build --no-tests libevent + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libevent - name: Build lz4 - run: python3 build/fbcode_builder/getdeps.py build --no-tests lz4 + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests lz4 - name: Build snappy - run: python3 build/fbcode_builder/getdeps.py build --no-tests snappy + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests snappy - name: Build zstd - run: python3 build/fbcode_builder/getdeps.py build --no-tests zstd + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests zstd - name: Build autoconf - run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests autoconf - name: Build automake - run: python3 build/fbcode_builder/getdeps.py build --no-tests automake + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests automake - name: Build libtool - run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libtool - name: Build libsodium - run: python3 build/fbcode_builder/getdeps.py build --no-tests libsodium + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libsodium - name: Build xz - run: python3 build/fbcode_builder/getdeps.py build --no-tests xz + run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests xz - name: Build folly run: python3 build/fbcode_builder/getdeps.py build --src-dir=. folly --project-install-prefix folly:/usr/local - name: Copy artifacts @@ -93,3 +99,5 @@ jobs: path: _artifacts - name: Test folly run: python3 build/fbcode_builder/getdeps.py test --src-dir=. folly --project-install-prefix folly:/usr/local + - name: Show disk space at end + run: df -h diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 9358c425e4a..f7baaf8e231 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -795,6 +795,12 @@ def setup_project_cmd_parser(self, parser): action="store_true", default=False, ) + parser.add_argument( + "--free-up-disk", + help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", + action="store_true", + default=False, + ) @cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") @@ -1015,6 +1021,19 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write(" - uses: actions/checkout@v2\n") + if build_opts.free_up_disk: + free_up_disk = "--free-up-disk " + if not build_opts.is_windows(): + out.write(" - name: Show disk space at start\n") + out.write(" run: df -h\n") + # remove the unused github supplied android dev tools + out.write(" - name: Free up disk space\n") + out.write(" run: sudo rm -rf /usr/local/lib/android\n") + out.write(" - name: Show disk space after freeing up\n") + out.write(" run: df -h\n") + else: + free_up_disk = "" + allow_sys_arg = "" if ( build_opts.allow_system_packages @@ -1065,7 +1084,7 @@ def write_job_for_platform(self, platform, args): # noqa: C901 has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n" + f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}{free_up_disk}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) @@ -1111,6 +1130,9 @@ def write_job_for_platform(self, platform, args): # noqa: C901 out.write( f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" ) + if build_opts.free_up_disk and not build_opts.is_windows(): + out.write(" - name: Show disk space at end\n") + out.write(" run: df -h\n") def setup_project_cmd_parser(self, parser): parser.add_argument( @@ -1155,6 +1177,12 @@ def setup_project_cmd_parser(self, parser): help="add a prefix to all job names", default=None, ) + parser.add_argument( + "--free-up-disk", + help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", + action="store_true", + default=False, + ) def get_arg_var_name(args): diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index de76a9525c4..f24ea7769b7 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -51,6 +51,7 @@ def __init__( lfs_path=None, shared_libs: bool = False, facebook_internal=None, + free_up_disk: bool = False, ) -> None: """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir, or for shipit-transformed repos, the build dir that @@ -65,6 +66,7 @@ def __init__( use_shipit - use real shipit instead of the simple shipit transformer vcvars_path - Path to external VS toolchain's vsvarsall.bat shared_libs - whether to build shared libraries + free_up_disk - take extra actions to save runner disk space """ if not install_dir: @@ -103,6 +105,7 @@ def __init__( self.allow_system_packages = allow_system_packages self.lfs_path = lfs_path self.shared_libs = shared_libs + self.free_up_disk = free_up_disk lib_path = None if self.is_darwin(): @@ -602,6 +605,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions: "allow_system_packages", "lfs_path", "shared_libs", + "free_up_disk" } }