free up disk space to allow linux CI to pass

Allow getdeps to free up some disk from the runner and intermediate build steps as looks like folly linux CI link step might be failing with out of disk space. Its an optional feature of getdeps with ia new --free-up-disk option to enable. When enabled it removes intermediate build state once deps have built, and also the massive unused android sdk from the github runners. Regenerated CI yaml with: ./build/fbcode_builder/getdeps.py generate-github-actions --src-dir=. --free-up-disk --os-type=linux --output-dir=.github/workflows folly
facebook · Sep 23, 2023 · 4a47cd6 · 4a47cd6
1 parent faa6889
commit 4a47cd6
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 18 deletions.
diff --git a/.github/workflows/getdeps_linux.yml b/.github/workflows/getdeps_linux.yml
@@ -15,6 +15,12 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
+    - name: Show disk space at start
+      run: df -h
+    - name: Free up disk space
+      run: sudo rm -rf /usr/local/lib/android
+    - name: Show disk space after freeing up
+      run: df -h
     - name: Fetch boost
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests boost
     - name: Fetch ninja
@@ -50,39 +56,39 @@ jobs:
     - name: Fetch xz
       run: python3 build/fbcode_builder/getdeps.py fetch --no-tests xz
     - name: Build boost
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests boost
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests boost
     - name: Build ninja
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests ninja
     - name: Build cmake
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests cmake
     - name: Build double-conversion
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests double-conversion
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests double-conversion
     - name: Build fmt
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests fmt
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests fmt
     - name: Build gflags
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests gflags
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests gflags
     - name: Build glog
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests glog
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests glog
     - name: Build googletest
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests googletest
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests googletest
     - name: Build libevent
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests libevent
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libevent
     - name: Build lz4
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests lz4
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests lz4
     - name: Build snappy
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests snappy
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests snappy
     - name: Build zstd
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests zstd
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests zstd
     - name: Build autoconf
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests autoconf
     - name: Build automake
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests automake
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests automake
     - name: Build libtool
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libtool
     - name: Build libsodium
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests libsodium
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests libsodium
     - name: Build xz
-      run: python3 build/fbcode_builder/getdeps.py build --no-tests xz
+      run: python3 build/fbcode_builder/getdeps.py build --free-up-disk --no-tests xz
     - name: Build folly
       run: python3 build/fbcode_builder/getdeps.py build --src-dir=. folly  --project-install-prefix folly:/usr/local
     - name: Copy artifacts
@@ -93,3 +99,5 @@ jobs:
         path: _artifacts
     - name: Test folly
       run: python3 build/fbcode_builder/getdeps.py test --src-dir=. folly  --project-install-prefix folly:/usr/local
+    - name: Show disk space at end
+      run: df -h
diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py
@@ -795,6 +795,12 @@ def setup_project_cmd_parser(self, parser):
             action="store_true",
             default=False,
         )
+        parser.add_argument(
+            "--free-up-disk",
+            help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
+            action="store_true",
+            default=False,
+        )
 
 
 @cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
@@ -1015,6 +1021,19 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
 
             out.write("    - uses: actions/checkout@v2\n")
 
+            if build_opts.free_up_disk:
+                free_up_disk = "--free-up-disk "
+                if not build_opts.is_windows():
+                    out.write("    - name: Show disk space at start\n")
+                    out.write("      run: df -h\n")
+                    # remove the unused github supplied android dev tools
+                    out.write("    - name: Free up disk space\n")
+                    out.write("      run: sudo rm -rf /usr/local/lib/android\n")
+                    out.write("    - name: Show disk space after freeing up\n")
+                    out.write("      run: df -h\n")
+            else:
+                free_up_disk = ""
+
             allow_sys_arg = ""
             if (
                 build_opts.allow_system_packages
@@ -1065,7 +1084,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                             has_same_repo_dep = True
                         out.write("    - name: Build %s\n" % m.name)
                         out.write(
-                            f"      run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
+                            f"      run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
                         )
 
             out.write("    - name: Build %s\n" % manifest.name)
@@ -1111,6 +1130,9 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 out.write(
                     f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
                 )
+            if build_opts.free_up_disk and not build_opts.is_windows():
+                out.write("    - name: Show disk space at end\n")
+                out.write("      run: df -h\n")
 
     def setup_project_cmd_parser(self, parser):
         parser.add_argument(
@@ -1155,6 +1177,12 @@ def setup_project_cmd_parser(self, parser):
             help="add a prefix to all job names",
             default=None,
         )
+        parser.add_argument(
+            "--free-up-disk",
+            help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
+            action="store_true",
+            default=False,
+        )
 
 
 def get_arg_var_name(args):

diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py
@@ -51,6 +51,7 @@ def __init__(
         lfs_path=None,
         shared_libs: bool = False,
         facebook_internal=None,
+        free_up_disk: bool = False,
     ) -> None:
         """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
                              or for shipit-transformed repos, the build dir that
@@ -65,6 +66,7 @@ def __init__(
         use_shipit - use real shipit instead of the simple shipit transformer
         vcvars_path - Path to external VS toolchain's vsvarsall.bat
         shared_libs - whether to build shared libraries
+        free_up_disk - take extra actions to save runner disk space
         """
 
         if not install_dir:
@@ -103,6 +105,7 @@ def __init__(
         self.allow_system_packages = allow_system_packages
         self.lfs_path = lfs_path
         self.shared_libs = shared_libs
+        self.free_up_disk = free_up_disk
 
         lib_path = None
         if self.is_darwin():
@@ -602,6 +605,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions:
             "allow_system_packages",
             "lfs_path",
             "shared_libs",
+            "free_up_disk"
         }
     }