Skip to content

Commit

Permalink
speed up file copy on windows
Browse files Browse the repository at this point in the history
Summary:
getdeps windows file copy is very slow, speed it up

Test Plan:

Fetch
```
rm -rf z:/installed/perl-* # z:/build/perl-* z:/extracted/perl-*
time python ./build/fbcode_builder/getdeps.py fetch perl
```

Before, 40s:
```
Building on {distro=None, distro_vers=10, fb=off, fbsource=off, os=windows, shared_libs=off, test=on}
Assessing perl...
Installing Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl -> Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc

real    0m40.150s
user    0m0.000s
sys     0m0.000s
```

After, 2s:
```
$ rm -rf z:/installed/perl-*
$ time python ./build/fbcode_builder/getdeps.py build perl
Building on {distro=None, distro_vers=10, fb=off, fbsource=off, os=windows, shared_libs=off, test=on}
Assessing perl...
Installing Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl -> Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc
---
+ robocopy.exe \
+      'Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl' \
+      'Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc' \
+      /E \
+      /XX \
+      /MT \
+      /NFL \
+      /NDL \
+      /NJH \
+      /NJS \
+      /NP

real    0m2.107s
user    0m0.000s
sys     0m0.000s
```
  • Loading branch information
ahornby committed Dec 4, 2024
1 parent 6c162c0 commit 500071b
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 19 deletions.
5 changes: 3 additions & 2 deletions build/fbcode_builder/getdeps/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from shlex import quote as shellquote
from typing import Optional

from .copytree import simple_copytree
from .dyndeps import create_dyn_dep_munger
from .envfuncs import add_path_entry, Env, path_search
from .fetcher import copy_if_different
Expand Down Expand Up @@ -1327,7 +1328,7 @@ def build(self, reconfigure: bool) -> None:
os.makedirs(dest_parent)
if os.path.isdir(full_src):
if not os.path.exists(full_dest):
shutil.copytree(full_src, full_dest)
simple_copytree(full_src, full_dest)
else:
shutil.copyfile(full_src, full_dest)
shutil.copymode(full_src, full_dest)
Expand All @@ -1339,7 +1340,7 @@ def build(self, reconfigure: bool) -> None:
os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
else:
if not os.path.exists(self.inst_dir):
shutil.copytree(self.src_dir, self.inst_dir)
simple_copytree(self.src_dir, self.inst_dir)


class SqliteBuilder(BuilderBase):
Expand Down
3 changes: 2 additions & 1 deletion build/fbcode_builder/getdeps/cargo.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import typing

from .builder import BuilderBase
from .copytree import simple_copytree

if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
Expand Down Expand Up @@ -79,7 +80,7 @@ def recreate_dir(self, src, dst) -> None:
os.remove(dst)
else:
shutil.rmtree(dst)
shutil.copytree(src, dst)
simple_copytree(src, dst)

def cargo_config_file(self):
build_source_dir = self.build_dir
Expand Down
42 changes: 26 additions & 16 deletions build/fbcode_builder/getdeps/copytree.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import subprocess

from .platform import is_windows
from .runcmd import run_cmd


PREFETCHED_DIRS = set()
Expand Down Expand Up @@ -64,19 +65,28 @@ def prefetch_dir_if_eden(dirpath) -> None:
subprocess.call(["edenfsctl", "prefetch", "--repo", root, glob, "--background"])
PREFETCHED_DIRS.add(dirpath)


# pyre-fixme[9]: ignore has type `bool`; used as `None`.
def copytree(src_dir, dest_dir, ignore: bool = None):
"""Recursively copy the src_dir to the dest_dir, filtering
out entries using the ignore lambda. The behavior of the
ignore lambda must match that described by `shutil.copytree`.
This `copytree` function knows how to prefetch data when
running in an eden repo.
TODO: I'd like to either extend this or add a variant that
uses watchman to mirror src_dir into dest_dir.
"""
prefetch_dir_if_eden(src_dir)
# pyre-fixme[6]: For 3rd param expected
# `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]],
# typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`.
return shutil.copytree(src_dir, dest_dir, ignore=ignore)
def simple_copytree(src_dir, dest_dir, symlinks=False):
"""A simple version of shutil.copytree() that can delegate to native tools if faster"""
if is_windows():
os.makedirs(dest_dir, exist_ok=True)
cmd = [
"robocopy.exe", src_dir, dest_dir,
# copy directories, including empty ones
"/E",
# Ignore Extra files in destination
"/XX",
# enable parallel copy
"/MT",
# be quiet
"/NFL", "/NDL", "/NJH", "/NJS", "/NP",
]
if symlinks:
cmd.append("/SL")
# robocopy exits with code 1 if it copied ok, hence allow_fail
# https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility
exit_code = run_cmd(cmd, allow_fail=True)
if exit_code > 1:
raise subprocess.CalledProcessError(p.returncode, cmd)
return dest_dir
else:
return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)

0 comments on commit 500071b

Please sign in to comment.