Skip to content

Commit

Permalink
speed up file copy on windows
Browse files Browse the repository at this point in the history
Summary:
getdeps windows file copy is very slow, speed it up

Test Plan:

Fetch
```
rm -rf z:/installed/perl-* # z:/build/perl-* z:/extracted/perl-*
time python ./build/fbcode_builder/getdeps.py fetch perl
```

Before, 40s:
```
Building on {distro=None, distro_vers=10, fb=off, fbsource=off, os=windows, shared_libs=off, test=on}
Assessing perl...
Installing Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl -> Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc

real    0m40.150s
user    0m0.000s
sys     0m0.000s
```

After:
```
$ rm -rf z:/installed/perl-*
04 Dec 2024 19:55:39
04 Dec 2024 19:55:39 [Exit: 0] alex@fridge:~/local/folly
$ time python ./build/fbcode_builder/getdeps.py build perl
04 Dec 2024 19:55:42
Building on {distro=None, distro_vers=10, fb=off, fbsource=off, os=windows, shared_libs=off, test=on}
Assessing perl...
Installing Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl -> Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc
---
+ robocopy.exe \
+      'Z:\extracted\perl-strawberry-perl-5.28.1.1-64bit-portable.zip\perl' \
+      'Z:\installed\perl-DRGAS42y_YT9hZddELBr4c7ZE50hkZy1Cybn7PLzwFc' \
+      /E \
+      /NFL \
+      /NDL \
+      /NJH \
+      /NJS \
+      NP


real    0m0.758s
user    0m0.000s
sys     0m0.000s
04 Dec 2024 19:55:43 [Exit: 0] alex@fridge:~/local/folly
```
  • Loading branch information
ahornby committed Dec 4, 2024
1 parent 6c162c0 commit 5ed41b0
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 3 deletions.
5 changes: 3 additions & 2 deletions build/fbcode_builder/getdeps/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from shlex import quote as shellquote
from typing import Optional

from .copytree import simple_copytree
from .dyndeps import create_dyn_dep_munger
from .envfuncs import add_path_entry, Env, path_search
from .fetcher import copy_if_different
Expand Down Expand Up @@ -1327,7 +1328,7 @@ def build(self, reconfigure: bool) -> None:
os.makedirs(dest_parent)
if os.path.isdir(full_src):
if not os.path.exists(full_dest):
shutil.copytree(full_src, full_dest)
simple_copytree(full_src, full_dest)
else:
shutil.copyfile(full_src, full_dest)
shutil.copymode(full_src, full_dest)
Expand All @@ -1339,7 +1340,7 @@ def build(self, reconfigure: bool) -> None:
os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
else:
if not os.path.exists(self.inst_dir):
shutil.copytree(self.src_dir, self.inst_dir)
simple_copytree(self.src_dir, self.inst_dir)


class SqliteBuilder(BuilderBase):
Expand Down
24 changes: 23 additions & 1 deletion build/fbcode_builder/getdeps/copytree.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import subprocess

from .platform import is_windows
from .runcmd import run_cmd


PREFETCHED_DIRS = set()
Expand Down Expand Up @@ -64,7 +65,6 @@ def prefetch_dir_if_eden(dirpath) -> None:
subprocess.call(["edenfsctl", "prefetch", "--repo", root, glob, "--background"])
PREFETCHED_DIRS.add(dirpath)


# pyre-fixme[9]: ignore has type `bool`; used as `None`.
def copytree(src_dir, dest_dir, ignore: bool = None):
"""Recursively copy the src_dir to the dest_dir, filtering
Expand All @@ -80,3 +80,25 @@ def copytree(src_dir, dest_dir, ignore: bool = None):
# `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]],
# typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`.
return shutil.copytree(src_dir, dest_dir, ignore=ignore)

def simple_copytree(src_dir, dest_dir, symlinks=False):
"""A simple version of shutil.copytree() that can delegate to native tools when its faster"""
if is_windows():
os.makedirs(dest_dir, exist_ok=True)
args = [
src_dir, dest_dir,
# copy directories, including empty ones
"/E",
# Ignore Extra files in destination
"/XX",
# enable parallel copy
"/MT:",
# be quiet
"/NFL", "/NDL", "/NJH", "/NJS", "/NP",
]
if symlinks:
args.append("/SL")
run_cmd(["robocopy.exe"] + args)
return dest_dir
else:
return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)

0 comments on commit 5ed41b0

Please sign in to comment.