diff --git a/python/setup.py b/python/setup.py
index 5b9915e4851a..a279bb7555b4 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -599,9 +599,14 @@ def build(build_python, build_java, build_cpp):
     )
 
 
-def walk_directory(directory):
+def _walk_thirdparty_dir(directory):
     file_list = []
     for root, dirs, filenames in os.walk(directory):
+        # Exclude generated bytecode cache directories and tests directories
+        # from vendored packages.
+        for exclude_dir in ["__pycache__", "tests"]:
+            if exclude_dir in dirs:
+                dirs.remove(exclude_dir)
         for name in filenames:
             file_list.append(os.path.join(root, name))
     return file_list
@@ -611,7 +616,7 @@ def copy_file(target_dir, filename, rootdir):
     # TODO(rkn): This feels very brittle. It may not handle all cases. See
     # https://github.com/apache/arrow/blob/master/python/setup.py for an
     # example.
-    # File names can be absolute paths, e.g. from walk_directory().
+    # File names can be absolute paths, e.g. from _walk_thirdparty_dir().
     source = os.path.relpath(filename, rootdir)
     destination = os.path.join(target_dir, source)
     # Create the target directory if it doesn't already exist.
@@ -650,12 +655,14 @@ def pip_run(build_ext):
         setup_spec.files_to_include += ray_files
 
         thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
-        setup_spec.files_to_include += walk_directory(thirdparty_dir)
+        setup_spec.files_to_include += _walk_thirdparty_dir(thirdparty_dir)
 
         runtime_env_agent_thirdparty_dir = os.path.join(
             ROOT_DIR, RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR
         )
-        setup_spec.files_to_include += walk_directory(runtime_env_agent_thirdparty_dir)
+        setup_spec.files_to_include += _walk_thirdparty_dir(
+            runtime_env_agent_thirdparty_dir
+        )
 
         # Copy over the autogenerated protobuf Python bindings.
         for directory in generated_python_directories: