[parsing] Add support for remote packages

TODO needs mutex for const function

BUILD.bazel

@@ -89,6 +89,7 @@ install(
         "//geometry:install",
         "//lcmtypes:install",
         "//manipulation/models:install_data",
+        "//multibody/parsing:install",
         "//setup:install",
         "//tools/install/libdrake:install",
         "//tools/workspace:install_external_packages",

bindings/pydrake/multibody/parsing_py.cc

@@ -32,12 +32,25 @@ PYBIND11_MODULE(parsing, m) {
     using Class = PackageMap;
     constexpr auto& cls_doc = doc.PackageMap;
     py::class_<Class> cls(m, "PackageMap", cls_doc.doc);
+    {
+      using Nested = PackageMap::RemotePackageParams;
+      constexpr auto& nested_doc = cls_doc.RemotePackageParams;
+      py::class_<Nested> nested(cls, "RemotePackageParams", nested_doc.doc);
+      nested.def(ParamInit<Nested>());
+      DefAttributesUsingSerialize(&nested, nested_doc);
+      DefReprUsingSerialize(&nested);
+      DefCopyAndDeepCopy(&nested);
+    }
     cls  // BR
         .def(py::init<>(), cls_doc.ctor.doc)
         .def(py::init<const Class&>(), py::arg("other"), "Copy constructor")
         .def("Add", &Class::Add, py::arg("package_name"),
             py::arg("package_path"), cls_doc.Add.doc)
         .def("AddMap", &Class::AddMap, py::arg("other_map"), cls_doc.AddMap.doc)
+        .def("AddPackageXml", &Class::AddPackageXml, py::arg("filename"),
+            cls_doc.AddPackageXml.doc)
+        .def("AddRemotePackage", &Class::AddRemotePackage,
+            py::arg("package_name"), py::arg("params"))
         .def("Contains", &Class::Contains, py::arg("package_name"),
             cls_doc.Contains.doc)
         .def("Remove", &Class::Remove, py::arg("package_name"),
@@ -53,8 +66,6 @@ PYBIND11_MODULE(parsing, m) {
               return self.GetPath(package_name);
             },
             py::arg("package_name"), cls_doc.GetPath.doc)
-        .def("AddPackageXml", &Class::AddPackageXml, py::arg("filename"),
-            cls_doc.AddPackageXml.doc)
         .def("PopulateFromFolder", &Class::PopulateFromFolder, py::arg("path"),
             cls_doc.PopulateFromFolder.doc)
         .def("PopulateFromEnvironment", &Class::PopulateFromEnvironment,

bindings/pydrake/multibody/test/parsing_test.py

@@ -68,6 +68,16 @@ def test_package_map(self):
         dut.PopulateFromEnvironment(environment_variable='TEST_TMPDIR')
         dut.PopulateFromFolder(path=tmpdir)
 
+        # Simple coverage for remote packages (and their Params).
+        params = PackageMap.RemotePackageParams()
+        params.urls = ["file:///tmp/does-not-exist.zip"]
+        params.sha256 = "0" * 64
+        params.archive_type = "zip"
+        params.strip_prefix = "prefix"
+        dut.AddRemotePackage(package_name="remote", params=params)
+        with self.assertRaisesRegex(RuntimeError, "downloader.*error"):
+            dut.GetPath("remote")
+
     def test_parser_file(self):
         """Calls every combination of arguments for the Parser methods which
         use a file_name (not contents) and inspects their return type.

multibody/parsing/BUILD.bazel

@@ -12,6 +12,7 @@ load(
     "drake_py_library",
     "drake_py_unittest",
 )
+load("@drake//tools/install:install.bzl", "install_files")
 load("//tools/lint:lint.bzl", "add_lint_tests")
 load("@drake//tools/workspace:forward_files.bzl", "forward_files")
 load(
@@ -60,15 +61,22 @@ drake_cc_library(
     srcs = ["package_map.cc"],
     hdrs = ["package_map.h"],
     data = [
+        ":package_downloader.py",
         "//:package.xml",
         "@models_internal//:package.xml",
     ],
     visibility = ["//visibility:public"],
     interface_deps = [
         "//common:essential",
+        "//common:name_value",
     ],
     deps = [
-        "//common",
+        "//common:find_cache",
+        "//common:find_resource",
+        "//common:find_runfiles",
+        "//common:scope_exit",
+        "//common:unused",
+        "//common/yaml",
         "@tinyxml2_internal//:tinyxml2",
     ],
 )
@@ -615,6 +623,20 @@ drake_cc_googletest(
     ],
 )
 
+drake_cc_googletest(
+    name = "package_map_remote_test",
+    data = [
+        "test/package_map_test_packages/compressed.zip",
+    ],
+    deps = [
+        ":package_map",
+        "//common:find_cache",
+        "//common:find_resource",
+        "//common:scope_exit",
+        "//common/test_utilities:expect_throws_message",
+    ],
+)
+
 drake_cc_googletest(
     name = "drake_manifest_resolution_test",
     data = [
@@ -663,4 +685,11 @@ drake_cc_googletest(
     ],
 )
 
+install_files(
+    name = "install",
+    dest = "share/drake/multibody/parsing",
+    files = ["package_downloader.py"],
+    visibility = ["//visibility:public"],
+)
+
 add_lint_tests()

multibody/parsing/package_map.cc

@@ -1,6 +1,9 @@
 #include "drake/multibody/parsing/package_map.h"
 
+#include <unistd.h>
+
 #include <algorithm>
+#include <cctype>
 #include <cstdlib>
 #include <filesystem>
 #include <initializer_list>
@@ -14,13 +17,15 @@
 #include <drake_vendor/tinyxml2.h>
 
 #include "drake/common/drake_assert.h"
-#include "drake/common/drake_path.h"
 #include "drake/common/drake_throw.h"
+#include "drake/common/find_cache.h"
 #include "drake/common/find_resource.h"
 #include "drake/common/find_runfiles.h"
 #include "drake/common/never_destroyed.h"
+#include "drake/common/scope_exit.h"
 #include "drake/common/text_logging.h"
 #include "drake/common/unused.h"
+#include "drake/common/yaml/yaml_io.h"
 
 namespace drake {
 namespace multibody {
@@ -31,13 +36,108 @@ using tinyxml2::XMLDocument;
 using tinyxml2::XMLElement;
 
 namespace {
+
 struct PackageData {
-  /* Directory in which the manifest resides. */
+  // XXX FetchStatus is_fetched;
+
+  /* Directory in which the manifest resides. If this is an undownloaded
+  remote package, this path will be empty until it is downloaded. */
   std::string path;
 
   /* Optional message declaring deprecation of the package. */
   std::optional<std::string> deprecated_message;
+
+  /* Iff this is a remote package, this will contain the details. */
+  std::optional<PackageMap::RemotePackageParams> remote;
+
+  // Below, we have a few tiny "summary" getters for the struct fields,
+  // to make the call sites a bit more readable.
+
+  /* Returns true iff this package exists on disk (i.e., iff the `path`
+  has been set). */
+  bool is_fetched() const { return path.size() > 0; }
+
+  /* Returns `path` if it is known, or else (for unfetched packages) the
+  first of the `remote->urls` . */
+  std::string printable_path() const {
+    return !path.empty() ? path : remote.value().urls.front();
+  }
+};
+
+struct DownloaderArgs {
+  template <typename Archive>
+  void Serialize(Archive* a) {
+    a->Visit(DRAKE_NVP(package_name));
+    remote.Serialize(a);
+    a->Visit(DRAKE_NVP(output_dir));
+  }
+  std::string package_name;
+  PackageMap::RemotePackageParams remote;
+  std::string output_dir;
 };
+
+void FetchContent(std::string_view package_name, PackageData* data) {
+  DRAKE_DEMAND(!package_name.empty());
+  DRAKE_DEMAND(data != nullptr);
+  DRAKE_DEMAND(!data->is_fetched());
+  DRAKE_DEMAND(data->remote.has_value());
+  DRAKE_DEMAND(data->path.empty());
+
+  // Find and/or create the cache_dir.
+  auto try_cache = internal::FindOrCreateCache("package_map");
+  if (!try_cache.error.empty()) {
+    throw std::runtime_error(fmt::format(
+        "PackageMap: when downloading '{}', could not create temporary cache "
+        "directory: {}",
+        package_name, try_cache.error));
+  }
+  const fs::path cache_dir = std::move(try_cache.abspath);
+
+  // See if the package has already been fetched.
+  const fs::path package_dir = cache_dir / data->remote->sha256;
+  std::error_code ec;
+  if (fs::is_directory(package_dir, ec)) {
+    data->path = package_dir.string();
+    return;
+  }
+
+  // Write the downloader arguments to a JSON file.
+  DownloaderArgs args{.package_name = std::string(package_name),
+                      .remote = *data->remote,
+                      .output_dir = package_dir.string()};
+  std::string json_filename = fs::path(cache_dir / ".fetch_XXXXXX").string();
+  const int temp_fd = ::mkstemp(json_filename.data());
+  ::close(temp_fd);
+  ScopeExit remove_yaml([&json_filename]() {
+    fs::remove(json_filename);
+  });
+  yaml::SaveJsonFile(json_filename, args);
+
+  // Shell out to the downloader to fetch the package.
+  const std::string downloader =
+      FindResourceOrThrow("drake/multibody/parsing/package_downloader.py");
+  const std::string command =
+      fmt::format("/usr/bin/python3 {} {}", downloader, json_filename);
+  const int returncode = std::system(command.c_str());
+  if (returncode != 0) {
+    throw std::runtime_error(fmt::format(
+        "PackageMap: when downloading '{}', the downloader experienced an "
+        "error",
+        package_name));
+  }
+
+  // Confirm that it actually fetched.
+  if (!fs::is_directory(package_dir, ec)) {
+    throw std::runtime_error(fmt::format(
+        "PackageMap: when downloading '{}', the downloader claimed success but "
+        "somehow did not actually download anything?!",
+        package_name));
+  }
+
+  // Success
+  data->path = package_dir.string();
+}
+
 }  // namespace
 
 struct PackageMap::Impl {
@@ -111,9 +211,75 @@ void PackageMap::Add(const std::string& package_name,
 
 void PackageMap::AddMap(const PackageMap& other_map) {
   for (const auto& [package_name, data] : other_map.impl_->map) {
-    Add(package_name, data.path);
-    SetDeprecated(package_name, data.deprecated_message);
+    if (data.is_fetched()) {
+      Add(package_name, data.path);
+      SetDeprecated(package_name, data.deprecated_message);
+    } else {
+      DRAKE_DEMAND(data.remote.has_value());
+      AddRemotePackage(package_name, *data.remote);
+    }
+  }
+}
+
+void PackageMap::AddRemotePackage(std::string package_name,
+                                  RemotePackageParams params) {
+  // Validate our arguments.
+  auto iter = impl_->map.find(package_name);
+  if (iter != impl_->map.end()) {
+    // Adding a 100% identical package is supported (and a no-op).
+    // Otherwise, it's an error.
+    if (iter->second.remote.has_value()) {
+      const RemotePackageParams& old_params = iter->second.remote.value();
+      const std::string old_json = yaml::SaveJsonString(old_params);
+      const std::string new_json = yaml::SaveJsonString(params);
+      if (new_json == old_json) {
+        drake::log()->trace("AddRemotePackage skipping duplicate '{}'",
+                            package_name);
+        return;
+      }
+    }
+    throw std::logic_error(fmt::format(
+        "PackageMap::AddRemotePackage cannot add '{}' because a package of "
+        "that name has already been registered",
+        package_name));
+  }
+  if (params.urls.empty()) {
+    throw std::logic_error(fmt::format(
+        "PackageMap::AddRemotePackage on '{}' requires at least one URL",
+        package_name));
+  }
+  for (const std::string_view url : params.urls) {
+    if (!((url.substr(0, 8) == "https://") || (url.substr(0, 7) == "http://") ||
+          (url.substr(0, 7) == "file://"))) {
+      throw std::logic_error(fmt::format(
+          "PackageMap::AddRemotePackage on '{}' used an unsupported URL '{}'",
+          package_name, url));
+    }
   }
+  if (!((params.sha256.size() == 64) &&
+        (std::all_of(params.sha256.begin(), params.sha256.end(), [](char ch) {
+          return std::isxdigit(ch);
+        })))) {
+    throw std::logic_error(fmt::format(
+        "PackageMap::AddRemotePackage on '{}' with invalid sha256 '{}'",
+        package_name, params.sha256));
+  }
+  if (params.archive_type.has_value()) {
+    const std::initializer_list<const char*> known_types = {
+        "zip", "tar", "gztar", "bztar", "xztar"};
+    if (std::count(known_types.begin(), known_types.end(),
+                   *params.archive_type) == 0) {
+      throw std::logic_error(fmt::format(
+          "PackageMap::AddRemotePackage on '{}' has unsupported archive "
+          "type '{}'",
+          package_name, *params.archive_type));
+    }
+  }
+
+  // Everything checks out, so we can add it now.
+  PackageData data;
+  data.remote = std::move(params);
+  impl_->map.emplace_hint(iter, std::move(package_name), std::move(data));
 }
 
 bool PackageMap::Contains(const std::string& package_name) const {
@@ -179,6 +345,12 @@ const std::string& PackageMap::GetPath(
     drake::log()->warn("PackageMap: {}", *warning);
   }
 
+  // If this is a remote package and we haven't fetched it yet, do that now.
+  if (!package_data.is_fetched()) {
+    FetchContent(package_name, const_cast<PackageData*>(&package_data));
+  }
+
+  DRAKE_DEMAND(!package_data.path.empty());
   return package_data.path;
 }
 
@@ -313,15 +485,17 @@ bool PackageMap::AddPackageIfNew(const std::string& package_name,
           "does not exist",
           package_name, path));
     }
-    impl_->map.insert(make_pair(package_name, PackageData{path}));
+    PackageData data;
+    data.path = path;
+    impl_->map.emplace(package_name, data);
   } else {
     // Don't warn if we've found the same path with a different spelling.
     const PackageData existing_data = impl_->map.at(package_name);
     if (!fs::equivalent(existing_data.path, path)) {
       drake::log()->warn(
           "PackageMap is ignoring newly-found path \"{}\" for package \"{}\""
           " and will continue using the previously-known path at \"{}\".",
-          path, package_name, existing_data.path);
+          path, package_name, existing_data.printable_path());
       return false;
     }
   }
@@ -383,7 +557,7 @@ std::ostream& operator<<(std::ostream& out, const PackageMap& package_map) {
     out << "  [EMPTY!]\n";
   }
   for (const auto& [package_name, data] : package_map.impl_->map) {
-    out << "  - " << package_name << ": " << data.path << "\n";
+    out << "  - " << package_name << ": " << data.printable_path() << "\n";
   }
   return out;
 }