From 59f7e0d4757a0c52866a5c7d62d6f1d6df023d96 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Fri, 7 Jun 2024 22:00:11 +0100 Subject: [PATCH 1/3] url: save -mu download to new cache location --- common/common.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index cdcb352b5a8ae..7a21081c1a0aa 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -191,6 +191,15 @@ int32_t cpu_get_num_math() { // CLI argument parsing // +static std::string get_cached_file(const std::string & path) { + std::string cache_directory = fs_get_cache_directory(); + const bool success = fs_create_directory_with_parents(cache_directory); + if (!success) { + throw std::runtime_error("failed to create cache directory: " + cache_directory); + } + return cache_directory + string_split(path, '/').back(); +} + void gpt_params_handle_model_default(gpt_params & params) { if (!params.hf_repo.empty()) { // short-hand to avoid specifying --hf-file -> default it to --model @@ -200,19 +209,13 @@ void gpt_params_handle_model_default(gpt_params & params) { } params.hf_file = params.model; } else if (params.model.empty()) { - std::string cache_directory = fs_get_cache_directory(); - const bool success = fs_create_directory_with_parents(cache_directory); - if (!success) { - throw std::runtime_error("failed to create cache directory: " + cache_directory); - } - params.model = cache_directory + string_split(params.hf_file, '/').back(); + params.model = get_cached_file(params.hf_file); } } else if (!params.model_url.empty()) { if (params.model.empty()) { auto f = string_split(params.model_url, '#').front(); f = string_split(f, '?').front(); - f = string_split(f, '/').back(); - params.model = "models/" + f; + params.model = get_cached_file(f); } } else if (params.model.empty()) { params.model = DEFAULT_MODEL_PATH; From bc5acfd124f36dcd8bd992825af1a4b988a7ab6b Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Sat, 8 Jun 2024 19:37:04 +0100 Subject: [PATCH 2/3] url: fs_get_cache_file_path util --- common/common.cpp | 22 +++++++++++----------- common/common.h | 1 + 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index ab4a0df68af55..65f69f9a3133f 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -191,15 +191,6 @@ int32_t cpu_get_num_math() { // CLI argument parsing // -static std::string get_cached_file(const std::string & path) { - std::string cache_directory = fs_get_cache_directory(); - const bool success = fs_create_directory_with_parents(cache_directory); - if (!success) { - throw std::runtime_error("failed to create cache directory: " + cache_directory); - } - return cache_directory + string_split(path, '/').back(); -} - void gpt_params_handle_model_default(gpt_params & params) { if (!params.hf_repo.empty()) { // short-hand to avoid specifying --hf-file -> default it to --model @@ -209,13 +200,13 @@ void gpt_params_handle_model_default(gpt_params & params) { } params.hf_file = params.model; } else if (params.model.empty()) { - params.model = get_cached_file(params.hf_file); + params.model = fs_get_cache_file_path(params.hf_file); } } else if (!params.model_url.empty()) { if (params.model.empty()) { auto f = string_split(params.model_url, '#').front(); f = string_split(f, '?').front(); - params.model = get_cached_file(f); + params.model = fs_get_cache_file_path(f); } } else if (params.model.empty()) { params.model = DEFAULT_MODEL_PATH; @@ -2282,6 +2273,15 @@ std::string fs_get_cache_directory() { return ensure_trailing_slash(cache_directory); } +std::string fs_get_cache_file_path(const std::string & path) { + std::string cache_directory = fs_get_cache_directory(); + const bool success = fs_create_directory_with_parents(cache_directory); + if (!success) { + throw std::runtime_error("failed to create cache directory: " + cache_directory); + } + return cache_directory + string_split(path, '/').back(); +} + // // Model utils diff --git a/common/common.h b/common/common.h index 038f9084f3424..304597dea460f 100644 --- a/common/common.h +++ b/common/common.h @@ -277,6 +277,7 @@ bool fs_validate_filename(const std::string & filename); bool fs_create_directory_with_parents(const std::string & path); std::string fs_get_cache_directory(); +std::string fs_get_cache_file_path(const std::string & path); // // Model utils From 76612a27be2d5e5d941b1555cd75a66e89938efa Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Sat, 8 Jun 2024 19:50:30 +0100 Subject: [PATCH 3/3] url: tweak sig of fs_get_cache_file --- common/common.cpp | 9 +++++---- common/common.h | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 65f69f9a3133f..1591790e6df4c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -200,13 +200,13 @@ void gpt_params_handle_model_default(gpt_params & params) { } params.hf_file = params.model; } else if (params.model.empty()) { - params.model = fs_get_cache_file_path(params.hf_file); + params.model = fs_get_cache_file(string_split(params.hf_file, '/').back()); } } else if (!params.model_url.empty()) { if (params.model.empty()) { auto f = string_split(params.model_url, '#').front(); f = string_split(f, '?').front(); - params.model = fs_get_cache_file_path(f); + params.model = fs_get_cache_file(string_split(f, '/').back()); } } else if (params.model.empty()) { params.model = DEFAULT_MODEL_PATH; @@ -2273,13 +2273,14 @@ std::string fs_get_cache_directory() { return ensure_trailing_slash(cache_directory); } -std::string fs_get_cache_file_path(const std::string & path) { +std::string fs_get_cache_file(const std::string & filename) { + GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos); std::string cache_directory = fs_get_cache_directory(); const bool success = fs_create_directory_with_parents(cache_directory); if (!success) { throw std::runtime_error("failed to create cache directory: " + cache_directory); } - return cache_directory + string_split(path, '/').back(); + return cache_directory + filename; } diff --git a/common/common.h b/common/common.h index 304597dea460f..2345d855eed3c 100644 --- a/common/common.h +++ b/common/common.h @@ -277,7 +277,7 @@ bool fs_validate_filename(const std::string & filename); bool fs_create_directory_with_parents(const std::string & path); std::string fs_get_cache_directory(); -std::string fs_get_cache_file_path(const std::string & path); +std::string fs_get_cache_file(const std::string & filename); // // Model utils