Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement repo import for prepackged CRS packages #140

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Implement repo import for prepackged CRS packages #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 33 additions & 26 deletions src/bpt/cli/cmd/repo_import.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,14 @@ namespace bpt::cli::cmd {

namespace {

void _import_file(bpt::crs::repository& repo, path_ref dirpath) { repo.import_dir(dirpath); }
void _import_file(bpt::crs::repository& repo, path_ref pkgpath) {
auto abs_path = fs::canonical(pkgpath);
if (fs::is_regular_file(abs_path)) {
repo.import_targz(pkgpath);
} else {
repo.import_dir(pkgpath);
}
}

int _repo_import(const options& opts) {
auto repo = bpt::crs::repository::open_existing(opts.repo.repo_dir);
Expand Down Expand Up @@ -69,22 +76,22 @@ int repo_import(const options& opts) {
return bpt_leaf_try { return _repo_import(opts); }
bpt_leaf_catch(bpt::crs::e_repo_import_pkg_already_present,
bpt::crs::e_repo_importing_package meta,
bpt::crs::e_repo_importing_dir from_dir) {
bpt::crs::e_repo_importing_path from_path) {
bpt_log(
error,
"Refusing to overwrite existing package .br.yellow[{}] (Importing from [.br.yellow[{}]])"_styled,
meta.value.id.to_string(),
from_dir.value.string());
from_path.value.string());
write_error_marker("repo-import-pkg-already-exists");
return 1;
}
bpt_leaf_catch(bpt::crs::e_repo_importing_dir crs_dir,
std::error_code ec,
bpt_leaf_catch(bpt::crs::e_repo_importing_path crs_path,
std::error_code ec,
boost::leaf::match<std::error_code, boost::leaf::category<neo::sqlite3::errc>>,
e_sqlite3_error const* sqlite_error) {
bpt_log(error,
"SQLite error while importing [.br.yellow[{}]]"_styled,
crs_dir.value.string());
crs_path.value.string());
bpt_log(error, " .br.red[{}]"_styled, ec.message());
if (sqlite_error) {
bpt_log(error, " .br.red[{}]"_styled, sqlite_error->value);
Expand All @@ -93,33 +100,33 @@ int repo_import(const options& opts) {
write_error_marker("repo-import-db-error");
return 1;
}
bpt_leaf_catch(bpt::e_json_parse_error parse_error,
bpt::crs::e_repo_importing_dir crs_dir,
bpt::crs::e_pkg_json_path pkg_json_path) {
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_dir.value.string());
bpt_leaf_catch(bpt::e_json_parse_error parse_error,
bpt::crs::e_repo_importing_path crs_path,
bpt::crs::e_pkg_json_path pkg_json_path) {
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_path.value.string());
bpt_log(error,
" JSON parse error in [.br.yellow[{}]]:"_styled,
pkg_json_path.value.string());
bpt_log(error, " .br.red[{}]"_styled, parse_error.value);
write_error_marker("repo-import-invalid-crs-json-parse-error");
return 1;
}
bpt_leaf_catch(bpt::crs::e_invalid_meta_data error,
bpt::crs::e_repo_importing_dir crs_dir,
bpt::crs::e_pkg_json_path pkg_json_path) {
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_dir.value.string());
bpt_leaf_catch(bpt::crs::e_invalid_meta_data error,
bpt::crs::e_repo_importing_path crs_path,
bpt::crs::e_pkg_json_path pkg_json_path) {
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, crs_path.value.string());
bpt_log(error,
"CRS data in [.br.yellow[{}]] is invalid: .br.red[{}]"_styled,
pkg_json_path.value.string(),
error.value);
write_error_marker("repo-import-invalid-crs-json");
return 1;
}
bpt_leaf_catch(bpt::crs::e_invalid_meta_data error,
bpt::crs::e_repo_importing_dir proj_dir,
bpt_leaf_catch(bpt::crs::e_invalid_meta_data error,
bpt::crs::e_repo_importing_path proj_path,
bpt::e_open_project,
bpt::e_parse_project_manifest_path proj_json_path) {
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, proj_dir.value.string());
bpt_log(error, "Error while importing [.br.yellow[{}]]"_styled, proj_path.value.string());
bpt_log(error,
"Project data in [.br.yellow[{}]] is invalid: .br.red[{}]"_styled,
proj_json_path.value.string(),
Expand All @@ -128,19 +135,19 @@ int repo_import(const options& opts) {
return 1;
}
bpt_leaf_catch(user_error<errc::invalid_pkg_filesystem> const& exc,
crs::e_repo_importing_dir crs_dir) {
crs::e_repo_importing_path crs_path) {
bpt_log(error,
"Error while importing [.br.yellow[{}]]: .br.red[{}]"_styled,
crs_dir.value.string(),
crs_path.value.string(),
exc.what());
write_error_marker("repo-import-noent");
return 1;
}
bpt_leaf_catch(bpt::crs::e_repo_importing_dir crs_dir,
bpt::e_read_file_path const* read_file,
bpt::e_copy_file const* copy_file,
std::error_code ec) {
bpt_log(error, "Error while importing [.br.red[{}]]:"_styled, crs_dir.value.string());
bpt_leaf_catch(bpt::crs::e_repo_importing_path crs_path,
bpt::e_read_file_path const* read_file,
bpt::e_copy_file const* copy_file,
std::error_code ec) {
bpt_log(error, "Error while importing [.br.red[{}]]:"_styled, crs_path.value.string());
if (read_file) {
bpt_log(error,
" Error reading file [.br.yellow[{}]]:"_styled,
Expand All @@ -156,13 +163,13 @@ int repo_import(const options& opts) {
write_error_marker("repo-import-noent");
return 1;
}
bpt_leaf_catch(crs::e_repo_importing_dir crs_dir,
bpt_leaf_catch(crs::e_repo_importing_path crs_path,
crs::e_repo_importing_package meta,
crs::e_repo_import_invalid_pkg_version err) {
bpt_log(info,
"Error while importing .br.yellow[{}] (from [.br.yellow[{}]]):"_styled,
meta.value.id.to_string(),
crs_dir.value.string());
crs_path.value.string());
bpt_log(error, "Invalid 'pkg-version' on package: .br.red[{}]"_styled, err.value);
write_error_marker("repo-import-invalid-pkg-version");
return 1;
Expand Down
4 changes: 2 additions & 2 deletions src/bpt/crs/error.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ struct e_repo_open_path {
std::filesystem::path value;
};

struct e_repo_importing_dir {
struct e_repo_importing_path {
std::filesystem::path value;
};

Expand All @@ -32,4 +32,4 @@ struct e_sync_remote {
neo::url value;
};

} // namespace bpt::crs
} // namespace bpt::crs
109 changes: 69 additions & 40 deletions src/bpt/crs/repo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
#include <neo/tar/util.hpp>
#include <neo/ufmt.hpp>

#include <fstream>

using namespace bpt;
using namespace bpt::crs;
using namespace neo::sqlite3::literals;
Expand Down Expand Up @@ -87,6 +89,17 @@ void archive_package_libraries(path_ref from_dir, path_ref dest_root, const pack
}
}

void expand_tgz(path_ref tgz_path, path_ref into) {
auto infile = bpt::open_file(tgz_path, std::ios::binary | std::ios::in);
fs::create_directories(into);
neo::expand_directory_targz(
neo::expand_options{
.destination_directory = into,
.input_name = tgz_path.string(),
},
infile);
}

} // namespace

void repository::_vacuum_and_compress() const {
Expand All @@ -97,6 +110,51 @@ void repository::_vacuum_and_compress() const {
bpt::compress_file_gz(_dirpath / "repo.db", _dirpath / "repo.db.gz").value();
}

package_info repository::_import_dir(const std::filesystem::path& dirpath) {
auto sd = sdist::from_directory(dirpath);
auto& pkg = sd.pkg;
BPT_E_SCOPE(e_repo_importing_package{pkg});
if (pkg.id.revision < 1) {
BOOST_LEAF_THROW_EXCEPTION(e_repo_import_invalid_pkg_version{
"Package pkg-version must be a positive non-zero integer in order to be imported into "
"a repository"});
}

// Copy the package into a temporary directory
fs::create_directories(tmp_dir());
auto prep_dir = bpt::temporary_dir::create_in(tmp_dir());
archive_package_libraries(dirpath, prep_dir.path(), pkg);
fs::create_directories(prep_dir.path());
bpt::write_file(prep_dir.path() / "pkg.json", pkg.to_json(2));

auto tmp_tgz = tmp_dir() / (prep_dir.path().filename().string() + ".tgz");
neo::compress_directory_targz(prep_dir.path(), tmp_tgz);
neo_defer { std::ignore = ensure_absent(tmp_tgz); };

neo::sqlite3::transaction_guard tr{_db.sqlite3_db()};
bpt_leaf_try {
db_exec( //
_prepare(R"(
INSERT INTO crs_repo_packages (meta_json)
VALUES (?)
)"_sql),
std::string_view(pkg.to_json()))
.value();
}
bpt_leaf_catch(matchv<neo::sqlite3::errc::constraint_unique>) {
BOOST_LEAF_THROW_EXCEPTION(current_error(), e_repo_import_pkg_already_present{});
};

auto dest_dir = subdir_of(pkg);
fs::create_directories(dest_dir);
move_file(tmp_tgz, dest_dir / "pkg.tgz").value();
bpt::copy_file(prep_dir.path() / "pkg.json", dest_dir / "pkg.json").value();
tr.commit();
_vacuum_and_compress();

return pkg;
}

repository repository::create(path_ref dirpath, std::string_view name) {
BPT_E_SCOPE(e_repo_open_path{dirpath});
std::filesystem::create_directories(dirpath);
Expand Down Expand Up @@ -135,48 +193,19 @@ fs::path repository::subdir_of(const package_info& pkg) const noexcept {
/ neo::ufmt("{}~{}", pkg.id.version.to_string(), pkg.id.revision);
}

void repository::import_dir(path_ref dirpath) {
BPT_E_SCOPE(e_repo_importing_dir{dirpath});
auto sd = sdist::from_directory(dirpath);
auto& pkg = sd.pkg;
BPT_E_SCOPE(e_repo_importing_package{pkg});
auto dest_dir = subdir_of(pkg);
fs::create_directories(dest_dir);
void repository::import_targz(path_ref tgz_path) {
BPT_E_SCOPE(e_repo_importing_path{tgz_path});
fs::create_directories(tmp_dir());
auto tmp_pkg_dir = bpt::temporary_dir::create_in(tmp_dir());
expand_tgz(tgz_path, tmp_pkg_dir.path());
auto pkg = _import_dir(tmp_pkg_dir.path());

// Copy the package into a temporary directory
auto prep_dir = bpt::temporary_dir::create_in(dest_dir);
archive_package_libraries(dirpath, prep_dir.path(), pkg);
fs::create_directories(prep_dir.path());
bpt::write_file(prep_dir.path() / "pkg.json", pkg.to_json(2));

auto tmp_tgz = pkg_dir() / (prep_dir.path().filename().string() + ".tgz");
neo::compress_directory_targz(prep_dir.path(), tmp_tgz);
neo_defer { std::ignore = ensure_absent(tmp_tgz); };

if (pkg.id.revision < 1) {
BOOST_LEAF_THROW_EXCEPTION(e_repo_import_invalid_pkg_version{
"Package pkg-version must be a positive non-zero integer in order to be imported into "
"a repository"});
}

neo::sqlite3::transaction_guard tr{_db.sqlite3_db()};
bpt_leaf_try {
db_exec( //
_prepare(R"(
INSERT INTO crs_repo_packages (meta_json)
VALUES (?)
)"_sql),
std::string_view(pkg.to_json()))
.value();
}
bpt_leaf_catch(matchv<neo::sqlite3::errc::constraint_unique>) {
BOOST_LEAF_THROW_EXCEPTION(current_error(), e_repo_import_pkg_already_present{});
};
NEO_EMIT(ev_repo_imported_package{*this, tgz_path, pkg});
}

move_file(tmp_tgz, dest_dir / "pkg.tgz").value();
bpt::copy_file(prep_dir.path() / "pkg.json", dest_dir / "pkg.json").value();
tr.commit();
_vacuum_and_compress();
void repository::import_dir(path_ref dirpath) {
BPT_E_SCOPE(e_repo_importing_path{dirpath});
auto pkg = _import_dir(dirpath);

NEO_EMIT(ev_repo_imported_package{*this, dirpath, pkg});
}
Expand Down
4 changes: 3 additions & 1 deletion src/bpt/crs/repo.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class repository {
, _dirpath(dirpath) {}

void _vacuum_and_compress() const;
package_info _import_dir(const std::filesystem::path& dirpath);

public:
static repository create(const std::filesystem::path& directory, std::string_view name);
Expand All @@ -31,6 +32,7 @@ class repository {
std::filesystem::path subdir_of(const package_info&) const noexcept;

auto pkg_dir() const noexcept { return _dirpath / "pkg"; }
auto tmp_dir() const noexcept { return _dirpath / "tmp"; }
auto& root() const noexcept { return _dirpath; }
std::string name() const;

Expand All @@ -49,4 +51,4 @@ struct ev_repo_imported_package {
package_info const& pkg_meta;
};

} // namespace bpt::crs
} // namespace bpt::crs