From 1a83e26d72eb2601798dfc07e6f5112964dc9413 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:26 +0100 Subject: [PATCH 1/9] refs: include committer info in `ref_update` struct The reference backends obtain the committer information from `git_committer_info(0)` when adding a reflog. The upcoming patches introduce support for migrating reflogs between the reference backends. This requires an interface to creating reflogs, including custom committer information. Add a new field `committer_info` to the `ref_update` struct, which is then used by the reference backends. If there is no `committer_info` provided, the reference backends default to using `git_committer_info(0)`. The field itself cannot be set to `git_committer_info(0)` since the values are dynamic and must be obtained right when the reflog is being committed. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs.c | 1 + refs/files-backend.c | 24 ++++++++++++++---------- refs/refs-internal.h | 1 + refs/reftable-backend.c | 12 +++++++++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/refs.c b/refs.c index 762f3e324d59c6..f003e51c6bf522 100644 --- a/refs.c +++ b/refs.c @@ -1151,6 +1151,7 @@ void ref_transaction_free(struct ref_transaction *transaction) for (i = 0; i < transaction->nr; i++) { free(transaction->updates[i]->msg); + free(transaction->updates[i]->committer_info); free((char *)transaction->updates[i]->new_target); free((char *)transaction->updates[i]->old_target); free(transaction->updates[i]); diff --git a/refs/files-backend.c b/refs/files-backend.c index 64f51f0da905a9..6078668c99ee25 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1858,6 +1858,9 @@ static int log_ref_write_fd(int fd, const struct object_id *old_oid, struct strbuf sb = STRBUF_INIT; int ret = 0; + if (!committer) + committer = git_committer_info(0); + strbuf_addf(&sb, "%s %s %s", oid_to_hex(old_oid), oid_to_hex(new_oid), committer); if (msg && *msg) { strbuf_addch(&sb, '\t'); @@ -1871,8 +1874,10 @@ static int log_ref_write_fd(int fd, const struct object_id *old_oid, } static int files_log_ref_write(struct files_ref_store *refs, - const char *refname, const struct object_id *old_oid, - const struct object_id *new_oid, const char *msg, + const char *refname, + const struct object_id *old_oid, + const struct object_id *new_oid, + const char *committer_info, const char *msg, int flags, struct strbuf *err) { int logfd, result; @@ -1889,8 +1894,7 @@ static int files_log_ref_write(struct files_ref_store *refs, if (logfd < 0) return 0; - result = log_ref_write_fd(logfd, old_oid, new_oid, - git_committer_info(0), msg); + result = log_ref_write_fd(logfd, old_oid, new_oid, committer_info, msg); if (result) { struct strbuf sb = STRBUF_INIT; int save_errno = errno; @@ -1974,8 +1978,7 @@ static int commit_ref_update(struct files_ref_store *refs, files_assert_main_repository(refs, "commit_ref_update"); clear_loose_ref_cache(refs); - if (files_log_ref_write(refs, lock->ref_name, - &lock->old_oid, oid, + if (files_log_ref_write(refs, lock->ref_name, &lock->old_oid, oid, NULL, logmsg, flags, err)) { char *old_msg = strbuf_detach(err, NULL); strbuf_addf(err, "cannot update the ref '%s': %s", @@ -2007,9 +2010,9 @@ static int commit_ref_update(struct files_ref_store *refs, if (head_ref && (head_flag & REF_ISSYMREF) && !strcmp(head_ref, lock->ref_name)) { struct strbuf log_err = STRBUF_INIT; - if (files_log_ref_write(refs, "HEAD", - &lock->old_oid, oid, - logmsg, flags, &log_err)) { + if (files_log_ref_write(refs, "HEAD", &lock->old_oid, + oid, NULL, logmsg, flags, + &log_err)) { error("%s", log_err.buf); strbuf_release(&log_err); } @@ -2969,7 +2972,8 @@ static int parse_and_write_reflog(struct files_ref_store *refs, } if (files_log_ref_write(refs, lock->ref_name, &lock->old_oid, - &update->new_oid, update->msg, update->flags, err)) { + &update->new_oid, update->committer_info, + update->msg, update->flags, err)) { char *old_msg = strbuf_detach(err, NULL); strbuf_addf(err, "cannot update the ref '%s': %s", diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 58aa56d1b27c85..0fd95cdacd99e4 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -113,6 +113,7 @@ struct ref_update { void *backend_data; unsigned int type; char *msg; + char *committer_info; /* * If this ref_update was split off of a symref update via diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index 647ef9b05b1dc9..e882602487c662 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -1379,11 +1379,21 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data } if (create_reflog) { + struct ident_split c; + ALLOC_GROW(logs, logs_nr + 1, logs_alloc); log = &logs[logs_nr++]; memset(log, 0, sizeof(*log)); - fill_reftable_log_record(log, &committer_ident); + if (u->committer_info) { + if (split_ident_line(&c, u->committer_info, + strlen(u->committer_info))) + BUG("failed splitting committer info"); + } else { + c = committer_ident; + } + + fill_reftable_log_record(log, &c); log->update_index = ts; log->refname = xstrdup(u->refname); memcpy(log->value.update.new_hash, From a3582e2eacfae4328146b5142a0950fffcc05198 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:27 +0100 Subject: [PATCH 2/9] refs: add `index` field to `struct ref_udpate` The reftable backend, sorts its updates by refname before applying them, this ensures that the references are stored sorted. When migrating reflogs from one backend to another, the order of the reflogs must be maintained. Add a new `index` field to the `ref_update` struct to facilitate this. This field is used in the reftable backend's sort comparison function `transaction_update_cmp`, to ensure that indexed fields maintain their order. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs/refs-internal.h | 7 +++++++ refs/reftable-backend.c | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 0fd95cdacd99e4..f5c733d099f0c6 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -115,6 +115,13 @@ struct ref_update { char *msg; char *committer_info; + /* + * The index overrides the default sort algorithm. This is needed + * when migrating reflogs and we want to ensure we carry over the + * same order. + */ + unsigned int index; + /* * If this ref_update was split off of a symref update via * split_symref_update(), then this member points at that diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index e882602487c662..c008f20be719fe 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -1279,8 +1279,17 @@ static int reftable_be_transaction_abort(struct ref_store *ref_store UNUSED, static int transaction_update_cmp(const void *a, const void *b) { - return strcmp(((struct reftable_transaction_update *)a)->update->refname, - ((struct reftable_transaction_update *)b)->update->refname); + struct reftable_transaction_update *update_a = (struct reftable_transaction_update *)a; + struct reftable_transaction_update *update_b = (struct reftable_transaction_update *)b; + + /* + * If there is an index set, it should take preference (default is 0). + * This ensures that updates with indexes are sorted amongst themselves. + */ + if (update_a->update->index || update_b->update->index) + return update_a->update->index - update_b->update->index; + + return strcmp(update_a->update->refname, update_b->update->refname); } static int write_transaction_table(struct reftable_writer *writer, void *cb_data) From 611986f3007d591731f56ad04cf3e00f0d2f163f Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:28 +0100 Subject: [PATCH 3/9] refs/files: add count field to ref_lock When refs are updated in the files-backend, a lock is obtained for the corresponding file path. This is the case even for reflogs, i.e. a lock is obtained on the reference path instead of the reflog path. This works, since generally, reflogs are updated alongside the ref. The upcoming patches will add support for reflog updates in ref transaction. This means, in a particular transaction we want to have ref updates and reflog updates. For a given ref in a given transaction there can be at most one update. But we can theoretically have multiple reflog updates for a given ref in a given transaction. A great example of this would be when migrating reflogs from one backend to another. There we would batch all the reflog updates for a given reference in a single transaction. The current flow does not support this, because currently refs & reflogs are treated as a single entity and capture the lock together. To separate this, add a count field to ref_lock. With this, multiple updates can hold onto a single ref_lock and the lock will only be released when all of them release the lock. This patch only adds the `count` field to `ref_lock` and adds the logic to increment and decrement the lock. In a follow up commit, we'll separate the reflog update logic from ref updates and utilize this functionality. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs/files-backend.c | 58 +++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/refs/files-backend.c b/refs/files-backend.c index 6078668c99ee25..02cb4907d8659e 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -71,6 +71,7 @@ struct ref_lock { char *ref_name; struct lock_file lk; struct object_id old_oid; + unsigned int count; /* track users of the lock (ref update + reflog updates) */ }; struct files_ref_store { @@ -638,9 +639,12 @@ int parse_loose_ref_contents(const struct git_hash_algo *algop, static void unlock_ref(struct ref_lock *lock) { - rollback_lock_file(&lock->lk); - free(lock->ref_name); - free(lock); + lock->count--; + if (!lock->count) { + rollback_lock_file(&lock->lk); + free(lock->ref_name); + free(lock); + } } /* @@ -696,6 +700,7 @@ static int lock_raw_ref(struct files_ref_store *refs, *lock_p = CALLOC_ARRAY(lock, 1); lock->ref_name = xstrdup(refname); + lock->count = 1; files_ref_path(refs, &ref_file, refname); retry: @@ -1169,6 +1174,7 @@ static struct ref_lock *lock_ref_oid_basic(struct files_ref_store *refs, goto error_return; lock->ref_name = xstrdup(refname); + lock->count = 1; if (raceproof_create_file(ref_file.buf, create_reflock, &lock->lk)) { unable_to_lock_message(ref_file.buf, errno, err); @@ -2535,6 +2541,12 @@ static int check_old_oid(struct ref_update *update, struct object_id *oid, return -1; } +struct files_transaction_backend_data { + struct ref_transaction *packed_transaction; + int packed_refs_locked; + struct strmap ref_locks; +}; + /* * Prepare for carrying out update: * - Lock the reference referred to by update. @@ -2557,11 +2569,14 @@ static int lock_ref_for_update(struct files_ref_store *refs, { struct strbuf referent = STRBUF_INIT; int mustexist = ref_update_expects_existing_old_ref(update); + struct files_transaction_backend_data *backend_data; int ret = 0; struct ref_lock *lock; files_assert_main_repository(refs, "lock_ref_for_update"); + backend_data = transaction->backend_data; + if ((update->flags & REF_HAVE_NEW) && ref_update_has_null_new_value(update)) update->flags |= REF_DELETING; @@ -2572,18 +2587,25 @@ static int lock_ref_for_update(struct files_ref_store *refs, goto out; } - ret = lock_raw_ref(refs, update->refname, mustexist, - affected_refnames, - &lock, &referent, - &update->type, err); - if (ret) { - char *reason; + lock = strmap_get(&backend_data->ref_locks, update->refname); + if (lock) { + lock->count++; + } else { + ret = lock_raw_ref(refs, update->refname, mustexist, + affected_refnames, + &lock, &referent, + &update->type, err); + if (ret) { + char *reason; + + reason = strbuf_detach(err, NULL); + strbuf_addf(err, "cannot lock ref '%s': %s", + ref_update_original_update_refname(update), reason); + free(reason); + goto out; + } - reason = strbuf_detach(err, NULL); - strbuf_addf(err, "cannot lock ref '%s': %s", - ref_update_original_update_refname(update), reason); - free(reason); - goto out; + strmap_put(&backend_data->ref_locks, update->refname, lock); } update->backend_data = lock; @@ -2730,11 +2752,6 @@ static int lock_ref_for_update(struct files_ref_store *refs, return ret; } -struct files_transaction_backend_data { - struct ref_transaction *packed_transaction; - int packed_refs_locked; -}; - /* * Unlock any references in `transaction` that are still locked, and * mark the transaction closed. @@ -2767,6 +2784,8 @@ static void files_transaction_cleanup(struct files_ref_store *refs, if (backend_data->packed_refs_locked) packed_refs_unlock(refs->packed_ref_store); + strmap_clear(&backend_data->ref_locks, 0); + free(backend_data); } @@ -2796,6 +2815,7 @@ static int files_transaction_prepare(struct ref_store *ref_store, goto cleanup; CALLOC_ARRAY(backend_data, 1); + strmap_init(&backend_data->ref_locks); transaction->backend_data = backend_data; /* From add2c4f6e225083747e86c2e2e89e80c97b28733 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:29 +0100 Subject: [PATCH 4/9] refs: extract out refname verification in transactions Unless the `REF_SKIP_REFNAME_VERIFICATION` flag is set for an update, the refname of the update is verified for: - Ensuring it is not a pseudoref. - Checking the refname format. These checks will also be needed in a following commit where the function to add reflog updates to the transaction is introduced. Extract the code out into a new static function. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/refs.c b/refs.c index f003e51c6bf522..9c9f4940c60d3c 100644 --- a/refs.c +++ b/refs.c @@ -1196,6 +1196,28 @@ struct ref_update *ref_transaction_add_update( return update; } +static int transaction_refname_valid(const char *refname, + const struct object_id *new_oid, + unsigned int flags, struct strbuf *err) +{ + if (flags & REF_SKIP_REFNAME_VERIFICATION) + return 1; + + if (is_pseudo_ref(refname)) { + strbuf_addf(err, _("refusing to update pseudoref '%s'"), + refname); + return 0; + } else if ((new_oid && !is_null_oid(new_oid)) ? + check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) : + !refname_is_safe(refname)) { + strbuf_addf(err, _("refusing to update ref with bad name '%s'"), + refname); + return 0; + } + + return 1; +} + int ref_transaction_update(struct ref_transaction *transaction, const char *refname, const struct object_id *new_oid, @@ -1213,21 +1235,8 @@ int ref_transaction_update(struct ref_transaction *transaction, return -1; } - if (!(flags & REF_SKIP_REFNAME_VERIFICATION) && - ((new_oid && !is_null_oid(new_oid)) ? - check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) : - !refname_is_safe(refname))) { - strbuf_addf(err, _("refusing to update ref with bad name '%s'"), - refname); + if (!transaction_refname_valid(refname, new_oid, flags, err)) return -1; - } - - if (!(flags & REF_SKIP_REFNAME_VERIFICATION) && - is_pseudo_ref(refname)) { - strbuf_addf(err, _("refusing to update pseudoref '%s'"), - refname); - return -1; - } if (flags & ~REF_TRANSACTION_UPDATE_ALLOWED_FLAGS) BUG("illegal flags 0x%x passed to ref_transaction_update()", flags); From 4483be36f4477252f785df0c8c40677df8c18828 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:30 +0100 Subject: [PATCH 5/9] refs: add `committer_info` to `ref_transaction_add_update()` The `ref_transaction_add_update()` creates the `ref_update` struct. To facilitate addition of reflogs in the next commit, the function needs to accommodate setting the `committer_info` field in the struct. So modify the function to also take `committer_info` as an argument and set it accordingly. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs.c | 7 +++++-- refs/files-backend.c | 14 ++++++++------ refs/refs-internal.h | 1 + refs/reftable-backend.c | 6 ++++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/refs.c b/refs.c index 9c9f4940c60d3c..782bf1090af651 100644 --- a/refs.c +++ b/refs.c @@ -1166,6 +1166,7 @@ struct ref_update *ref_transaction_add_update( const struct object_id *new_oid, const struct object_id *old_oid, const char *new_target, const char *old_target, + const char *committer_info, const char *msg) { struct ref_update *update; @@ -1190,8 +1191,10 @@ struct ref_update *ref_transaction_add_update( oidcpy(&update->new_oid, new_oid); if ((flags & REF_HAVE_OLD) && old_oid) oidcpy(&update->old_oid, old_oid); - if (!(flags & REF_SKIP_CREATE_REFLOG)) + if (!(flags & REF_SKIP_CREATE_REFLOG)) { + update->committer_info = xstrdup_or_null(committer_info); update->msg = normalize_reflog_message(msg); + } return update; } @@ -1253,7 +1256,7 @@ int ref_transaction_update(struct ref_transaction *transaction, ref_transaction_add_update(transaction, refname, flags, new_oid, old_oid, new_target, - old_target, msg); + old_target, NULL, msg); return 0; } diff --git a/refs/files-backend.c b/refs/files-backend.c index 02cb4907d8659e..255fed8354cae9 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1270,7 +1270,7 @@ static void prune_ref(struct files_ref_store *refs, struct ref_to_prune *r) ref_transaction_add_update( transaction, r->name, REF_NO_DEREF | REF_HAVE_NEW | REF_HAVE_OLD | REF_IS_PRUNING, - null_oid(), &r->oid, NULL, NULL, NULL); + null_oid(), &r->oid, NULL, NULL, NULL, NULL); if (ref_transaction_commit(transaction, &err)) goto cleanup; @@ -2417,7 +2417,7 @@ static int split_head_update(struct ref_update *update, transaction, "HEAD", update->flags | REF_LOG_ONLY | REF_NO_DEREF, &update->new_oid, &update->old_oid, - NULL, NULL, update->msg); + NULL, NULL, update->committer_info, update->msg); /* * Add "HEAD". This insertion is O(N) in the transaction @@ -2481,7 +2481,8 @@ static int split_symref_update(struct ref_update *update, transaction, referent, new_flags, update->new_target ? NULL : &update->new_oid, update->old_target ? NULL : &update->old_oid, - update->new_target, update->old_target, update->msg); + update->new_target, update->old_target, NULL, + update->msg); new_update->parent_update = update; @@ -2914,7 +2915,7 @@ static int files_transaction_prepare(struct ref_store *ref_store, packed_transaction, update->refname, REF_HAVE_NEW | REF_NO_DEREF, &update->new_oid, NULL, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); } } @@ -3094,12 +3095,13 @@ static int files_transaction_finish_initial(struct files_ref_store *refs, ref_transaction_add_update(loose_transaction, update->refname, update->flags & ~REF_HAVE_OLD, update->new_target ? NULL : &update->new_oid, NULL, - update->new_target, NULL, NULL); + update->new_target, NULL, update->committer_info, + NULL); } else { ref_transaction_add_update(packed_transaction, update->refname, update->flags & ~REF_HAVE_OLD, &update->new_oid, &update->old_oid, - NULL, NULL, NULL); + NULL, NULL, update->committer_info, NULL); } } diff --git a/refs/refs-internal.h b/refs/refs-internal.h index f5c733d099f0c6..79b287c5ec5c7d 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -162,6 +162,7 @@ struct ref_update *ref_transaction_add_update( const struct object_id *new_oid, const struct object_id *old_oid, const char *new_target, const char *old_target, + const char *committer_info, const char *msg); /* diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index c008f20be719fe..b2e3ba877de9e5 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -1078,7 +1078,8 @@ static int reftable_be_transaction_prepare(struct ref_store *ref_store, new_update = ref_transaction_add_update( transaction, "HEAD", u->flags | REF_LOG_ONLY | REF_NO_DEREF, - &u->new_oid, &u->old_oid, NULL, NULL, u->msg); + &u->new_oid, &u->old_oid, NULL, NULL, NULL, + u->msg); string_list_insert(&affected_refnames, new_update->refname); } @@ -1161,7 +1162,8 @@ static int reftable_be_transaction_prepare(struct ref_store *ref_store, transaction, referent.buf, new_flags, u->new_target ? NULL : &u->new_oid, u->old_target ? NULL : &u->old_oid, - u->new_target, u->old_target, u->msg); + u->new_target, u->old_target, + u->committer_info, u->msg); new_update->parent_update = u; From 84675fa2717e08b39bf810eb9a439068ac915dfb Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:31 +0100 Subject: [PATCH 6/9] refs: introduce the `ref_transaction_update_reflog` function Introduce a new function `ref_transaction_update_reflog`, for clients to add a reflog update to a transaction. While the existing function `ref_transaction_update` also allows clients to add a reflog entry, this function does a few things more, It: - Enforces that only a reflog entry is added and does not update the ref itself. - Allows the users to also provide the committer information. This means clients can add reflog entries with custom committer information. The `transaction_refname_valid()` function also modifies the error message selectively based on the type of the update. This change also affects reflog updates which go through `ref_transaction_update()`. A follow up commit will utilize this function to add reflog support to `git refs migrate`. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs.c | 39 +++++++++++++++++++++++++++++++++++---- refs.h | 14 ++++++++++++++ refs/files-backend.c | 24 ++++++++++++++++-------- 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/refs.c b/refs.c index 782bf1090af651..8b3882cff17e5e 100644 --- a/refs.c +++ b/refs.c @@ -1207,14 +1207,14 @@ static int transaction_refname_valid(const char *refname, return 1; if (is_pseudo_ref(refname)) { - strbuf_addf(err, _("refusing to update pseudoref '%s'"), - refname); + const char *what = flags & REF_LOG_ONLY ? "reflog for pseudoref" : "pseudoref"; + strbuf_addf(err, _("refusing to update %s '%s'"), what, refname); return 0; } else if ((new_oid && !is_null_oid(new_oid)) ? check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) : !refname_is_safe(refname)) { - strbuf_addf(err, _("refusing to update ref with bad name '%s'"), - refname); + const char *what = flags & REF_LOG_ONLY ? "reflog with bad name" : "ref with bad name"; + strbuf_addf(err, _("refusing to update %s '%s'"), what, refname); return 0; } @@ -1257,6 +1257,37 @@ int ref_transaction_update(struct ref_transaction *transaction, ref_transaction_add_update(transaction, refname, flags, new_oid, old_oid, new_target, old_target, NULL, msg); + + return 0; +} + +int ref_transaction_update_reflog(struct ref_transaction *transaction, + const char *refname, + const struct object_id *new_oid, + const struct object_id *old_oid, + const char *committer_info, unsigned int flags, + const char *msg, unsigned int index, + struct strbuf *err) +{ + struct ref_update *update; + + assert(err); + + flags |= REF_LOG_ONLY | REF_NO_DEREF; + + if (!transaction_refname_valid(refname, new_oid, flags, err)) + return -1; + + update = ref_transaction_add_update(transaction, refname, flags, + new_oid, old_oid, NULL, NULL, + committer_info, msg); + /* + * While we do set the old_oid value, we unset the flag to skip + * old_oid verification which only makes sense for refs. + */ + update->flags &= ~REF_HAVE_OLD; + update->index = index; + return 0; } diff --git a/refs.h b/refs.h index a5bedf48cf6de9..b0dfc65ed2e59c 100644 --- a/refs.h +++ b/refs.h @@ -727,6 +727,20 @@ int ref_transaction_update(struct ref_transaction *transaction, unsigned int flags, const char *msg, struct strbuf *err); +/* + * Similar to`ref_transaction_update`, but this function is only for adding + * a reflog update. Supports providing custom committer information. The index + * field can be utiltized to order updates as desired. When not used, the + * updates default to being ordered by refname. + */ +int ref_transaction_update_reflog(struct ref_transaction *transaction, + const char *refname, + const struct object_id *new_oid, + const struct object_id *old_oid, + const char *committer_info, unsigned int flags, + const char *msg, unsigned int index, + struct strbuf *err); + /* * Add a reference creation to transaction. new_oid is the value that * the reference should have after the update; it must not be diff --git a/refs/files-backend.c b/refs/files-backend.c index 255fed8354cae9..c11213f52065bc 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -3080,10 +3080,12 @@ static int files_transaction_finish_initial(struct files_ref_store *refs, } /* - * packed-refs don't support symbolic refs and root refs, so we - * have to queue these references via the loose transaction. + * packed-refs don't support symbolic refs, root refs and reflogs, + * so we have to queue these references via the loose transaction. */ - if (update->new_target || is_root_ref(update->refname)) { + if (update->new_target || + is_root_ref(update->refname) || + (update->flags & REF_LOG_ONLY)) { if (!loose_transaction) { loose_transaction = ref_store_transaction_begin(&refs->base, 0, err); if (!loose_transaction) { @@ -3092,11 +3094,17 @@ static int files_transaction_finish_initial(struct files_ref_store *refs, } } - ref_transaction_add_update(loose_transaction, update->refname, - update->flags & ~REF_HAVE_OLD, - update->new_target ? NULL : &update->new_oid, NULL, - update->new_target, NULL, update->committer_info, - NULL); + if (update->flags & REF_LOG_ONLY) + ref_transaction_add_update(loose_transaction, update->refname, + update->flags, &update->new_oid, + &update->old_oid, NULL, NULL, + update->committer_info, update->msg); + else + ref_transaction_add_update(loose_transaction, update->refname, + update->flags & ~REF_HAVE_OLD, + update->new_target ? NULL : &update->new_oid, NULL, + update->new_target, NULL, update->committer_info, + NULL); } else { ref_transaction_add_update(packed_transaction, update->refname, update->flags & ~REF_HAVE_OLD, From 297c09eabb1e8b44230bca86fc7fd344175e0be7 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:32 +0100 Subject: [PATCH 7/9] refs: allow multiple reflog entries for the same refname The reference transaction only allows a single update for a given reference to avoid conflicts. This, however, isn't an issue for reflogs. There are no conflicts to be resolved in reflogs and when migrating reflogs between backends we'd have multiple reflog entries for the same refname. So allow multiple reflog updates within a single transaction. Also the reflog creation logic isn't exposed to the end user. While this might change in the future, currently, this reduces the scope of issues to think about. In the reftable backend, the writer sorts all updates based on the update_index before writing to the block. When there are multiple reflogs for a given refname, it is essential that the order of the reflogs is maintained. So add the `index` value to the `update_index`. The `index` field is only set when multiple reflog entries for a given refname are added and as such in most scenarios the old behavior remains. This is required to add reflog migration support to `git refs migrate`. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs/files-backend.c | 15 +++++++++++---- refs/reftable-backend.c | 22 +++++++++++++++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/refs/files-backend.c b/refs/files-backend.c index c11213f52065bc..8953d1c6d37b13 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2611,6 +2611,9 @@ static int lock_ref_for_update(struct files_ref_store *refs, update->backend_data = lock; + if (update->flags & REF_LOG_ONLY) + goto out; + if (update->type & REF_ISSYMREF) { if (update->flags & REF_NO_DEREF) { /* @@ -2829,13 +2832,16 @@ static int files_transaction_prepare(struct ref_store *ref_store, */ for (i = 0; i < transaction->nr; i++) { struct ref_update *update = transaction->updates[i]; - struct string_list_item *item = - string_list_append(&affected_refnames, update->refname); + struct string_list_item *item; if ((update->flags & REF_IS_PRUNING) && !(update->flags & REF_NO_DEREF)) BUG("REF_IS_PRUNING set without REF_NO_DEREF"); + if (update->flags & REF_LOG_ONLY) + continue; + + item = string_list_append(&affected_refnames, update->refname); /* * We store a pointer to update in item->util, but at * the moment we never use the value of this field @@ -3035,8 +3041,9 @@ static int files_transaction_finish_initial(struct files_ref_store *refs, /* Fail if a refname appears more than once in the transaction: */ for (i = 0; i < transaction->nr; i++) - string_list_append(&affected_refnames, - transaction->updates[i]->refname); + if (!(transaction->updates[i]->flags & REF_LOG_ONLY)) + string_list_append(&affected_refnames, + transaction->updates[i]->refname); string_list_sort(&affected_refnames); if (ref_update_reject_duplicates(&affected_refnames, err)) { ret = TRANSACTION_GENERIC_ERROR; diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index b2e3ba877de9e5..bec5962debea7b 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -990,8 +990,9 @@ static int reftable_be_transaction_prepare(struct ref_store *ref_store, if (ret) goto done; - string_list_append(&affected_refnames, - transaction->updates[i]->refname); + if (!(transaction->updates[i]->flags & REF_LOG_ONLY)) + string_list_append(&affected_refnames, + transaction->updates[i]->refname); } /* @@ -1301,6 +1302,7 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data struct reftable_log_record *logs = NULL; struct ident_split committer_ident = {0}; size_t logs_nr = 0, logs_alloc = 0, i; + uint64_t max_update_index = ts; const char *committer_info; int ret = 0; @@ -1405,7 +1407,19 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data } fill_reftable_log_record(log, &c); - log->update_index = ts; + + /* + * Updates are sorted by the writer. So updates for the same + * refname need to contain different update indices. + */ + log->update_index = ts + u->index; + + /* + * Note the max update_index so the limit can be set later on. + */ + if (log->update_index > max_update_index) + max_update_index = log->update_index; + log->refname = xstrdup(u->refname); memcpy(log->value.update.new_hash, u->new_oid.hash, GIT_MAX_RAWSZ); @@ -1469,6 +1483,8 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data * and log blocks. */ if (logs) { + reftable_writer_set_limits(writer, ts, max_update_index); + ret = reftable_writer_add_logs(writer, logs, logs_nr); if (ret < 0) goto done; From 246cebe3205694ce19eceaa795d20f24ba762c49 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Mon, 16 Dec 2024 17:44:33 +0100 Subject: [PATCH 8/9] refs: add support for migrating reflogs The `git refs migrate` command was introduced in 25a0023f28 (builtin/refs: new command to migrate ref storage formats, 2024-06-06) to support migrating from one reference backend to another. One limitation of the command was that it didn't support migrating repositories which contained reflogs. A previous commit, added support for adding reflog updates in ref transactions. Using the added functionality bake in reflog support for `git refs migrate`. To ensure that the order of the reflogs is maintained during the migration, we add the index for each reflog update as we iterate over the reflogs from the old reference backend. This is to ensure that the order is maintained in the new backend. Helped-by: Patrick Steinhardt Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- Documentation/git-refs.txt | 2 - refs.c | 92 ++++++++++++++++++++++++++------------ t/t1460-refs-migrate.sh | 73 +++++++++++++++++++++--------- 3 files changed, 115 insertions(+), 52 deletions(-) diff --git a/Documentation/git-refs.txt b/Documentation/git-refs.txt index ce31f93061db5e..9829984b0a4c4f 100644 --- a/Documentation/git-refs.txt +++ b/Documentation/git-refs.txt @@ -57,8 +57,6 @@ KNOWN LIMITATIONS The ref format migration has several known limitations in its current form: -* It is not possible to migrate repositories that have reflogs. - * It is not possible to migrate repositories that have worktrees. * There is no way to block concurrent writes to the repository during an diff --git a/refs.c b/refs.c index 8b3882cff17e5e..5d541ddc41aa84 100644 --- a/refs.c +++ b/refs.c @@ -30,6 +30,7 @@ #include "date.h" #include "commit.h" #include "wildmatch.h" +#include "ident.h" /* * List of all available backends @@ -2673,6 +2674,7 @@ struct migration_data { struct ref_store *old_refs; struct ref_transaction *transaction; struct strbuf *errbuf; + struct strbuf sb; }; static int migrate_one_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, @@ -2705,6 +2707,52 @@ static int migrate_one_ref(const char *refname, const char *referent UNUSED, con return ret; } +struct reflog_migration_data { + unsigned int index; + const char *refname; + struct ref_store *old_refs; + struct ref_transaction *transaction; + struct strbuf *errbuf; + struct strbuf *sb; +}; + +static int migrate_one_reflog_entry(struct object_id *old_oid, + struct object_id *new_oid, + const char *committer, + timestamp_t timestamp, int tz, + const char *msg, void *cb_data) +{ + struct reflog_migration_data *data = cb_data; + const char *date; + int ret; + + date = show_date(timestamp, tz, DATE_MODE(NORMAL)); + strbuf_reset(data->sb); + /* committer contains name and email */ + strbuf_addstr(data->sb, fmt_ident("", committer, WANT_BLANK_IDENT, date, 0)); + + ret = ref_transaction_update_reflog(data->transaction, data->refname, + new_oid, old_oid, data->sb->buf, + REF_HAVE_NEW | REF_HAVE_OLD, msg, + data->index++, data->errbuf); + return ret; +} + +static int migrate_one_reflog(const char *refname, void *cb_data) +{ + struct migration_data *migration_data = cb_data; + struct reflog_migration_data data = { + .refname = refname, + .old_refs = migration_data->old_refs, + .transaction = migration_data->transaction, + .errbuf = migration_data->errbuf, + .sb = &migration_data->sb, + }; + + return refs_for_each_reflog_ent(migration_data->old_refs, refname, + migrate_one_reflog_entry, &data); +} + static int move_files(const char *from_path, const char *to_path, struct strbuf *errbuf) { struct strbuf from_buf = STRBUF_INIT, to_buf = STRBUF_INIT; @@ -2771,13 +2819,6 @@ static int move_files(const char *from_path, const char *to_path, struct strbuf return ret; } -static int count_reflogs(const char *reflog UNUSED, void *payload) -{ - size_t *reflog_count = payload; - (*reflog_count)++; - return 0; -} - static int has_worktrees(void) { struct worktree **worktrees = get_worktrees(); @@ -2802,8 +2843,9 @@ int repo_migrate_ref_storage_format(struct repository *repo, struct ref_store *old_refs = NULL, *new_refs = NULL; struct ref_transaction *transaction = NULL; struct strbuf new_gitdir = STRBUF_INIT; - struct migration_data data; - size_t reflog_count = 0; + struct migration_data data = { + .sb = STRBUF_INIT, + }; int did_migrate_refs = 0; int ret; @@ -2815,21 +2857,6 @@ int repo_migrate_ref_storage_format(struct repository *repo, old_refs = get_main_ref_store(repo); - /* - * We do not have any interfaces that would allow us to write many - * reflog entries. Once we have them we can remove this restriction. - */ - if (refs_for_each_reflog(old_refs, count_reflogs, &reflog_count) < 0) { - strbuf_addstr(errbuf, "cannot count reflogs"); - ret = -1; - goto done; - } - if (reflog_count) { - strbuf_addstr(errbuf, "migrating reflogs is not supported yet"); - ret = -1; - goto done; - } - /* * Worktrees complicate the migration because every worktree has a * separate ref storage. While it should be feasible to implement, this @@ -2855,17 +2882,21 @@ int repo_migrate_ref_storage_format(struct repository *repo, * This operation is safe as we do not yet modify the main * repository. * - * 3. If we're in dry-run mode then we are done and can hand over the + * 3. Enumerate all reflogs and write them into the new ref storage. + * This operation is safe as we do not yet modify the main + * repository. + * + * 4. If we're in dry-run mode then we are done and can hand over the * directory to the caller for inspection. If not, we now start * with the destructive part. * - * 4. Delete the old ref storage from disk. As we have a copy of refs + * 5. Delete the old ref storage from disk. As we have a copy of refs * in the new ref storage it's okay(ish) if we now get interrupted * as there is an equivalent copy of all refs available. * - * 5. Move the new ref storage files into place. + * 6. Move the new ref storage files into place. * - * 6. Change the repository format to the new ref format. + * 7. Change the repository format to the new ref format. */ strbuf_addf(&new_gitdir, "%s/%s", old_refs->gitdir, "ref_migration.XXXXXX"); if (!mkdtemp(new_gitdir.buf)) { @@ -2907,6 +2938,10 @@ int repo_migrate_ref_storage_format(struct repository *repo, if (ret < 0) goto done; + ret = refs_for_each_reflog(old_refs, migrate_one_reflog, &data); + if (ret < 0) + goto done; + ret = ref_transaction_commit(transaction, errbuf); if (ret < 0) goto done; @@ -2982,6 +3017,7 @@ int repo_migrate_ref_storage_format(struct repository *repo, } ref_transaction_free(transaction); strbuf_release(&new_gitdir); + strbuf_release(&data.sb); return ret; } diff --git a/t/t1460-refs-migrate.sh b/t/t1460-refs-migrate.sh index 1bfff3a7afd5ac..f59bc4860f19c4 100755 --- a/t/t1460-refs-migrate.sh +++ b/t/t1460-refs-migrate.sh @@ -7,23 +7,44 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +# Migrate the provided repository from one format to the other and +# verify that the references and logs are migrated over correctly. +# Usage: test_migration +# is the relative path to the repo to be migrated. +# is the ref format to be migrated to. +# (true or false) whether to skip reflog verification. test_migration () { - git -C "$1" for-each-ref --include-root-refs \ + repo=$1 && + format=$2 && + skip_reflog_verify=${3:-false} && + git -C "$repo" for-each-ref --include-root-refs \ --format='%(refname) %(objectname) %(symref)' >expect && - git -C "$1" refs migrate --ref-format="$2" && - git -C "$1" for-each-ref --include-root-refs \ + if ! $skip_reflog_verify + then + git -C "$repo" reflog --all >expect_logs && + git -C "$repo" reflog list >expect_log_list + fi && + + git -C "$repo" refs migrate --ref-format="$2" && + + git -C "$repo" for-each-ref --include-root-refs \ --format='%(refname) %(objectname) %(symref)' >actual && test_cmp expect actual && + if ! $skip_reflog_verify + then + git -C "$repo" reflog --all >actual_logs && + git -C "$repo" reflog list >actual_log_list && + test_cmp expect_logs actual_logs && + test_cmp expect_log_list actual_log_list + fi && - git -C "$1" rev-parse --show-ref-format >actual && - echo "$2" >expect && + git -C "$repo" rev-parse --show-ref-format >actual && + echo "$format" >expect && test_cmp expect actual } test_expect_success 'setup' ' - rm -rf .git && - # The migration does not yet support reflogs. - git config --global core.logAllRefUpdates false + rm -rf .git ' test_expect_success "superfluous arguments" ' @@ -78,19 +99,6 @@ do test_cmp expect err ' - test_expect_success "$from_format -> $to_format: migration with reflog fails" ' - test_when_finished "rm -rf repo" && - git init --ref-format=$from_format repo && - test_config -C repo core.logAllRefUpdates true && - test_commit -C repo logged && - test_must_fail git -C repo refs migrate \ - --ref-format=$to_format 2>err && - cat >expect <<-EOF && - error: migrating reflogs is not supported yet - EOF - test_cmp expect err - ' - test_expect_success "$from_format -> $to_format: migration with worktree fails" ' test_when_finished "rm -rf repo" && git init --ref-format=$from_format repo && @@ -141,7 +149,7 @@ do test_commit -C repo initial && test-tool -C repo ref-store main update-ref "" refs/heads/broken \ "$(test_oid 001)" "$ZERO_OID" REF_SKIP_CREATE_REFLOG,REF_SKIP_OID_VERIFICATION && - test_migration repo "$to_format" && + test_migration repo "$to_format" true && test_oid 001 >expect && git -C repo rev-parse refs/heads/broken >actual && test_cmp expect actual @@ -195,6 +203,27 @@ do git -C repo rev-parse --show-ref-format >actual && test_cmp expect actual ' + + test_expect_success "$from_format -> $to_format: reflogs of symrefs with target deleted" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit -C repo initial && + git -C repo branch branch-1 HEAD && + git -C repo symbolic-ref refs/heads/symref refs/heads/branch-1 && + cat >input <<-EOF && + delete refs/heads/branch-1 + EOF + git -C repo update-ref --stdin $to_format: reflogs order is retained" ' + test_when_finished "rm -rf repo" && + git init --ref-format=$from_format repo && + test_commit --date "100005000 +0700" --no-tag -C repo initial && + test_commit --date "100003000 +0700" --no-tag -C repo second && + test_migration repo "$to_format" + ' done done From 8ddcdc1bb33ccf803461dd2365146f9341bf9312 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Fri, 20 Dec 2024 13:58:37 +0100 Subject: [PATCH 9/9] refs: mark invalid refname message for translation The error message produced by `transaction_refname_valid()` changes based on whether the update is a ref update or a reflog update, with the use of a ternary operator. This breaks translation since the sub-msg is not marked for translation. Fix this by setting the entire message using a `if {} else {}` block and marking each message for translation. Signed-off-by: Karthik Nayak Signed-off-by: Junio C Hamano --- refs.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/refs.c b/refs.c index 5d541ddc41aa84..c55583986940d8 100644 --- a/refs.c +++ b/refs.c @@ -1208,14 +1208,22 @@ static int transaction_refname_valid(const char *refname, return 1; if (is_pseudo_ref(refname)) { - const char *what = flags & REF_LOG_ONLY ? "reflog for pseudoref" : "pseudoref"; - strbuf_addf(err, _("refusing to update %s '%s'"), what, refname); + const char *refusal_msg; + if (flags & REF_LOG_ONLY) + refusal_msg = _("refusing to update reflog for pseudoref '%s'"); + else + refusal_msg = _("refusing to update pseudoref '%s'"); + strbuf_addf(err, refusal_msg, refname); return 0; } else if ((new_oid && !is_null_oid(new_oid)) ? check_refname_format(refname, REFNAME_ALLOW_ONELEVEL) : !refname_is_safe(refname)) { - const char *what = flags & REF_LOG_ONLY ? "reflog with bad name" : "ref with bad name"; - strbuf_addf(err, _("refusing to update %s '%s'"), what, refname); + const char *refusal_msg; + if (flags & REF_LOG_ONLY) + refusal_msg = _("refusing to update reflog with bad name '%s'"); + else + refusal_msg = _("refusing to update ref with bad name '%s'"); + strbuf_addf(err, refusal_msg, refname); return 0; }