-
Notifications
You must be signed in to change notification settings - Fork 4
/
work_kernel_tree.sh
executable file
·273 lines (221 loc) · 12.6 KB
/
work_kernel_tree.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
#!/usr/bin/env bash
set -e
function display_alert() {
echo "--> $*"
}
echo "::group::Read kernel.org versions"
# Read the current versions of kernel from kernel.org JSON releases. Again, thanks, kernel.org.
curl --silent "https://www.kernel.org/releases.json" > /tmp/kernel-releases.json
echo "Kernel releases versions from JSON:"
cat /tmp/kernel-releases.json | jq -r ".releases[].version"
declare -ag WANTED_KERNEL_VERSIONS
mapfile -t WANTED_KERNEL_VERSIONS < <(cat /tmp/kernel-releases.json | jq -r ".releases[].version" | grep -v -e "^next\-" | sed -e 's|-rc|.-rc|' | cut -d "." -f 1,2)
#declare -ag WANTED_KERNEL_VERSIONS=("5.19" "5.18" "5.17" "5.15" "5.10" "4.19" "4.9" "4.4")
# Show the array
display_alert "Wanted kernel versions:" "${WANTED_KERNEL_VERSIONS[@]}"
echo "::endgroup::"
echo "::group::Prepare basics"
ONLINE="${ONLINE:-"yes"}"
EXPORT_SHALLOW_PER_VERSION="yes"
EXPORT_COMPLETE="yes" # note: The complete .tar is bigger than 2gb, and that does not fit into GH Releases 2gb limit for any single file. Use ORAS and ghcr.io.
BASE_WORK_DIR="${BASE_WORK_DIR:-"/Volumes/LinuxDev/shallow_git_tree_work"}"
WORKDIR="${BASE_WORK_DIR}/kernel"
SHALLOWED_TREES_DIR="${WORKDIR}/shallow_trees"
COMPLETE_TREES_DIR="${WORKDIR}/complete_trees"
OUTPUT_DIR_ORAS="${WORKDIR}/output_oras"
KERNEL_GIT_TREE="${WORKDIR}/worktree"
KERNEL_TORVALDS_BUNDLE_DIR="${WORKDIR}/bundle-torvalds"
ALL_VERSIONS_FILE="${OUTPUT_DIR_ORAS}/shallow_versions.txt"
mkdir -p "${BASE_WORK_DIR}" "${WORKDIR}" "${SHALLOWED_TREES_DIR}" "${COMPLETE_TREES_DIR}" "${KERNEL_GIT_TREE}" "${KERNEL_TORVALDS_BUNDLE_DIR}" "${OUTPUT_DIR_ORAS}"
GIT_TORVALDS_BUNDLE_URL="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/clone.bundle" # Thanks, kernel.org!
GIT_TORVALDS_BUNDLE_ID="$(echo -n "${GIT_TORVALDS_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL.
GIT_TORVALDS_BUNDLE_FILE="${KERNEL_TORVALDS_BUNDLE_DIR}/${GIT_TORVALDS_BUNDLE_ID}.gitbundle" # final filename of bundle
GIT_TORVALDS_BUNDLE_REMOTE_NAME="torvalds-gitbundle" # name of the remote that will point to bundle
GIT_TORVALDS_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" # Torvalds live tree git:// URL
GIT_TORVALDS_LIVE_REMOTE_NAME="torvalds-live" # name of the remote that will point to live Torvalds tree
GIT_STABLE_LIVE_GIT_URL="git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" # Stable live tree git:// URL
GIT_STABLE_LIVE_REMOTE_NAME="stable-live" # name of the remote that will point to live stable tree
# 1st stage Global:
# Init an empty git repo
if [[ ! -d "${KERNEL_GIT_TREE}/.git" ]]; then
display_alert "Initting git tree"
git init --initial-branch="armbian_unused_initial_branch" "${KERNEL_GIT_TREE}"
else
display_alert "Git tree already initted"
fi
echo "::endgroup::"
echo "::group::Fetching Torvalds bundle"
## From now on, everything is done inside the git worktree...
cd "${KERNEL_GIT_TREE}" || exit 2
if ! git config "remote.${GIT_TORVALDS_BUNDLE_REMOTE_NAME}.url"; then
# Grab torvald's gitbundle via http from kernel.org
if [[ ! -f "${GIT_TORVALDS_BUNDLE_FILE}" ]]; then # Download the bundle file if it does not exist.
display_alert "Downloading Git cold bundle via HTTP" "${GIT_TORVALDS_BUNDLE_URL}"
wget --continue --progress=dot:giga --output-document="${GIT_TORVALDS_BUNDLE_FILE}" "${GIT_TORVALDS_BUNDLE_URL}"
else
display_alert "Cold bundle file exists, using it" "${GIT_TORVALDS_BUNDLE_FILE}" "git"
fi
# Fetch from Torvalds bundle (very slow) into 'torvalds-gitbundle' branch
display_alert "Fetching from cold git bundle, wait" "${GIT_TORVALDS_BUNDLE_ID}"
git bundle verify "${GIT_TORVALDS_BUNDLE_FILE}" # Make sure bundle is valid.
git remote add "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}" "${GIT_TORVALDS_BUNDLE_FILE}" # Add the remote pointing to the cold bundle file
git fetch --progress --verbose "${GIT_TORVALDS_BUNDLE_REMOTE_NAME}" # Fetch it!
else
display_alert "Torvalds bundle already fetched..."
fi
# At this stage, we've all blobs, but no tags!
echo "::endgroup::"
echo "::group::Fetching Torvalds live"
# 2nd stage Global:
# Add torvalds live git as remote 'torvalds-live'
if ! git config "remote.${GIT_TORVALDS_LIVE_REMOTE_NAME}.url"; then
display_alert "Adding torvalds live remote" "${GIT_TORVALDS_LIVE_REMOTE_NAME}"
git remote add "${GIT_TORVALDS_LIVE_REMOTE_NAME}" "${GIT_TORVALDS_LIVE_GIT_URL}"
else
display_alert "Torvalds live remote already exists..."
fi
# Fetch from it (to update), also bring in the tags. Around a 60mb download, quite fast.
if [[ "${ONLINE}" == "yes" ]]; then
display_alert "Fetching from torvalds live" "${GIT_TORVALDS_LIVE_REMOTE_NAME}"
git fetch --progress --verbose --tags "${GIT_TORVALDS_LIVE_REMOTE_NAME}" # Fetch it! (including tags!)
# create a local branch from the fetched
display_alert "Creating local branch 'torvalds-master' from torvalds live" "${GIT_TORVALDS_LIVE_REMOTE_NAME}"
git branch --force "torvalds-master" FETCH_HEAD
fi
echo "::endgroup::"
echo "::group::Adding stable remote"
# Now, add the stable remote. Do NOT fetch from it, it's huge and has a lot more than we need.
if ! git config "remote.${GIT_STABLE_LIVE_REMOTE_NAME}.url"; then
display_alert "Adding stable live remote" "${GIT_STABLE_LIVE_REMOTE_NAME}"
git remote add "${GIT_STABLE_LIVE_REMOTE_NAME}" "${GIT_STABLE_LIVE_GIT_URL}"
else
display_alert "Stable live remote already exists..."
fi
echo "::endgroup::"
# 3rd stage: For each version, eg: 5.17
# - Fetch from stable git source (not bundle) into `stable-5.17` branch
# - include tags
# - if this fails (eg: an unreleased kernel at that moment, tolerate and go ahead, torvalds should have -rc1)
WANTED_KERNEL_VERSIONS_COUNT=${#WANTED_KERNEL_VERSIONS[@]}
display_alert "Wanted kernel versions: ${WANTED_KERNEL_VERSIONS_COUNT}"
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
echo "::group::Fetching stable remote for ${KERNEL_VERSION}"
display_alert "Fetching stable kernel version: ${KERNEL_VERSION}"
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
KERNEL_VERSION_REMOTE_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
# Fetch the branch from the stable live into the local branch. Since I don't specify "--tags", it will only fetch the tags for the branch. Those DON'T include the -rc tags which came from torvalds live
if [[ "${ONLINE}" == "yes" ]]; then
declare -i STABLE_EXISTS=0
git fetch --progress --verbose "${GIT_STABLE_LIVE_REMOTE_NAME}" "${KERNEL_VERSION_REMOTE_BRANCH_NAME}:${KERNEL_VERSION_LOCAL_BRANCH_NAME}" && STABLE_EXISTS=1
if [[ ${STABLE_EXISTS} -eq 0 ]]; then
display_alert "Stable branch does not exist, copying torvalds-master to" "${KERNEL_VERSION_REMOTE_BRANCH_NAME}"
git branch --force "${KERNEL_VERSION_LOCAL_BRANCH_NAME}" "torvalds-master"
fi
fi
echo "::endgroup::"
done
# 4th stage: For each version, eg 5.17
# - Find the earliest tag with 5.17 in it
# - find the _date_ for such a tag
# - Export a shallow bundle via the date for that version;
# - include the shallow marker file (.git/shallow)
if [[ "${EXPORT_SHALLOW_PER_VERSION}" == "yes" ]]; then
echo "Writing file with all versions: ${ALL_VERSIONS_FILE}"
echo "${WANTED_KERNEL_VERSIONS[@]}" > "${ALL_VERSIONS_FILE}"
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
echo "::group::Exporting shallow for ${KERNEL_VERSION}"
cd "${KERNEL_GIT_TREE}" || exit 2
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
display_alert "Finding shallow point for version: ${KERNEL_VERSION}" "on local branch" "${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
# shit happens upstream too, so filter out "-dontuse" tags.
KERNEL_VERSION_FIRST_RC_TAG_NAME="$(git tag -l | grep "^v$(echo -n "${KERNEL_VERSION}" | sed -e 's/\./\\\./')-rc" | grep -v "\-dontuse" | sort -n | head -1)"
display_alert "Found first RC for version:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
# Now translate that tag into a date, which what we're gonna use to shallow the bundle.
# Attention: date has timezone part.
KERNEL_VERSION_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}")"
display_alert "Date for first RC tag:" "${KERNEL_VERSION}" "${KERNEL_VERSION_FIRST_RC_TAG_NAME}" "'${KERNEL_VERSION_SHALLOW_AT_DATE}'"
# Clone from the worktree into a new directory, shallowing in the process. This is the only way to make it consistently shallow without jumping through hoops.
KERNEL_VERSION_SHALLOWED_WORKDIR="${SHALLOWED_TREES_DIR}/shallow-${KERNEL_VERSION}-${KERNEL_VERSION_FIRST_RC_TAG_NAME}"
if [[ ! -d "${KERNEL_VERSION_SHALLOWED_WORKDIR}" ]]; then
display_alert "Making shallow tree" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
# --progress --verbose -- too much output for github actions
git clone --no-checkout \
--single-branch --branch="${KERNEL_VERSION_LOCAL_BRANCH_NAME}" \
--tags --shallow-since="${KERNEL_VERSION_SHALLOW_AT_DATE}" \
"file://${KERNEL_GIT_TREE}" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
else
display_alert "Shallow tree already exists" "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
fi
OUTPUT_BUNDLE_FILE_NAME_TARBALL="${OUTPUT_DIR_ORAS}/linux-shallow-${KERNEL_VERSION}.git.tar"
cd "${KERNEL_VERSION_SHALLOWED_WORKDIR}"
# Create a 'master' branch, which is the default branch name for git. Copy the shallowed branch.
git branch --force master "${KERNEL_VERSION_LOCAL_BRANCH_NAME}"
# Remove the origin remote, otherwise it would be exported due to "--all" below.
if git config "remote.origin.url"; then
git remote rm origin
fi
# remove hooks, if dir exists
if [[ -d "${KERNEL_VERSION_SHALLOWED_WORKDIR}/.git/hooks" ]]; then
rm -rf "${KERNEL_VERSION_SHALLOWED_WORKDIR}/.git/hooks"
fi
# list all tags in the shallow tree
echo -n "all tags ${KERNEL_VERSION}: "
git -C "${KERNEL_VERSION_SHALLOWED_WORKDIR}" tag -l | cat | xargs echo -n || true
echo ""
# list all branches in the shallow tree
echo -n "all branches ${KERNEL_VERSION}: "
git -C "${KERNEL_VERSION_SHALLOWED_WORKDIR}" branch -a | cat | xargs echo -n || true
echo ""
# export a .tar of .git. This is gonna be uploaded into ghcr.io via ORAS.
tar cf "${OUTPUT_BUNDLE_FILE_NAME_TARBALL}" .git
# List the outputs with sizes
ls -laht "${OUTPUT_DIR_ORAS}/linux-shallow-${KERNEL_VERSION}".* || true
echo "::endgroup::"
done
fi
# 5th stage: export complete tree for the active versions, not shallow.
# Will be used for the separate-git+worktree version.
if [[ "${EXPORT_COMPLETE}" == "yes" ]]; then
echo "::group::Exporting complete tree for multiple worktree seeding"
KERNEL_VERSION_COMPLETE_WORKDIR="${COMPLETE_TREES_DIR}/complete1"
display_alert "Making complete tree" "${KERNEL_VERSION_COMPLETE_WORKDIR}"
if [[ ! -d "${KERNEL_VERSION_COMPLETE_WORKDIR}" ]]; then
echo "Empty init..."
git init --initial-branch="armbian_unused_first_branch" "${KERNEL_VERSION_COMPLETE_WORKDIR}"
fi
declare -a WANTED_BRANCHES=()
for KERNEL_VERSION in "${WANTED_KERNEL_VERSIONS[@]}"; do
KERNEL_VERSION_LOCAL_BRANCH_NAME="linux-${KERNEL_VERSION}.y"
WANTED_BRANCHES+=("${KERNEL_VERSION_LOCAL_BRANCH_NAME}:${KERNEL_VERSION_LOCAL_BRANCH_NAME}")
done
# Include a 'master' reference from torvalds-master; this way the produced export has the expected 'master' branch
WANTED_BRANCHES+=("torvalds-master:master")
# Do a single fetch against all the branches...
cd "${KERNEL_VERSION_COMPLETE_WORKDIR}" || exit 3
echo "adding branches ${WANTED_BRANCHES[*]}..."
# --progress --verbose -- too much output for github actions
git fetch "file://${KERNEL_GIT_TREE}" "${WANTED_BRANCHES[@]}"
# list all tags in the complete tree
echo -n "all tags (complete): "
git -C "${KERNEL_VERSION_COMPLETE_WORKDIR}" tag -l | cat | xargs echo -n || true
echo ""
# list all branches in the complete tree
echo -n "all branches (complete):"
git -C "${KERNEL_VERSION_COMPLETE_WORKDIR}" branch -a | cat | xargs echo -n || true
echo ""
# remove hooks, if dir exists
if [[ -d "${KERNEL_VERSION_COMPLETE_WORKDIR}/.git/hooks" ]]; then
rm -rf "${KERNEL_VERSION_COMPLETE_WORKDIR}/.git/hooks"
fi
# show du human total size of the complete tree
echo -n "total size:"
du -hsc "${KERNEL_VERSION_COMPLETE_WORKDIR}"
# export the complete tree
OUTPUT_BUNDLE_FILE_NAME_COMPLETE="${OUTPUT_DIR_ORAS}/linux-complete.git.tar"
echo "Exporting .tar ${OUTPUT_BUNDLE_FILE_NAME_COMPLETE} "
tar cf "${OUTPUT_BUNDLE_FILE_NAME_COMPLETE}" .git
ls -lah "${OUTPUT_BUNDLE_FILE_NAME_COMPLETE}"
echo "::endgroup::"
fi
# In GHA, cache the full git work tree, if it fits in GHA cache;
# If cache hit, skip 1st stage, but _always_ execute stages 2-4 (to update the cache)