armbian-next: introduce ORAS-based kernel git bare tree seeding/bundles
- armbian-next: fixes to ORAS-related logging - armbian-next: actually enable ORAS-based kernel git bare tree seeding/bundles; enable cleaning of bundle articfacts after confirmed working - armbian-next: introduce ORAS-based kernel git bare tree seeding/bundles (although it is a .tar, not a bundle); this is 20x faster than cloning - armbian-next: introduce `ORAS` tooling; pull and push functions & downloader/launcher
This commit is contained in:
Ricardo Pardini
parent
76e276c6a9
commit
2c0e9182ed
@ -1,4 +1,4 @@
|
||||
# This is run under do_with_retries.
|
||||
# This is NOT run under do_with_retries.
|
||||
function download_git_kernel_bundle() {
|
||||
# See https://mirrors.edge.kernel.org/pub/scm/.bundles/pub/scm/linux/kernel/git/
|
||||
|
||||
@ -38,6 +38,44 @@ function download_git_kernel_bundle() {
|
||||
return 0
|
||||
}
|
||||
|
||||
function download_git_kernel_gitball_via_oras() {
|
||||
declare git_bundles_dir="${SRC}/cache/git-bundles/kernel"
|
||||
declare git_kernel_ball_fn="linux-complete.git.tar"
|
||||
|
||||
run_host_command_logged mkdir -p "${git_bundles_dir}"
|
||||
|
||||
# defines outer scope value
|
||||
linux_kernel_clone_tar_file="${git_bundles_dir}/${git_kernel_ball_fn}"
|
||||
|
||||
# if the file already exists, do nothing
|
||||
if [[ -f "${linux_kernel_clone_tar_file}" ]]; then
|
||||
display_alert "Kernel git-tarball already exists" "${git_kernel_ball_fn}" "cachehit"
|
||||
return 0
|
||||
fi
|
||||
|
||||
#do_with_retries 5 xxx ?
|
||||
oras_pull_artifact_file "ghcr.io/rpardini/armbian-git-shallow/kernel-git:latest" "${git_bundles_dir}" "${git_kernel_ball_fn}"
|
||||
|
||||
# sanity check
|
||||
if [[ ! -f "${linux_kernel_clone_tar_file}" ]]; then
|
||||
exit_with_error "Kernel git-tarball download failed ${linux_kernel_clone_tar_file}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
|
||||
}
|
||||
|
||||
function kernel_cleanup_bundle_artifacts() {
|
||||
declare git_bundles_dir="${SRC}/cache/git-bundles/kernel"
|
||||
|
||||
if [[ -d "${git_bundles_dir}" ]]; then
|
||||
display_alert "Cleaning up Kernel git bundle artifacts" "no longer needed" "cachehit"
|
||||
run_host_command_logged rm -rf "${git_bundles_dir}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function kernel_download_bundle_with_axel() {
|
||||
display_alert "Downloading Kernel bundle" "${bundle_type}; this might take a long time" "info"
|
||||
declare -a verbose_params=()
|
||||
@ -47,6 +85,44 @@ function kernel_download_bundle_with_axel() {
|
||||
"${linux_clone_bundle_url}"
|
||||
}
|
||||
|
||||
function kernel_prepare_bare_repo_from_oras_gitball() {
|
||||
kernel_git_bare_tree="${SRC}/cache/git-bare/kernel" # sets the outer scope variable
|
||||
declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/.git/armbian-bare-tree-done"
|
||||
|
||||
if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then
|
||||
display_alert "Preparing bare kernel git tree" "this might take a long time" "info"
|
||||
|
||||
if [[ -d "${kernel_git_bare_tree}" ]]; then
|
||||
display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info"
|
||||
run_host_command_logged rm -rf "${kernel_git_bare_tree}"
|
||||
fi
|
||||
|
||||
# now, make sure we've the bundle downloaded correctly...
|
||||
# this defines linux_kernel_clone_bundle_file
|
||||
declare linux_kernel_clone_tar_file
|
||||
download_git_kernel_gitball_via_oras # sets linux_kernel_clone_tar_file or dies
|
||||
|
||||
# Just extract the tar_file into the "${kernel_git_bare_tree}" directory, no further work needed.
|
||||
run_host_command_logged mkdir -p "${kernel_git_bare_tree}"
|
||||
# @TODO chance of a pv thingy here?
|
||||
run_host_command_logged tar -xf "${linux_kernel_clone_tar_file}" -C "${kernel_git_bare_tree}"
|
||||
|
||||
# sanity check
|
||||
if [[ ! -d "${kernel_git_bare_tree}/.git" ]]; then
|
||||
exit_with_error "Kernel bare tree is missing .git directory ${kernel_git_bare_tree}"
|
||||
fi
|
||||
|
||||
# write the marker file
|
||||
touch "${kernel_git_bare_tree_done_marker}"
|
||||
else
|
||||
display_alert "Kernel bare tree already exists" "${kernel_git_bare_tree}" "cachehit"
|
||||
fi
|
||||
|
||||
git_ensure_safe_directory "${kernel_git_bare_tree}"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function kernel_prepare_bare_repo_from_bundle() {
|
||||
kernel_git_bare_tree="${SRC}/cache/git-bare/kernel" # sets the outer scope variable
|
||||
declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/.git/armbian-bare-tree-done"
|
||||
@ -82,6 +158,7 @@ function kernel_prepare_git_pre_fetch() {
|
||||
|
||||
# shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context.
|
||||
if [[ "${do_add_origin}" == "yes" ]]; then
|
||||
# @TODO: let's not add remotes anymore please? it's unwieldy and unnecessary
|
||||
display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git"
|
||||
regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source
|
||||
|
||||
|
||||
@ -12,8 +12,10 @@ function compile_kernel() {
|
||||
|
||||
# Prepare the git bare repo for the kernel; shared between all kernel builds
|
||||
declare kernel_git_bare_tree
|
||||
LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \
|
||||
kernel_prepare_bare_repo_from_bundle # this sets kernel_git_bare_tree
|
||||
# alternative # LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \
|
||||
# alternative # kernel_prepare_bare_repo_from_bundle # this sets kernel_git_bare_tree
|
||||
LOG_SECTION="kernel_prepare_bare_repo_from_oras_gitball" do_with_logging_unless_user_terminal do_with_hooks \
|
||||
kernel_prepare_bare_repo_from_oras_gitball # this sets kernel_git_bare_tree
|
||||
|
||||
# prepare the working copy; this is the actual kernel source tree for this build
|
||||
declare checked_out_revision_mtime="" checked_out_revision_ts="" checked_out_revision="undetermined" # set by fetch_from_repo
|
||||
@ -48,6 +50,9 @@ function compile_kernel() {
|
||||
|
||||
rm -f linux-firmware-image-*.deb # remove firmware image packages here - easier than patching ~40 packaging scripts at once
|
||||
run_host_command_logged rsync --remove-source-files -r ./*.deb "${DEB_STORAGE}/"
|
||||
|
||||
# kernel build worked; let's clean up the git-bundle cache, since the git-bare cache is proven working.
|
||||
kernel_cleanup_bundle_artifacts
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
114
lib/functions/general/oci-oras.sh
Normal file
114
lib/functions/general/oci-oras.sh
Normal file
@ -0,0 +1,114 @@
|
||||
function run_tool_oras() {
|
||||
# Default version
|
||||
ORAS_VERSION=${ORAS_VERSION:-0.16.0} # https://github.com/oras-project/oras/releases
|
||||
|
||||
if [[ -z "${DIR_ORAS}" ]]; then
|
||||
display_alert "DIR_ORAS is not set, using default" "ORAS" "debug"
|
||||
if [[ -n "${SRC}" ]]; then
|
||||
DIR_ORAS="${SRC}/cache/tools/oras"
|
||||
else
|
||||
display_alert "Missing DIR_ORAS, or SRC fallback" "DIR_ORAS: ${DIR_ORAS}; SRC: ${SRC}" "ORAS" "err"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
display_alert "DIR_ORAS is set to ${DIR_ORAS}" "ORAS" "debug"
|
||||
fi
|
||||
|
||||
mkdir -p "${DIR_ORAS}"
|
||||
|
||||
declare MACHINE="${BASH_VERSINFO[5]}" ORAS_OS ORAS_ARCH
|
||||
display_alert "Running ORAS" "ORAS version ${ORAS_VERSION}" "debug"
|
||||
MACHINE="${BASH_VERSINFO[5]}"
|
||||
case "$MACHINE" in
|
||||
*darwin*) ORAS_OS="darwin" ;;
|
||||
*linux*) ORAS_OS="linux" ;;
|
||||
*)
|
||||
exit_with_error "unknown os: $MACHINE"
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$MACHINE" in
|
||||
*aarch64*) ORAS_ARCH="arm64" ;;
|
||||
*x86_64*) ORAS_ARCH="amd64" ;;
|
||||
*)
|
||||
exit_with_error "unknown arch: $MACHINE"
|
||||
;;
|
||||
esac
|
||||
|
||||
declare ORAS_FN="oras_${ORAS_VERSION}_${ORAS_OS}_${ORAS_ARCH}"
|
||||
declare ORAS_FN_TARXZ="${ORAS_FN}.tar.gz"
|
||||
declare DOWN_URL="https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/${ORAS_FN_TARXZ}"
|
||||
declare ORAS_BIN="${DIR_ORAS}/${ORAS_FN}"
|
||||
declare ACTUAL_VERSION
|
||||
|
||||
if [[ ! -f "${ORAS_BIN}" ]]; then
|
||||
display_alert "Cache miss, downloading..."
|
||||
display_alert "MACHINE: ${MACHINE}" "ORAS" "debug"
|
||||
display_alert "Down URL: ${DOWN_URL}" "ORAS" "debug"
|
||||
display_alert "ORAS_BIN: ${ORAS_BIN}" "ORAS" "debug"
|
||||
|
||||
display_alert "Downloading required" "ORAS tooling" "info"
|
||||
run_host_command_logged wget --progress=dot:giga -O "${ORAS_BIN}.tar.gz" "${DOWN_URL}"
|
||||
run_host_command_logged tar -xf "${ORAS_BIN}.tar.gz" -C "${DIR_ORAS}" "oras"
|
||||
run_host_command_logged rm -rf "${ORAS_BIN}.tar.gz"
|
||||
run_host_command_logged mv -v "${DIR_ORAS}/oras" "${ORAS_BIN}"
|
||||
run_host_command_logged chmod -v +x "${ORAS_BIN}"
|
||||
fi
|
||||
ACTUAL_VERSION="$("${ORAS_BIN}" version | grep "^Version" | xargs echo -n)"
|
||||
display_alert "Running ORAS ${ACTUAL_VERSION}" "ORAS" "debug"
|
||||
|
||||
# Run oras with it
|
||||
display_alert "Calling ORAS" "$*" "debug"
|
||||
"${ORAS_BIN}" "$@"
|
||||
}
|
||||
|
||||
function oras_push_artifact_file() {
|
||||
declare image_full_oci="${1}" # Something like "ghcr.io/rpardini/armbian-git-shallow/kernel-git:latest"
|
||||
declare upload_file="${2}" # Absolute path to the file to upload including the path and name
|
||||
declare upload_file_base_path upload_file_name
|
||||
display_alert "Pushing ${upload_file}" "ORAS to ${image_full_oci}" "info"
|
||||
|
||||
# make sure file exists
|
||||
if [[ ! -f "${upload_file}" ]]; then
|
||||
display_alert "File not found: ${upload_file}" "ORAS upload" "err"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# split the path and the filename
|
||||
upload_file_base_path="$(dirname "${upload_file}")"
|
||||
upload_file_name="$(basename "${upload_file}")"
|
||||
display_alert "upload_file_base_path: ${upload_file_base_path}" "ORAS upload" "debug"
|
||||
display_alert "upload_file_name: ${upload_file_name}" "ORAS upload" "debug"
|
||||
|
||||
pushd "${upload_file_base_path}" || exit_with_error "Failed to pushd to ${upload_file_base_path} - ORAS upload"
|
||||
run_tool_oras push --verbose "${image_full_oci}" "${upload_file_name}:application/vnd.unknown.layer.v1+tar"
|
||||
popd || exit_with_error "Failed to popd" "ORAS upload"
|
||||
return 0
|
||||
}
|
||||
|
||||
# oras pull is very hard to work with, since we don't determine the filename until after the download.
|
||||
function oras_pull_artifact_file() {
|
||||
declare image_full_oci="${1}" # Something like "ghcr.io/rpardini/armbian-git-shallow/kernel-git:latest"
|
||||
declare target_dir="${2}" # temporary directory we'll use for the download to workaround oras being maniac
|
||||
declare target_fn="${3}"
|
||||
|
||||
declare full_temp_dir="${target_dir}/${target_fn}.oras.pull.tmp"
|
||||
declare full_tmp_file_path="${full_temp_dir}/${target_fn}"
|
||||
run_host_command_logged mkdir -p "${full_temp_dir}"
|
||||
|
||||
pushd "${full_temp_dir}" &> /dev/null || exit_with_error "Failed to pushd to ${full_temp_dir} - ORAS download"
|
||||
run_tool_oras pull --verbose "${image_full_oci}"
|
||||
popd &> /dev/null || exit_with_error "Failed to popd - ORAS download"
|
||||
|
||||
# sanity check; did we get the file we expected?
|
||||
if [[ ! -f "${full_tmp_file_path}" ]]; then
|
||||
exit_with_error "File not found after ORAS pull: ${full_tmp_file_path} - ORAS download"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# move the file to the target directory
|
||||
run_host_command_logged mv "${full_tmp_file_path}" "${target_dir}"
|
||||
|
||||
# remove the temp directory
|
||||
run_host_command_logged rm -rf "${full_temp_dir}"
|
||||
}
|
||||
16
lib/tools/oras.sh
Executable file
16
lib/tools/oras.sh
Executable file
@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
SRC="$(
|
||||
cd "$(dirname "$0")/../.."
|
||||
pwd -P
|
||||
)"
|
||||
|
||||
# shellcheck source=lib/single.sh
|
||||
source "${SRC}"/lib/single.sh
|
||||
|
||||
# initialize logging variables.
|
||||
logging_init
|
||||
|
||||
run_tool_oras "$@"
|
||||
Loading…
x
Reference in New Issue
Block a user