From 8e3d92635df787aa6315668ba90a2b4905ea4922 Mon Sep 17 00:00:00 2001 From: Matthias Kruk Date: Thu, 15 Apr 2021 10:19:19 +0900 Subject: [PATCH] buildbot: Pick up tasks from a queue and place built packages in another queue This commit modifies buildbot to pick up tasks from a queue and place built debian packages in another queue. This allows for very straightforward parallelization of builds, once the start/stop mechanism has been changed. --- buildbot.sh | 172 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 56 deletions(-) diff --git a/buildbot.sh b/buildbot.sh index a8f58e1..19b2501 100644 --- a/buildbot.sh +++ b/buildbot.sh @@ -12,7 +12,7 @@ build_packages() { npkgs=0 - if ! output=(cd "$sourcetree" && dpkg-buildpackage --sign-key="$gpgkey"); then + if ! output=$(cd "$sourcetree" && dpkg-buildpackage --sign-key="$gpgkey"); then log_error "Could not build $sourcetree" echo "$output" | log_highlight "dpkg-buildpackage" | log_error @@ -41,94 +41,154 @@ build_packages() { return 0 } -queue_put_package() { - local queue="$1" +build_repository() { + local target="$1" + local gpgkey="$2" + local workdir="$3" + + local repository + local branch + local buildroot + local packages + local err + + err=1 - local lock - local sem + # $target points to refs/heads/, so we need to mangle it a bit + repository="${target%/refs/heads*}" # repository might be bare + repository="${repository%/.git}" # or not bare + branch="${target##*/}" - lock="$queue/lock" - sem="$queue/sem" + log_info "Going to build $repository#$branch" - if ! mutex_lock "$lock"; then - log_error "Could not acquire lock on $queue" + buildroot="$workdir/buildroot" + + if ! mkdir -p "$buildroot"; then + log_error "Could not create buildroot" return 1 fi - log_info "Moving $package to $queue" - if ! mv "$package" "$queue/queue"; then - log_error "Could not move $package" + if ! git clone "$repository" -b "$branch" "$buildroot" &>/dev/null; then + log_error "Could not clone $repository#$branch to $buildroot" + elif ! packages=$(build_packages "$buildroot" "$gpgkey"); then + log_error "Could not build package" else - if ! sem_post "$sem"; then - log_error "Could not post semaphore $sem" - fi + log_info "Build succeeded: $repository" + echo "$packages" + err=0 fi - mutex_unlock "$lock" - - return 0 + return "$err" } -move_packages() { - local queue="$1" +dispatch_tasks() { + local gpgkey="$1" + local taskq="$2" + local doneq="$3" - local lock - local sem + if ! sem_init "$sem" 0; then + log_error "Another instance is already running" + return 1 + fi - lock="$resultdir/lock" - sem="$resultdir/queued" + while ! sem_trywait "$sem"; do + local workitem + local workdir + local package + local result - while read -r package; do - queue_put_package "$queue" "$package" - done + if ! workdir=$(mktemp -d); then + log_error "Could not create workdir" + return 1 + fi - return 0 -} + workitem=$(queue_get "$taskq") -build_repository() { - local repository="$1" - local gpgkey="$2" - local resultdir="$3" + log_info "Starting build of $workitem" - local buildroot - local packages - local err + if ! result=$(build_repository "$workitem" "$gpgkey" "$workdir"); then + log_error "Build of $workitem failed" + continue + fi - err=1 + while read -r package; do + while ! queue_put_file "$doneq" "$package"; do + log_error "Could not put $package in queue. Trying again in a bit." + log_error "This usually means the disk with the queue is full, or permissions have been changed." + sleep 60 + done + done <<< "$result" - if ! buildroot=$(mktemp -d); then - log_error "Could not create buildroot" - return 1 - fi + if ! rm -rf "$workdir"; then + log_error "Could not remove workdir $workdir" + fi + done - if ! git clone "$repository" "$buildroot/source" &>/dev/null; then - log_error "Could not checkout source" - elif ! packages=$(build_packages "$buildroot/source" "$gpgkey"); then - log_error "Could not build package" - elif ! move_packages "$resultdir" <<< "$packages"; then - log_error "Could not move packages" - else - log_info "Build succeeded: $repository" - err=0 + if ! sem_destroy "$sem"; then + log_error "Could not clean up semaphore $sem" fi - if ! rm -rf "$buildroot"; then - log_error "Could not clean up $buildroot" + return 0 +} + +stop() { + if ! sem_post "$sem"; then + log_error "Looks like no other instances are running" + return 1 fi - return "$err" + return 0 } +verbosity_increase() { + local verbosity + + verbosity=$(log_get_verbosity) + ((verbosity++)) + log_set_verbosity "$verbosity" + return 0 +} + +verbosity_decrease() { + local verbosity + + verbosity=$(log_get_verbosity) + ((verbosity--)) + log_set_verbosity "$verbosity" + return 0 +} main() { + local gpgkey + local tqueue + local dqueue + opt_add_arg "k" "gpgkey" "yes" "" "The GPG key id to use" - opt_add_arg "q" "queue" "yes" "" "The queue where new packages should be placed" - opt_add_arg "r" "repository" "yes" "" "The repository to watch" + opt_add_arg "t" "task-queue" "yes" "" "The queue to watch for tasks" + opt_add_arg "d" "done-queue" "yes" "" "The queue to place build artifacts" + opt_add_arg "v" "verbose" "no" 0 "Be more verbose" verbosity_increase + opt_add_arg "w" "shush" "no" 0 "Be less verbose" verbosity_decrease + opt_add_arg "s" "stop" "no" 0 "Stop the running instance" if ! opt_parse "$@"; then return 1 fi + if (( $(opt_get "stop") > 0 )); then + if ! stop; then + return 1 + fi + + return 0 + fi + + gpgkey=$(opt_get "gpgkey") + tqueue=$(opt_get "task-queue") + dqueue=$(opt_get "done-queue") + + dispatch_tasks "$gpgkey" "$tqueue" "$dqueue" /dev/null & + disown + return 0 } @@ -137,7 +197,7 @@ main() { exit 1 fi - if ! include "log" "opt" "conf" "sem"; then + if ! include "log" "opt" "queue"; then exit 1 fi -- 2.47.3