buildbot: Pick up tasks from a queue and place built packages in another queue

This commit modifies buildbot to pick up tasks from a queue and place built debian
packages in another queue. This allows for very straightforward parallelization of
builds, once the start/stop mechanism has been changed.

diff --git a/buildbot.sh b/buildbot.sh
index a8f58e1e9e784efaa0aeb057cad00a5dd68ee41b..19b2501a34c01358eb90b23ac88956aa099ba459 100644 (file)
--- a/buildbot.sh
+++ b/buildbot.sh
@@ -12,7 +12,7 @@ build_packages() {
 
        npkgs=0
 
-       if ! output=(cd "$sourcetree" && dpkg-buildpackage --sign-key="$gpgkey"); then
+       if ! output=$(cd "$sourcetree" && dpkg-buildpackage --sign-key="$gpgkey"); then
                log_error "Could not build $sourcetree"
                echo "$output" | log_highlight "dpkg-buildpackage" | log_error
 
@@ -41,94 +41,154 @@ build_packages() {
        return 0
 }
 
-queue_put_package() {
-       local queue="$1"
+build_repository() {
+       local target="$1"
+       local gpgkey="$2"
+       local workdir="$3"
+
+       local repository
+       local branch
+       local buildroot
+       local packages
+       local err
+
+       err=1
 
-       local lock
-       local sem
+       # $target points to refs/heads/<branch>, so we need to mangle it a bit
+       repository="${target%/refs/heads*}" # repository might be bare
+       repository="${repository%/.git}"    # or not bare
+       branch="${target##*/}"
 
-       lock="$queue/lock"
-       sem="$queue/sem"
+       log_info "Going to build $repository#$branch"
 
-       if ! mutex_lock "$lock"; then
-               log_error "Could not acquire lock on $queue"
+       buildroot="$workdir/buildroot"
+
+       if ! mkdir -p "$buildroot"; then
+               log_error "Could not create buildroot"
                return 1
        fi
 
-       log_info "Moving $package to $queue"
-       if ! mv "$package" "$queue/queue"; then
-               log_error "Could not move $package"
+       if ! git clone "$repository" -b "$branch" "$buildroot" &>/dev/null; then
+               log_error "Could not clone $repository#$branch to $buildroot"
+       elif ! packages=$(build_packages "$buildroot" "$gpgkey"); then
+               log_error "Could not build package"
        else
-               if ! sem_post "$sem"; then
-                       log_error "Could not post semaphore $sem"
-               fi
+               log_info "Build succeeded: $repository"
+               echo "$packages"
+               err=0
        fi
 
-       mutex_unlock "$lock"
-
-       return 0
+       return "$err"
 }
 
-move_packages() {
-       local queue="$1"
+dispatch_tasks() {
+       local gpgkey="$1"
+       local taskq="$2"
+       local doneq="$3"
 
-       local lock
-       local sem
+       if ! sem_init "$sem" 0; then
+               log_error "Another instance is already running"
+               return 1
+       fi
 
-       lock="$resultdir/lock"
-       sem="$resultdir/queued"
+       while ! sem_trywait "$sem"; do
+               local workitem
+               local workdir
+               local package
+               local result
 
-       while read -r package; do
-               queue_put_package "$queue" "$package"
-       done
+               if ! workdir=$(mktemp -d); then
+                       log_error "Could not create workdir"
+                       return 1
+               fi
 
-       return 0
-}
+               workitem=$(queue_get "$taskq")
 
-build_repository() {
-       local repository="$1"
-       local gpgkey="$2"
-       local resultdir="$3"
+               log_info "Starting build of $workitem"
 
-       local buildroot
-       local packages
-       local err
+               if ! result=$(build_repository "$workitem" "$gpgkey" "$workdir"); then
+                       log_error "Build of $workitem failed"
+                       continue
+               fi
 
-       err=1
+               while read -r package; do
+                       while ! queue_put_file "$doneq" "$package"; do
+                               log_error "Could not put $package in queue. Trying again in a bit."
+                               log_error "This usually means the disk with the queue is full, or permissions have been changed."
+                               sleep 60
+                       done
+               done <<< "$result"
 
-       if ! buildroot=$(mktemp -d); then
-               log_error "Could not create buildroot"
-               return 1
-       fi
+               if ! rm -rf "$workdir"; then
+                       log_error "Could not remove workdir $workdir"
+               fi
+       done
 
-       if ! git clone "$repository" "$buildroot/source" &>/dev/null; then
-               log_error "Could not checkout source"
-       elif ! packages=$(build_packages "$buildroot/source" "$gpgkey"); then
-               log_error "Could not build package"
-       elif ! move_packages "$resultdir" <<< "$packages"; then
-               log_error "Could not move packages"
-       else
-               log_info "Build succeeded: $repository"
-               err=0
+       if ! sem_destroy "$sem"; then
+               log_error "Could not clean up semaphore $sem"
        fi
 
-       if ! rm -rf "$buildroot"; then
-               log_error "Could not clean up $buildroot"
+       return 0
+}
+
+stop() {
+       if ! sem_post "$sem"; then
+               log_error "Looks like no other instances are running"
+               return 1
        fi
 
-       return "$err"
+       return 0
 }
 
+verbosity_increase() {
+       local verbosity
+
+       verbosity=$(log_get_verbosity)
+       ((verbosity++))
+       log_set_verbosity "$verbosity"
+       return 0
+}
+
+verbosity_decrease() {
+       local verbosity
+
+       verbosity=$(log_get_verbosity)
+       ((verbosity--))
+       log_set_verbosity "$verbosity"
+       return 0
+}
 
 main() {
+       local gpgkey
+       local tqueue
+       local dqueue
+
        opt_add_arg "k" "gpgkey"     "yes" "" "The GPG key id to use"
-       opt_add_arg "q" "queue"      "yes" "" "The queue where new packages should be placed"
-       opt_add_arg "r" "repository" "yes" "" "The repository to watch"
+       opt_add_arg "t" "task-queue" "yes" "" "The queue to watch for tasks"
+       opt_add_arg "d" "done-queue" "yes" "" "The queue to place build artifacts"
+       opt_add_arg "v" "verbose"    "no" 0 "Be more verbose"                      verbosity_increase
+       opt_add_arg "w" "shush"      "no" 0 "Be less verbose"                      verbosity_decrease
+       opt_add_arg "s" "stop"       "no" 0 "Stop the running instance"
 
        if ! opt_parse "$@"; then
                return 1
        fi
 
+       if (( $(opt_get "stop") > 0 )); then
+               if ! stop; then
+                       return 1
+               fi
+
+               return 0
+       fi
+
+       gpgkey=$(opt_get "gpgkey")
+       tqueue=$(opt_get "task-queue")
+       dqueue=$(opt_get "done-queue")
+
+       dispatch_tasks "$gpgkey" "$tqueue" "$dqueue" </dev/null &>/dev/null &
+       disown
+
        return 0
 }
 
@@ -137,7 +197,7 @@ main() {
                exit 1
        fi
 
-       if ! include "log" "opt" "conf" "sem"; then
+       if ! include "log" "opt" "queue"; then
                exit 1
        fi