From c591f8291b87cc94a57ed25d8793944f2fead338 Mon Sep 17 00:00:00 2001 From: Matthias Kruk Date: Sat, 24 Jul 2021 16:09:31 +0900 Subject: [PATCH] foundry: Implement process and endpoint monitoring To ensure proper operation of the build system, the management script must make sure that the individual processes of foundry are running, and that the IPC endpoints that are used for messaging are existing. This commit adds a process watchdog to foundry.sh which ensures that all processes are running as configured. Further, this commit adds code to ensure that IPC endpoints are existing, and periodically logs the contents of the IPC endpoints' queues to a file so the messages sent between components of the buildsystem can be monitored by an administrator. --- foundry.sh | 182 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 161 insertions(+), 21 deletions(-) mode change 100644 => 100755 foundry.sh diff --git a/foundry.sh b/foundry.sh old mode 100644 new mode 100755 index 6ea9063..831f515 --- a/foundry.sh +++ b/foundry.sh @@ -1,13 +1,73 @@ #!/bin/bash topics=("commits") -endpoints=() -processes=() +FOUNDRY_ROOT="/var/lib/foundry" + +get_endpoint_names() { + local config + + while read -r config; do + conf_get "endpoint" "$config" + done < <(conf_get_domains) + + return 0 +} + +get_process_names() { + local config + + while read -r config; do + echo "$config" + done < <(conf_get_domains) + + return 0 +} + +get_topic_names() { + local config + + while read -r config; do + conf_get "topic" "$config" + done < <(conf_get_domains) + + return 0 +} + +_endpoint_message_log() { + local endpoint="$1" + local msg="$2" + local logfile="$3" + + if ! echo "$msg" >> "$logfile"; then + return 1 + fi + + return 0 +} monitor_endpoints() { local endpoints=("$@") while inst_running; do + local endpoint + + for endpoint in "${endpoints[@]}"; do + local logfile + + logfile="$FOUNDRY_ROOT/endpoints/$endpoint/queue" + logdir="${logfile%/*}" + + if ! mkdir -p "$logdir"; then + continue + fi + + if ! :> "$logfile"; then + continue + fi + + ipc_endpoint_foreach_message "$endpoint" _endpoint_message_log "$logfile" + done + sleep 5 done @@ -15,10 +75,11 @@ monitor_endpoints() { } watch_endpoints() { - local endpoints=("$@") - + local endpoints local endpoint + readarray -t endpoints < <(get_endpoint_names) + # This function starts the monitor that watches all endpoints in the # build system. Because this function is executed before the other # components of the build system are started, endpoints are likely not @@ -31,22 +92,67 @@ watch_endpoints() { fi done - if ! monitor_endpoints "${endpoints[@]}" &; then - return 1 - fi - + monitor_endpoints "${endpoints[@]}" & return 0 } process_is_running() { local process="$1" - return 0 + local command + local proc + + if ! command=$(conf_get "command" "$process"); then + return 1 + fi + + while read -r proc; do + if [[ "$proc" == *" --name $process "* ]]; then + return 0 + fi + done < <(inst_list "$command") + + return 1 } process_start() { local process="$1" + local command + local args + local param + + args=() + command="" + + conf_get_names "$process" | log_highlight "config names" | log_debug + + while read -r param; do + local value + + if ! value=$(conf_get "$param" "$process"); then + continue + fi + + if [[ "$param" == "command" ]]; then + command="$value" + else + args+=("--$param" "$value") + fi + done < <(conf_get_names "$process") + + log_highlight "cmd" <<< "$command ${args[*]}" | log_debug + + if [[ -z "$command" ]]; then + log_error "No command in configuration for $process" + return 1 + fi + + if ! "$command" "${args[@]}"; then + log_error "$command returned an error" + return 1 + fi + return 0 } @@ -55,12 +161,16 @@ process_watchdog() { local process + log_info "Process watchdog ready." + while inst_running; do for process in "${processes[@]}"; do if process_is_running "$process"; then continue fi + log_info "Process $process is not running. Starting it." + if ! process_start "$process"; then log_error "Could not start $process" fi @@ -73,11 +183,21 @@ process_watchdog() { } watch_processes() { - local processes=("$@") + local processes - if ! process_watchdog "${processes[@]}" &; then - return 1 - fi + readarray -t processes < <(conf_get_domains) + array_to_lines "${processes[@]}" | + log_highlight "processes" | + log_info + + process_watchdog "${processes[@]}" & + + return 0 +} + +handle_admin_message() { + local endpoint="$1" + local msg="$2" return 0 } @@ -86,10 +206,30 @@ handle_message() { local endpoint="$1" local msg="$2" - return 1 + local fmsg + local msgtype + + if ! fmsg=$(ipc_msg_get_data "$msg"); then + return 1 + fi + + if ! msgtype=$(foundry_msg_get_type "$fmsg"); then + return 1 + fi + + if [[ "$msgtype" != "admin" ]]; then + log_warn "Ignoring unexpected message" + return 1 + fi + + if ! handle_admin_message "$endpoint" "$fmsg"; then + return 1 + fi + + return 0 } -smelter_run() { +foundry_run() { local endpoint local topic @@ -98,19 +238,19 @@ smelter_run() { return 1 fi - for topic in "${topics[@]}"; do + while read -r topic; do if ! ipc_endpoint_subscribe "$endpoint" "$topic"; then log_error "Could not subscribe to $topic" return 1 fi - done + done < <(get_topic_names) - if ! watch_endpoints "${endpoints[@]}"; then + if ! watch_endpoints; then log_error "Couldn't start endpoint monitor" return 1 fi - if ! watch_processes "${processes[@]}"; then + if ! watch_processes; then log_error "Couldn't start process monitor" return 1 fi @@ -133,7 +273,7 @@ main() { return 1 fi - if ! inst_singleton smelter_run; then + if ! inst_singleton foundry_run; then return 1 fi @@ -145,7 +285,7 @@ main() { exit 1 fi - if ! include "log" "opt" "inst" "ipc" "foundry/msg"; then + if ! include "log" "conf" "opt" "inst" "ipc" "foundry/msg"; then exit 1 fi -- 2.47.3