bash-aliases

#!/bin/bash 
### This file ~/wsprdaemon/bash-aliases  includes functions and aliases which are helpful in running and debugging WD systems
#
#

function wd-decoding-logs-times() {
    ls -lt `find /dev/shm/wsprdaemon/recording.d/ -name decoding_daemon.log`
}

function wd-kiwi-logtimes() {
    date
    ls -l ~/wsprdaemon/watchdog_daemon.pid
    ls -lt /dev/shm/wsprdaemon/recording.d/KIWI_*/*/decoding_daemon.log | tail -5
}

get-radiod-restart-logs() {
    local show_previous=0
    if [[ "${1:-}" == "-1" ]]; then
        show_previous=1
    fi

    local services
    services=$(systemctl list-units --type=service --state=running --no-legend | awk '/radiod@/ {print $1}')

    if [[ -z "${services}" ]]; then
        echo "No running radiod@* services found"
        return 0
    fi

    local service
    while read -r service; do
        [[ -z "${service}" ]] && continue

        local start_timestamp
        start_timestamp=$(systemctl show -p ActiveEnterTimestamp --value "${service}")

        echo "=== ${service} ==="
        if (( show_previous )); then
            # Show restarts up to and including current start
            local until_time
            until_time=$(date -d "${start_timestamp}" "+%Y-%m-%d %H:%M:%S")
            sudo journalctl -u "${service}" --boot --until="${until_time}" --no-pager | grep -i "restart"
        else
            # Show restarts after current start
            local since_time
            since_time=$(date -d "${start_timestamp} + 1 second" "+%Y-%m-%d %H:%M:%S")
            sudo journalctl -u "${service}" --since="${since_time}" --no-pager | grep -i "restart"
        fi
        echo "=== Started: ${start_timestamp} ==="
        echo
    done <<< "${services}"
}

alias chc='clickhouse-client'
alias ch=chc

wd-sftp-wd-test() {
    local rc
    local start_epoch
    local end_epoch
    local runtime_seconds

    source ~/wsprdaemon/wsprdaemon.conf

    if ! declare -p WD_SERVER_USER_LIST &> /dev/null; then
        echo "ERROR: WD_SERVER_USER_LIST is not defined in ~/wsprdaemon/wsprdaemon.conf"
        echo "This WD client is not configured to sftp upload extended spots to the WD servers"
        return 1
    fi

    if [[ ${#WD_SERVER_USER_LIST[@]} -eq 0 ]]; then
        echo "ERROR: WD_SERVER_USER_LIST is defined but empty"
        return 1
    fi

    echo "Found WD_SERVER_USER_LIST with ${#WD_SERVER_USER_LIST[@]} server(s):"
    for entry in "${WD_SERVER_USER_LIST[@]}"; do
        echo "  $entry"
    done
    echo ""

    if [[ ! -d ~/.ssh ]]; then
        echo "ERROR: The ~/.ssh directory which contains ssh keys doesn't exist."
        echo "Run 'ssh-keygen -t ed25519 -N \"\"' to create them"
        return 1
    fi

    local public_key_file_path_list=($(find ~/.ssh -name '*.pub'))
    if [[ ${#public_key_file_path_list[@]} -eq 0 ]]; then
        echo "ERROR: No ssh public key found in ~/.ssh"
        echo "Run 'ssh-keygen -t ed25519 -N \"\"' to create one"
        return 1
    fi

    local public_key_file_path="${public_key_file_path_list[0]}"
    (( ${verbosity-0} )) && echo "Public key file: ${public_key_file_path}"
    (( ${verbosity-0} )) && echo "Public key: $(< ${public_key_file_path})"
    echo ""

    local tmpfile=$(mktemp)
    local testfile=$(mktemp)
    echo "sftp test $(date)" > "${testfile}"
    trap "rm -f '$tmpfile' '$testfile'" RETURN

    local nc_timeout=${WD_SERVER_NC_TIMEOUT_SECONDS:-2}
    local sftp_timeout=$(( nc_timeout * 10 ))
    local overall_rc=0

    for server_entry in "${WD_SERVER_USER_LIST[@]}"; do
        local server_user="${server_entry%@*}"
        local server_url="${server_entry#*@}"

        echo "========================================"
        echo "Testing: ${server_entry}"
        echo "========================================"

        # Test 1: TCP connectivity
        echo "  Testing TCP connectivity: nc -vz -w ${nc_timeout} ${server_url} 22"
        start_epoch=${EPOCHSECONDS}
        nc -vz -w ${nc_timeout} ${server_url} 22 &> ${tmpfile}
        rc=$?
        end_epoch=${EPOCHSECONDS}
        runtime_seconds=$(( end_epoch - start_epoch ))

        if (( rc )); then
            printf "  ✗ After %2d seconds, nc failed (rc=${rc}): %s\n" "${runtime_seconds}" "$(< ${tmpfile})"
            overall_rc=1
            echo ""
            continue
        fi
        printf "  ✓ After %2d seconds, TCP connection to ${server_url}:22 successful\n" "${runtime_seconds}"

        # Test 2: SFTP autologin with REAL host key checking (matches upload function)
        echo "  Testing SFTP autologin (with host key check): sftp -o BatchMode=yes ${server_entry}"
        start_epoch=${EPOCHSECONDS}
        sftp -o BatchMode=yes -o ConnectTimeout=${sftp_timeout} -o StrictHostKeyChecking=accept-new -b /dev/null "${server_entry}" &> ${tmpfile}
        rc=$?
        end_epoch=${EPOCHSECONDS}
        runtime_seconds=$(( end_epoch - start_epoch ))

        if (( rc == 0 )); then
            printf "  ✓ After %2d seconds, SFTP autologin to ${server_entry} successful\n" "${runtime_seconds}"
        else
            # Check for host key issues
            if grep -q "REMOTE HOST IDENTIFICATION HAS CHANGED" "${tmpfile}" 2>/dev/null; then
                printf "  ✗ After %2d seconds, SFTP failed - HOST KEY CHANGED\n" "${runtime_seconds}"
                echo "    The server's SSH key has changed. Run:"
                echo "      ssh-keygen -R ${server_url}"
                echo "    Then re-run this test to accept the new key."
            elif grep -q "Host key verification failed\|not known\|No ED25519 host key" "${tmpfile}" 2>/dev/null; then
                printf "  ✗ After %2d seconds, SFTP failed - HOST KEY UNKNOWN\n" "${runtime_seconds}"
                echo "    Run: ssh-keyscan -H ${server_url} >> ~/.ssh/known_hosts"
            else
                printf "  ✗ After %2d seconds, SFTP autologin failed (rc=${rc})\n" "${runtime_seconds}"
                echo "    Output: $(< ${tmpfile})"
                echo ""
                echo "    Verify that this public key is registered on ${server_url}:"
                echo "    $(< ${public_key_file_path})"
            fi
            overall_rc=1
            echo ""
            continue
        fi

        # Test 3: Actual file upload/delete (catches permission issues)
        echo "  Testing file upload: put/rm test file"
        start_epoch=${EPOCHSECONDS}
        sftp -o BatchMode=yes -o ConnectTimeout=${sftp_timeout} -o StrictHostKeyChecking=accept-new "${server_entry}" <<EOF &> ${tmpfile}
put ${testfile} uploads/.sftp_test_$$
rm uploads/.sftp_test_$$
EOF
        rc=$?
        end_epoch=${EPOCHSECONDS}
        runtime_seconds=$(( end_epoch - start_epoch ))

        if (( rc == 0 )); then
            printf "  ✓ After %2d seconds, file upload test successful\n" "${runtime_seconds}"
        else
            printf "  ✗ After %2d seconds, file upload test failed (rc=${rc})\n" "${runtime_seconds}"
            echo "    Output: $(< ${tmpfile})"
            overall_rc=1
        fi
        echo ""
    done

    if (( overall_rc == 0 )); then
        echo "========================================"
        echo "✓ All ${#WD_SERVER_USER_LIST[@]} server(s) passed all tests"
        echo "========================================"
    else
        echo "========================================"
        echo "✗ Some servers failed tests"
        echo "========================================"
    fi

    return ${overall_rc}
}

### TBD: This function should be used by assign_fft_cores()
function show_hyperthread_pairs()
{
    local core_id=$1

    # If a specific core is requested
    if [[ -n $core_id ]]; then
        if [[ -f /sys/devices/system/cpu/cpu${core_id}/topology/thread_siblings_list ]]; then
            local siblings=$(cat /sys/devices/system/cpu/cpu${core_id}/topology/thread_siblings_list)
            echo "CPU $core_id: siblings = $siblings"
        else
            echo "ERROR: CPU $core_id not found"
            return 1
        fi
        return 0
    fi

    # Show all unique physical cores and their hyperthreads
    declare -A seen

    for cpu in /sys/devices/system/cpu/cpu[0-9]*; do
        [[ -f $cpu/topology/thread_siblings_list ]] || continue

        local core_num=$(basename $cpu | sed 's/cpu//')
        local siblings=$(cat $cpu/topology/thread_siblings_list)

        # Skip if we've already seen this sibling pair
        [[ ${seen[$siblings]} ]] && continue
        seen[$siblings]=1

        echo "Physical core: $siblings"
    done
}

### In ascending PID order restrict each fft thread to its own CPU core and restrict the fft's proc_888 thread and all 40+ the lin threads to 
### the hyperthread pair of the FFT's core
### TBD: find the hyperthread pair.  This code assumes it is CPU + 1
function assign_fft_cores() {
    local cores=(0 2 4 6)
    local core_index=0
    echo "Assigning FFT threads and their siblings to cores..."
    echo "==================================================="

    # Find all threads named 'fft', sort by PID
    local fft_pids=$(ps -eLo tid,comm | grep '[[:space:]]fft$' | awk '{print $1}' | sort -n)

    if [ -z "$fft_pids" ]; then
        echo "No threads named 'fft' found."
        return 1
    fi

    # Assign each thread to a core
    for pid in $fft_pids; do
        local core=${cores[$core_index]}
        local other_core=$(( core + 1 ))

        # Set CPU affinity of FFT thread using taskset
        if sudo taskset -cp $core $pid > /dev/null 2>&1; then
            echo "FFT  Thread $pid -> Core $core"
        else
            echo "FFT  Failed to assign thread $pid to core $core"
        fi

        # Get the thread group ID (main process ID) from /proc
        local tgid=$(grep "^Tgid:" /proc/$pid/status 2>/dev/null | awk '{print $2}')

        if [ -z "$tgid" ]; then
            echo "Could not determine TGID for thread $pid"
            core_index=$(( (core_index + 1) % ${#cores[@]} ))
            continue
        fi

        # Assign ALL other threads in the same process to the adjacent core
        local assigned_count=0
        if [ -d "/proc/$tgid/task" ]; then
            for task in /proc/$tgid/task/*; do
                local task_tid=$(basename "$task")

                # Skip the fft thread itself
                if [ "$task_tid" = "$pid" ]; then
                    continue
                fi

                # Assign this thread to the adjacent core
                if sudo taskset -cp $other_core $task_tid > /dev/null 2>&1; then
                    local task_name=$(cat "$task/comm" 2>/dev/null)
                    echo "     Thread $task_tid ($task_name) -> Core $other_core"
                    assigned_count=$(( assigned_count + 1 ))
                fi
            done
        fi

        echo "  Assigned $assigned_count sibling threads to core $other_core"
        echo ""

        # Move to next core pair, wrap around if needed
        core_index=$(( (core_index + 1) % ${#cores[@]} ))
    done

    echo "Assignment complete."
}

function rx888-info() {
    # The RX888 uses the Cypress FX3 which has vendor ID 04b4, so get a list of Cypress devices
    lsusb -d 04b4: 2>/dev/null | while read -r line; do
        # Extract bus and device numbers
        bus=$(echo "$line" | awk '{print $2}')
        device=$(echo "$line" | awk '{print $4}' | tr -d ':')
        product=$(echo "$line" | cut -d' ' -f7-)

        # Get detailed info including serial number
        serial=$(lsusb -v -s "$bus:$device" 2>/dev/null | grep "iSerial" | awk '{$1=$2=""; print $0}' | sed 's/^[[:space:]]*//')

        if [ -z "$serial" ]; then
            echo "WARNING: Cypress FX3 at USB $bus:$device reports no 'iSerial' line, so it appears not to be a RX888"
            serial="(none)"
        fi
        ### Find the/etc/radio/...*conf file which contains the line 'serial = $serial'
        conf_file_path=$(grep  "^serial.*$serial" /etc/radio/*.conf)
        if [[ -z "$conf_file_path" ]]; then
            printf "Rx?: 'no .conf file found with 'serial = $serial'"
        else
            conf_file_path=${conf_file_path%%.conf}
            local rx_number="${conf_file_path: -1}"
            printf "Rx${rx_number}: $bus:$device $serial is specfied in '$conf_file_path'\n"
        fi
    done
}

wd-ka9qctl() {
    # Get all services from config files in /etc/radio/
    local RADIOD_SERVICES=()
    local DECODE_SERVICES=()
    local RECORD_SERVICES=()
    local REPORTER_SERVICES=()
    local ALL_SERVICES=()

    # Find radiod services
    for conf in /etc/radio/radiod@*.conf; do
        if [ -f "$conf" ]; then
            local svc=$(basename "$conf" .conf).service
            RADIOD_SERVICES+=("$svc")
        fi
    done

    # Find ft4-decode and ft8-decode services
    for conf in /etc/radio/ft4-decode*.conf /etc/radio/ft8-decode*.conf; do
        if [ -f "$conf" ]; then
            local svc=$(basename "$conf" .conf).service
            DECODE_SERVICES+=("$svc")
        fi
    done

    # Find ft4-record and ft8-record services
    for conf in /etc/radio/ft4-record*.conf /etc/radio/ft8-record*.conf; do
        if [ -f "$conf" ]; then
            local svc=$(basename "$conf" .conf).service
            RECORD_SERVICES+=("$svc")
        fi
    done

    # Find pskreporter services (handle both formats: pskreporter@*.conf and *-pskreporter.conf)
    for conf in /etc/radio/pskreporter*.conf /etc/radio/*-pskreporter.conf; do
        if [ -f "$conf" ]; then
            local basename=$(basename "$conf" .conf)
            # Convert ft4-pskreporter to pskreporter@ft4 format
            if [[ "$basename" == *-pskreporter ]]; then
                local prefix=${basename%-pskreporter}
                local svc="pskreporter@${prefix}.service"
            else
                local svc="${basename}.service"
            fi
            REPORTER_SERVICES+=("$svc")
        fi
    done

    # Populate all services list
    for svc in "${RADIOD_SERVICES[@]}"; do
        ALL_SERVICES+=("$svc")
    done
    for svc in "${DECODE_SERVICES[@]}"; do
        ALL_SERVICES+=("$svc")
    done
    for svc in "${RECORD_SERVICES[@]}"; do
        ALL_SERVICES+=("$svc")
    done
    for svc in "${REPORTER_SERVICES[@]}"; do
        ALL_SERVICES+=("$svc")
    done

    cmd_start() {
        echo "Starting all KA9Q services..."
        for svc in "${ALL_SERVICES[@]}"; do
            echo "  Starting $svc..."
            sudo systemctl start "$svc" 2>/dev/null || true
        done
        echo "Done."
    }

    cmd_stop() {
        echo "Stopping all KA9Q services..."
        # Stop in reverse order
        for ((i=${#ALL_SERVICES[@]}-1; i>=0; i--)); do
            svc="${ALL_SERVICES[$i]}"
            echo "  Stopping $svc..."
            sudo systemctl stop "$svc" 2>/dev/null || true
        done
        echo "Done."
    }

    cmd_restart() {
        cmd_stop
        sleep 2
        cmd_start
    }

    cmd_status() {
        printf "%-55s %-18s %s\n" "COMMAND" "STATUS" "UPTIME"
        echo "--------------------------------------------------------------------------------------------"

        # Check each radiod service
        for svc in "${RADIOD_SERVICES[@]}"; do
            check_service "sudo systemctl status   $svc"
        done

        # Check decode services
        for svc in "${DECODE_SERVICES[@]}"; do
            check_service "sudo systemctl status   $svc"
        done

        # Check record services
        for svc in "${RECORD_SERVICES[@]}"; do
            check_service "sudo systemctl status   $svc"
        done

        # Check reporter services
        for svc in "${REPORTER_SERVICES[@]}"; do
            check_service "sudo systemctl status   $svc"
        done
    }

    check_service() {
        local cmd="$1"
        local service=$(echo "$cmd" | awk '{print $NF}')

        if sudo systemctl is-active --quiet "$service" 2>/dev/null; then
            local uptime=$(sudo systemctl show "$service" -p ActiveEnterTimestamp --value)
            if [ -n "$uptime" ]; then
                local seconds=$(($(date +%s) - $(date -d "$uptime" +%s)))
                local hours=$((seconds / 3600))
                local minutes=$(((seconds % 3600) / 60))
                local secs=$((seconds % 60))
                printf "%-55s [running]          %dh %dm %ds\n" "$cmd" $hours $minutes $secs
            else
                printf "%-55s [running]          \n" "$cmd"
            fi
        elif sudo systemctl is-enabled --quiet "$service" 2>/dev/null; then
            printf "%-55s [stopped]\n" "$cmd"
        else
            local status=$(sudo systemctl status "$service" 2>&1)
            if echo "$status" | grep -q "could not be found"; then
                printf "%-55s [ERROR: none found]\n" "$cmd"
            else
                local rc=$(sudo systemctl show "$service" -p ExecMainStatus --value 2>/dev/null)
                if [ -n "$rc" ] && [ "$rc" != "0" ]; then
                    printf "%-55s [ERROR: rc=%s]\n" "$cmd" "$rc"
                else
                    printf "%-55s [stopped]\n" "$cmd"
                fi
            fi
        fi
    }

    # Main command dispatcher
    case "${1:-status}" in
        start)
            cmd_start
            ;;
        stop)
            cmd_stop
            ;;
        restart)
            cmd_restart
            ;;
        status)
            cmd_status
            ;;
        *)
            echo "Usage: wd-ka9qctl {start|stop|restart|status}"
            echo ""
            echo "Controls KA9Q radio daemon and associated services (from /etc/radio/*.conf):"
            echo "  - radiod@*.service"
            echo "  - ft4-decode*.service, ft8-decode*.service"
            echo "  - ft4-record*.service, ft8-record*.service"
            echo "  - pskreporter*.service"
            return 1
            ;;
    esac
}

#############################################
function wd-rac-status()
{
    (cd ${HOME}/wsprdaemon/bin; ./frpc status)
}

function wd-rac-purge() 
{
    read -p "Are you *SURE* you want to purge any RAC connection?  If you are remotely logged on, you may not be able to re-access this server [yN] =>"
    REPLY=${REPLY:=N} ; REPLY=${REPLY:0:1} ; REPLY=${REPLY^}
    if [[ ${REPLY} != "Y" ]]; then
        echo "Aborting this command"
        return 0
    fi
    sudo systemctl stop wd-remote-access.service
    sudo systemctl disable wd_remote_access.service
    sudo systemctl disable wd-remote-access.service
    sudo rm -f /etc/systemd/system/wd-remote-access.service
    sudo rm -f /etc/systemd/system/wd_remote_access.service
    sudo systemctl daemon-reload
    ### Setup RAC again if it is defined in wsprdaemon.org
    ${HOME}/wsprdaemon/wsprdaemon.sh -V
}

function wd-ws8 ()
{
    local ws8_usb_device="/dev/ttyACM0"
    local attached_pids_list=($(sudo lsof ${ws8_usb_device} 2>&1 | awk '/^screen/{print $2}'))
    if (( ${#attached_pids_list[@]} )); then
        read -p "Found ${#attached_pids_list[@]} screen sessions attached to the WS8.  Kill those sessions and connect? => "
        sudo kill ${attached_pids_list[@]}
    fi
    sudo screen ${ws8_usb_device}
}

function wd-usbreset() {
    if [[ -z "$1" ]]; then
        echo "Usage: usbreset <bus-port | vendor:product>"
        return 1
    fi

    # If user passed a device ID (like 04b4:00f1), find its bus-port
    if [[ "$1" =~ : ]]; then
        devpath=$(lsusb | grep "$1" | awk '{print $2"-"$4}' | sed 's/://')
        if [[ -z "$devpath" ]]; then
            echo "Device ID $1 not found"
            return 1
        fi
    else
        devpath="$1"
    fi

    echo "Resetting USB device at $devpath..."
    echo -n "$devpath" | sudo tee /sys/bus/usb/drivers/usb/unbind
    sleep 1
    echo -n "$devpath" | sudo tee /sys/bus/usb/drivers/usb/bind
}

function wd-web-restart()
{ 
    local web_daemon_pid="$(< ~/wsprdaemon/ka9q_web_daemon.pid)"
    if [[ -z "$web_daemon_pid" ]]; then
        echo "Ka9q-web is not running"
        return 0
    fi
    local web_pid=$(ps --ppid "$web_daemon_pid" -o pid=)
    kill $web_pid
}


function wd-most-recent-log-error()
{
    local grep_arg="${1-ERROR:}"
    local dir_list=( $(find . -type d ) )
    local dir
    for dir in ${dir_list[@]}; do
        local log_file_list=( $( find ${dir} -maxdepth 1 -type f -name '*.log' ) )
        local log_file
        for log_file in ${log_file_list[@]}; do
            local grep_last_line="$( grep "${grep_arg}" $(realpath ${log_file}) | grep -v start | tail -n 1)"
            if [[ -n "${grep_last_line}" ]]; then
                echo "${grep_last_line}"
            fi
        done
    done
    echo "$(date -u): search complete"
}

### If source and destination directory trees are on the same file system, wd-mvsync uses the much faster'mv' rather than 'rsync' to move the files
function wd-mvsync()
{
    local src_dir=$1
    local dst_dir=$2
    local rc

    local file
    find "${src_dir}" -type f | while IFS= read -r file; do
        local rel_path="${file#${src_dir}/}"
        local dest_path="${dst_dir}/${rel_path}"

        mkdir -p "$(dirname "$dest_path")"
        mv "${file}" "${dest_path}"
        rc=$? ; if (( rc )); then
            echo "ERROR: 'mv ${file} ${dest_path}' => ${rc}"
            return 1
        fi
    done
    return 0
 }

function wd-rsync() 
{
    local src_dir=${1-}
    if [[ -z "${src_dir}" ]]; then
        echo "Missing <SRC_DIR> argument.  usage: wd-rsync <SRC_DIR> <DST_DIR>"
        return
    fi
    if ! [[ -d "${src_dir}" ]]; then
        echo "<SRC_DIR> doesn't exist.  usage: wd-rsync <SRC_DIR> <DST_DIR>"
        return
    fi
    local dst_dir=${2-}
    if [[ -z "${dst_dir}" ]]; then
        echo "Missing <DST_DIR> argument.  usage: wd-rsync <SRC_DIR> <DST_DIR>"
        return
    fi
    if ! [[ -d "${dst_dir}" ]]; then
        echo "<DST_DIR> doesn't exist.  usage: wd-rsync <SRC_DIR> <DST_DIR>"
        return
    fi
    if (( $(stat -c %d "${src_dir}") == $(stat -c %d "${dst_dir}") )); then
        echo "${src_dir} and ${dst_dir} are in the file system, so we can 'mv' the files rather than 'rsync' them"
        wd-mvsync ${src_dir} ${dst_dir}
        return
    fi

    echo "${src_dir} and ${dst_dir} are on different file systems, so we must run 'rsync' to transfer them"
    echo "Executing: rsync -ah --remove-source-files --info=progress2 --stats ${src_dir} ${dst_dir}"
    rsync -ah --remove-source-files --info=progress2 --stats "${src_dir}" "${dst_dir}"
    local rc=$? ; if (( rc )); then
        echo "ERROR: ' rsync -ah --remove-source-files --info=progress2 --stats ${src_dir} ${dst_dir}' => ${rc}"
        return 1
    fi
    
    local src_files_list=( $(find ${src_dir} -type f) )
    if (( ${#src_files_list[@]} ));then
        echo "ERROR: found ${#src_files_list[@]} files in ${src_dir} after rsync"
        return 2
    fi
    rm -r ${src_dir}
    echo "All files in ${src_dir} have been moved to the corresponding subdir in ${dst_dir} and ${src_dir} has been deleted"
    return 0
}

function wd-pcmrecord-restarts()
{
    sudo journalctl --since "6 hours ago" -u radiod@rx888-wsprdaemon | grep "No rx888 data for 5 seconds, quitting"
}

function wd-pcmrecord-status()
{
    ps -eo pid,lstart,etime,cmd | grep pcmrecord | grep -v grep
}

function cdt()
{
    local wd_tmp_file_system_path
    if [[ -d /run/wsprdaemon/recording.d ]]; then
        wd_tmp_file_system_path="/run/wsprdaemon"
    elif [[ -d /dev/shm/wsprdaemon/recording.d ]]; then
        wd_tmp_file_system_path="/dev/shm/wsprdaemon"
    else
        echo "ERROR: dan't find WD's tmp file system"
        return
    fi
    local recording_dir_path="${wd_tmp_file_system_path}/recording.d"
    if ! [[ -d ${recording_dir_path} ]]; then
        echo "Can't find expected directory '${recording_dir_path}'"
        return
    fi
    local receiver_dir_path_list=( $(find ${recording_dir_path} -maxdepth 1 -type d -name 'KA9Q*') )
    case ${#receiver_dir_path_list[@]} in
        0)
            echo "Can't find any 'KA9Q*' directories in '${recording_dir_path}'"
            return 0
            ;;
        1)
            echo "Found one 'KA9Q*' directories in '${recording_dir_path}', so changing to it"
            cd ${receiver_dir_path_list[0]}
            ;;
        *)
            printf "Found ${#receiver_dir_path_list[@]} 'KA9Q*' directories in '${recording_dir_path}'. Choose one:\n"
            local index
            for (( index=0; index < ${#receiver_dir_path_list[@]};  ++index )); do
                printf "%2d: ${receiver_dir_path_list[index]}\n" ${index}
            done
            read -p "Select [0-$(( ${#receiver_dir_path_list[@]} - 1))].  Default is 0 => "
            cd ${receiver_dir_path_list[${REPLY-0}]}
            ;;
    esac
}

function wdsr()
{
    local wd_is_running
    local wd_last_start_epoch

    local wd_watchdog_pid_file_path=~/wsprdaemon/watchdog_daemon.pid
    if ! [[ -f ${wd_watchdog_pid_file_path} ]]; then
        echo "Can't find '${wd_watchdog_pid_file_path}', so WD can't be running"
        return 0
    fi
    local wd_pid=$(< ${wd_watchdog_pid_file_path})
    local rc
    local wd_run_time=$(ps -p ${wd_pid} -o etime=)
    rc=$? ; if (( rc )) || [[ -z "${wd_run_time}" ]]; then
        echo "WD's watchdog_daemon() pid ${wd_pid} is not running"
    else
        echo "WD's watchdog_daemon() pid ${wd_pid} has been running for D-HH:MM:SS: ${wd_run_time}"
    fi
    local pid_file_path_list=( $(find /dev/shm/wsprdaemon/ -path /dev/shm/wsprdaemon/wav-archive -prune -o -type f -name '*.pid') )
    echo "Checking ${#pid_file_path_list[@]} pid files.."

    local running_pid_count=0
    local dead_pid_count=0
    local pid_file
    for pid_file in ${pid_file_path_list[@]} ; do
        set +x
        local pid=$(<${pid_file})
        local pid_run_time=$(ps -p ${pid} -o etime= 2> /dev/null)
        rc=$? ; set +x;if (( rc )) || [[ -z "${pid_run_time}" ]];  then
            echo "PID ${pid} stored in ${pid_file} is not running"
            (( ++dead_pid_count ))
        else
            (( ${verbosity-0} )) && echo "Deamon ${pid_file} is running"
            (( ++running_pid_count ))
        fi
    done
    echo "Found ${running_pid_count} running PID files and ${dead_pid_count} dead pids"
    (  source ~/wsprdaemon/running.jobs
    echo "There are ${#RUNNING_JOBS[@]} running receiver jobs"
    )
}

#shopt -s -o nounset
function get-github-projects-list()
{
    # Extract the array definition into a temp file
    local tmpfile=$1
    awk '
    BEGIN { in_array = 0 }
    /^\s*declare\s+GITHUB_PROJECTS_LIST=\(/ { in_array = 1 }
    in_array {
    print
    if (/\)\s*$/) {
        in_array = 0
        exit
    }
}
' ~/wsprdaemon/ka9q-utils.sh > "$tmpfile"
}

function wd-commit-info() 
{
    local tmpfile=$(mktemp)
    get-github-projects-list ${tmpfile}
    # Now source the extracted array definition
    # KA9Q_RADIO_COMMIT_CHECK="main"
    unset KA9Q_RADIO_COMMIT_CHECK KA9Q_FT8_COMMIT_CHECK PSK_UPLOADER_COMMIT_CHECK ONION_COMMIT_CHECK KA9Q_WEB_COMMIT_CHECK
    KA9Q_RADIO_LIBS_NEEDED="NONE"
    ONION_LIBS_NEEDED="NONE"
    source "$tmpfile"
    ka9q_list=( "${GITHUB_PROJECTS_LIST[@]}")
    source ~/wsprdaemon/wsprdaemon.conf
    source "$tmpfile"
    as_configured_list=( "${GITHUB_PROJECTS_LIST[@]}")
    #cat "$tmpfile"
    rm "$tmpfile"

    local format="%20s %20s %42s %20s %42s %42s\n"
    printf "${format}" "Service" "ka9q-utils.sh check" "ka9q-utils.sh commit" "as configured check" "as configured commit" "running commit"

    local index
    for (( index=0; index < ${#ka9q_list[@]}; ++index)); do
        local ka9q_service_info_list=( ${ka9q_list[index]} )
        local commit_dir=${ka9q_service_info_list[0]}

        local ka9q_commit_check=${ka9q_service_info_list[1]}
        local ka9q_commit_val=${ka9q_service_info_list[6]}

        local as_configured_info_list=( ${as_configured_list[index]} )
        local as_configured_commit_check=${as_configured_info_list[1]}
        local as_configured_commit_val=${as_configured_info_list[6]}

        local running_commit=$(cd ~/wsprdaemon/${commit_dir}; git rev-parse HEAD)
        set +x
        printf "${format}" ${commit_dir} ${ka9q_commit_check} ${ka9q_commit_val} ${as_configured_commit_check} ${as_configured_commit_val} ${running_commit}
    done
}

function wd-cpu-watch()
{
    watch -n 1 'bash -c '\''for cpu in $(ls -d /sys/devices/system/cpu/cpu[0-9]* | sort -V); do cur=$(( $(< $cpu/cpufreq/scaling_cur_freq)/1000 )); min=$(( $(< $cpu/cpufreq/scaling_min_freq)/1000 )); max=$(( $(< $cpu/cpufreq/scaling_max_freq)/1000 )); scaled=$(( cur * 80 / max )); stars=$(printf "%*s" "$scaled" "" | tr " " "*"); printf "%-6s %-80s cur=%5d MHz  min=%5d MHz  max=%5d MHz\n" "$(basename $cpu)" "$stars" "$cur" "$min" "$max"; done'\'''
}

function wd-cpu-info() 
{
    local sys_list=( $(ls -dv /sys/devices/system/cpu/cpu[0-9]*) )   ### Get numericallyt sorted list 
    local cpu_list=( ${sys_list[@]##*/} )                            ### Chop off the poth to just get a list of core names
    local sys_cpu_base_path="/sys/devices/system/cpu"

    local cpu
    for cpu in ${cpu_list[@]} ; do
        local cpu_max_freq=$(cat "${sys_cpu_base_path}/${cpu}/cpufreq/scaling_max_freq" 2>/dev/null)
        [ -n "${cpu_max_freq}" ] && printf "%5s: %s\n" "${cpu}" "${cpu_max_freq}"
    done
    printf "Available CPU frequencies: $(< /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies)\n"
}

function wd-cpu-set-core-speeds() {
    local speed01="$1"   # MHz for cores 0 and 1
    local speedrest="$2" # MHz for all other cores

    if [[ -z "$speed01" || -z "$speedrest" ]]; then
        echo "Usage: set_core_speeds <speed-for-core0-1-MHz> <speed-for-rest-MHz>"
        return 1
    fi

    # convert MHz → kHz
    local s01=$((speed01 * 1000))
    local srest=$((speedrest * 1000))

    echo "🔧 Setting CPU0-1 to ${speed01} MHz, others to ${speedrest} MHz"

    for cpu in /sys/devices/system/cpu/cpu[0-9]*; do
        n=${cpu##*cpu}  # extract number after "cpu"
        if (( n == 0 || n == 1 )); then
            echo "$s01" | sudo tee "$cpu/cpufreq/scaling_max_freq" > /dev/null
        else
            echo "$srest" | sudo tee "$cpu/cpufreq/scaling_max_freq" > /dev/null
        fi
    done

    # Disable turbo boost (CPB)
    if [[ -w /sys/devices/system/cpu/cpufreq/boost ]]; then
        echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost > /dev/null
        echo "🚫 Turbo Boost disabled"
    else
        echo "⚠️ Could not disable Turbo Boost: /sys/devices/system/cpu/cpufreq/boost not found"
    fi

    # Print summary table
    echo
    printf "%-5s %-10s\n" "CPU" "MaxFreq(MHz)"
    echo "-------------------"
    for cpu in /sys/devices/system/cpu/cpu[0-9]*; do
        n=${cpu##*cpu}
        freq=$(<"$cpu/cpufreq/scaling_max_freq")
        printf "%-5s %-10s\n" "$n" "$((freq / 1000))"
    done
}

function wd-fft-utils() {
    local rc
    local tmp_file="/tmp/wd-fft-utils.txt"

    local service_name=$(systemctl list-units --no-legend 'radiod@*.service' | awk '{for(i=1;i<=NF;i++) if($i ~ /^radiod@/) print $i}')
    sudo journalctl --no-pager -n 2000 -u ${service_name} > ${tmp_file}
    rc=$? ; if (( rc )); then
        echo "Can't find log lines for radiod.  'journalctl --nopager -n 2000 -u radiod@* > ${tmp_file}' returned error ${rc}"
        return 1
    fi
    if [[ ! -s  ${tmp_file} ]]; then
        echo " 'journalctl --nopager -n 2000 -u radiod@* > ${tmp_file}' returned no error, but ${tmp_file} is empty"
        return 2
    fi
    local tmp_file_size=$(wc -l <  ${tmp_file})
    (( ${verbosity-0} )) && echo "${tmp_file} has ${tmp_file_size} lines"
    awk '/Start/ {last=NR; next} {lines[NR]=$0} END {for (i=last+1; i<=NR; i++) print lines[i]}' ${tmp_file} > ${tmp_file}.end
    rc=$? ; if (( rc )); then
        echo "awk '/Start/ {last=NR; next} {lines[NR]=$0} END {for (i=last+1; i<=NR; i++) print lines[i]}' ${tmp_file} > ${tmp_file}.end => ${rc}"
        return 3
    fi
    if [[ ! -s ${tmp_file}.end ]]; then
        echo "Couldn't find a 'Start' line in ${tmp_file}"
        return 4
    fi
    local nominal_sample_rate=$(awk -F'nominal sample rate ' '/nominal sample rate/{print $2}' ${tmp_file}.end | awk '{print $1}')
    local wisdom_fft_rates_list=($( awk -F'-o /tmp/wisdomf' '/-o \/tmp\/wisdomf /{print $2}' ${tmp_file}.end  | awk '{print $1}' | sed 's/",//' | sort -u | sort -rn -k1.3 ) )
    if (( ${#wisdom_fft_rates_list[@]} == 0 )); then
        echo "FFT planning is optimal, so no update is needed"
        return 0
    fi
    echo "Found $(wc -l < ${tmp_file}.end) lines and ${#wisdom_fft_rates_list[@]} unplanned rates reported after the most recent 'Start' line
nominal_sample_rate=${nominal_sample_rate}
wisdom_fft_rates_list=${wisdom_fft_rates_list[*]}"
    read -p "Do you want to stop this WD server and run the FFT tool so the FFTs for those rates are optimized? [Y,n] => "
    if [[ -n ${REPLY-} && ${REPLY^^} != "Y" ]]; then
        echo "No changes have been made"
        return 0
    fi
    wd-killall
    echo "$(date): Running the FFT planning command.  This may take minutes or even hours to complete. So leave this server undisturbed during this operation to maximize its optimizations:"
    set -x
    time fftwf-wisdom -v -T 1 -w /var/lib/ka9q-radio/wisdom -o /tmp/wisdomf ${wisdom_fft_rates_list[@]}
    set +x
    local current_wisdom_file_size=$(stat -c %s /etc/fftw/wisdomf)
    local new_wisdom_file_size=$(stat -c %s /tmp/wisdomf)

    echo "FFT optimization is complete
The newly created wisdom file     /tmp/wisdom      has ${new_wisdom_file_size} bytes
The currently installed versioon /etc/fftw/wisdom  has ${current_wisdom_file_size} bytes"
    if (( new_wisdom_file_size > current_wisdom_file_size )); then
        read -p "Since the new wisdomf is larger than the current wisdomf, it is recommended that you install the new one.  Proceed? [Y,n] => "
        if  [[ -z ${REPLY-} || ${REPLY^^} == "Y" ]]; then
            sudo cp -p /etc/fftw/wisdomf /etc/fftw/wisdomf.save
            sudo cp -p /tmp/wisdomf /etc/fftw/wisdomf
            echo "/etc/fftw/wisdom has been changed"
        else
            echo "/etc/fftw/wisdom is unchanged"
        fi
    else
        echo "ERROR: it is unexpected that the new wisdomf not larger than the current wisdomf, so do nothing"
    fi
    return 0
}
function wd-killweb() {
    local web_pid=$(ps aux | grep web | grep -v grep | awk '{print $2}')

    if [[ -z "${web_pid}" ]]; then
        echo "Nor Ka9qQ-web process is running"
        return 0
    fi
    sudo kill ${web_pid}
}

function wd-pcmrecord-version() {
    ~/wsprdaemon/ka9q-radio/pcmrecord --version
}

declare -A KA9Q_wspr_band_freq_list=(
    [160]="1836600"
    [80]="3568600"
    [60]="5287200"
    [40]="7038600"
    [30]="10138700"
    [20]="14095600"
    [17]="18104600"
    [15]="21094600"
    [12]="24924600"
    [10]="28124600"
     [6]="50293000"
)

declare -A KA9Q_time_band_freq_list=(
    [2.5]="2500000"
      [5]="5000000"
     [10]="10000000"
     [15]="15000000"
     [20]="20000000"
     [25]="25000000"
   [3.33]="330000"
   [7.85]="785000"
  [14.67]="14670000"
)

### USAGE: wd-pcmrecord-file-summaries [band]
### USAGE: wdpfs [band]
### USAGE:   Runs the python program which validates the pcmrecord.log files  Defaults to 20m for WSPR and 10 MHz for WWV
alias wdps='wd-pcmrecord-summaries'
function wd-pcmrecord-summaries()
{
    local pcmrecord_log_dir
    wd-get-ka9q-receiver-dir "pcmrecord_log_dir"
    rc=$? && (( rc )) && echo "Can't find any KA9Q_* directories" && return $rc

    local create_time_file_list=($( find $pcmrecord_log_dir -type f -name wav_file_create_time.log | grep WWV_[1-9] ))
    if (( ${#create_time_file_list[@]} == 0 )); then
        echo "Can't find any 'wav_file_create_time.log' files under $pcmrecord_log_dir"
        return 1
    fi

    python3 ~/wsprdaemon/wd-validate-wav-logs.py ${create_time_file_list[@]}
}

### USAGE: wd-pcmrecord-filetime [band]
### USAGE: wdpf [band]
### USAGE:     Lists the create and last write times of the wav files of the band.  Defaults to 20m for WSPR and 10 MHz for WWV
alias wdpf='wd-pcmrecord-filetimes'
function wd-pcmrecord-filetimes()
{
    local pcmrecord_log_dir
    wd-get-ka9q-receiver-dir "pcmrecord_log_dir"
    rc=$? && (( rc )) && echo "Can't find any KA9Q_* directories" && return $rc

    local band
    local tuning_freq
    if [[ $pcmrecord_log_dir =~ WWV|CHU ]]; then
        band=${1-10}     ### Defaults to WWV_10
        tuning_freq="${KA9Q_time_band_freq_list[${band}]}_iq"
    else
        band=${1-20}      ### Defaults to 20m
        tuning_freq="${KA9Q_wspr_band_freq_list[${band}]}_usb"
    fi
 
    local wav_files_list=( $(find $pcmrecord_log_dir -maxdepth 1 -type f -name "*Z_${tuning_freq}*.wav") )
    (( ${#wav_files_list[@]} == 0 )) && echo "Can'find any '*Z_${tuning_freq}*.wav' files in $pcmrecord_log_dir" && return 1
    (( ${verbosity-0} )) && echo "Found  ${#wav_files_list[@]} '*Z_${tuning_freq}*.wav' files in $pcmrecord_log_dir"

    local pcmrecord_errors_file="$pcmrecord_log_dir/pcmrecord-errors.log"
    if [[ ! -f $pcmrecord_errors_file ]]; then
        echo "WARNING: can't file expected file $pcmrecord_errors_file"
    else
        local pcmrecord_parent_pid_file_list=( $(find $pcmrecord_log_dir -maxdepth 1 -name "*pid") )
        case ${#pcmrecord_parent_pid_file_list[@]} in
            0)
                echo "Can't find the expect .pid file in $pcmrecord_log_dir"
                ;;
            1) 
                local pid_run_time
                pid_run_time=$( ps -o etime --ppid $(< ${pcmrecord_parent_pid_file_list[0]}) --no-headers )
                rc=$? ; if (( rc )) ; then
                    echo "Pid $(< ${pcmrecord_parent_pid_file_list[0]}) in ${pcmrecord_parent_pid_file_list[0]} is not active"
                else
                    echo "$(date): pcmrecord has been running for $pid_run_time"
                fi
                ;;
            *)
                echo "ERROR: found ${#pcmrecord_parent_pid_file_list[@]} pid files:\n ${pcmrecord_parent_pid_file_list[*]}"
                ;;
        esac
    fi

    local wav_file
    for wav_file in ${wav_files_list[@]}; do
        local file_stat_list=($( stat $wav_file  | awk '/File/{printf "%s:", $2} /Size/{printf " Size:%s", $2} /Birth/{printf " Birth:%s", $3} /Change/{printf " Change:%s", $3}  END {printf "\n"}' ))
        echo "${file_stat_list[0]} ${file_stat_list[1]} ${file_stat_list[3]} ${file_stat_list[2]}"
    done
}

function wd-pcmrecord-errors() 
{
    local rc

    local pcmrecord_log_dir
    wd-get-ka9q-receiver-dir "pcmrecord_log_dir"
    rc=$? && (( rc )) && return $rc

    local pcmrecord_error_log_file="${pcmrecord_log_dir}/pcmrecord-errors.log"
    if [[ ! -f ${pcmrecord_error_log_file} ]]; then
        echo "Can't find ${pcmrecord_error_log_file}"
        return 1
    fi
    read -p "Run tail -f ${pcmrecord_log_dir}? [Yn] => "
    tail -f ${pcmrecord_error_log_file}
}

function wd-get-ka9q-receiver-dir() 
{
    local __return_var_name=$1

    local wd_recording_root_dir="/dev/shm/wsprdaemon/recording.d"
    local receiver_dir_list=( $( find ${wd_recording_root_dir} -maxdepth 1 -type d -name 'KA9Q*' | sort -t _ -k 3,3 -k 2,2n) )     ### sort so list is 'KA9Q_0 KA9Q_1 KA9Q_0_WWV KA9Q_1_WWV

    local ka9q_receiver_dir
    case ${#receiver_dir_list[@]} in
        0)
            echo "Can't find any KA9Q* directories in ${wd_recording_root_dir}"
            return 1
            ;;
        1)
            ka9q_receiver_dir=${receiver_dir_list[0]}
            ;;
        *)
            local index
            for (( index=0; index < ${#receiver_dir_list[@]}; ++index )) do
                echo "${index}:  ${receiver_dir_list[index]}"
            done
            read -p "There are multiple KA9Q receivers.  Which of their pcmrecord-errors.log files do you want to view? [0-$((  ${#receiver_dir_list[@]} - 1 ))] (default = 0) => "
            [[ -z "$REPLY" ]] && index=0 || index=$REPLY
            ka9q_receiver_dir=${receiver_dir_list[index]}
            ;;
    esac
    eval $__return_var_name=\${ka9q_receiver_dir}
    return 0
}

function wd-show-file-write-time() {
    local files_list=( $@ )

    if (( ${#files_list[@]} == 0 )); then
        echo "usage: $0 <FILE0> [<FILE2> ...]"
        return
    fi
    local file
    for file in ${files_list[@]}; do
        if lsof $file >& /dev/null; then
            continue
        fi
        local date_time_list=( $(stat $file | awk '/Modify/{printf "%s %s", $2, $3} ') )
        local date=${date_time_list[0]}
        local hhmmss=${date_time_list[1]%.*}
        local microseconds=${date_time_list[1]##*.}
        local seconds=$(( 10#${hhmmss##*:} ))                  ### converts second 00-59 to a decimal integer
        local max_delay_secs=${RECORDING_MAX_DELAY_SECONDS-2}  ### If 'stat' reports last write is greater than 2 seconds
        local min_early_secs=${RECORDING_MIN_EARLY_SECONDS-59} ### and it is earlier than second 59, then log error
        if (( (seconds > max_delay_secs) && (seconds < min_early_secs) )); then
            echo "ERROR: File ${file##*/} last write time ${date_time_list[*]} is not in second 00"
        else
            echo "File ${file##*/} last write time ${date_time_list[*]} shows it has been closed in second $seconds"
        fi
    done
    return 0
}

function get_pid_start_epoch() {
    local pid="$1"

    local CLK_TCK=$(getconf CLK_TCK)
    local boot_epoch=$(awk '/btime/ {print $2}' /proc/stat)
    local start_ticks=$(awk '{print $22}' /proc/"$pid"/stat)
    local start_epoch=$((boot_epoch + (start_ticks / CLK_TCK)))

    echo $start_epoch
}

### Looks at the most recent 400 wspr cycles and looks for missing cycles
### It expects at least 1 spot in every 2 minute cycle and accomodates 5 minute cycles with spots
alias wdlc='wd-wsprlog-check'      ## == "WD wspr log check
function wd-wsprlog-check() {
    local wspr_cycles_to_search=${1-400}

    local verbosity=${verbosity-0}
    local wd_watchdog_pid_file=~/wsprdaemon/watchdog_daemon.pid
    local wd_watchdog_start_epoch=0

    if [[ ! -f $wd_watchdog_pid_file ]]; then
        printf "WD isn't running, since there is no  wd_watchdog_pid_file '$wd_watchdog_pid_file'"
    else
        local  wd_watchdog_pid=$(<$wd_watchdog_pid_file)
        if ! ps $wd_watchdog_pid > /dev/null; then
            printf "The WD watchdog PID $wd_watchdog_pid recorded in '$wd_watchdog_pid_file' is not active"
        else
            wd_watchdog_start_epoch=$(get_pid_start_epoch $wd_watchdog_pid)
            #printf "The currently running WD started at epoch %s which is %s\n"  "$wd_watchdog_start_epoch" "$(TZ=UTC date -d @$wd_watchdog_start_epoch "+%Y-%m-%d %H:%M:%S")"
        fi
    fi

    local wspr_log_files="/var/log/wspr.log"
    if [[ ! -f $wspr_log_files ]]; then
        echo "ERROR: the expected file  $wspr_log_files doesn't exist"
        return 1
    fi
    if [[ -f /var/log/wspr.log.1 ]]; then
        wspr_log_files="/var/log/wspr.log.1 $wspr_log_files"
    fi
    awk '{printf "%s %s\n", $1, $2}' $wspr_log_files | sort | uniq -c | tail -n $wspr_cycles_to_search > /tmp/wspr-spots-counts.log

    local last_line
    local last_line_epoch=0
    local contiguous_cycles_count=0
    local spot_count_line
    local spot_count_line_list

    while read spot_count_line; do
        spot_count_line_list=( $spot_count_line )

        if (( ${#spot_count_line_list[@]} != 3 )) ; then
            echo "ERROR: bad line $spot_count_line"
            return
        fi

        local this_line_epoch=$( TZ=UTC date -d "20${spot_count_line_list[1]:0:2}-${spot_count_line_list[1]:2:2}-${spot_count_line_list[1]:4:2} ${spot_count_line_list[2]:0:2}:${spot_count_line_list[2]:2:2}" +%s )

        if [[ -z "$this_line_epoch" ]]; then
            echo "ERROR: bad line $spot_count_line"
            return
        fi
        if (( wd_watchdog_start_epoch && ( this_line_epoch > wd_watchdog_start_epoch ) )); then
            if false && (( contiguous_cycles_count )); then
                printf "(%d) contiguous cycles\n"  $contiguous_cycles_count
                contiguous_cycles_count=0
            fi
            printf "%s: is the currently running WD start time\n" "$(TZ=UTC date -d @$wd_watchdog_start_epoch "+%y%m%d %H%M")"
            wd_watchdog_start_epoch=0
        fi
 
        if (( last_line_epoch == 0 )) ; then
            printf "%s %s: with %3d spots is the first cycle of the log\n" ${spot_count_line_list[1]} ${spot_count_line_list[2]} ${spot_count_line_list[0]}
            last_line="$spot_count_line"
            last_line_epoch=$this_line_epoch
            continue
        fi

        local last_line_list=( $last_line )
        (( verbosity > 1 ))  && printf "\n\nThis epoch is %d at %s %s, last epoch is %d at %s %s\n" $this_line_epoch ${spot_count_line_list[1]} ${spot_count_line_list[2]} $last_line_epoch "${last_line_list[1]}"  "${last_line_list[2]}"
        local expected_line_epoch
        local last_line_minute_digit="${last_line: -1}"
        if [[ "${last_line: -1}" == "5" ]]; then
            ### After a 5/15/25/35/45/55 minute spot, the next onw should be one minute later
            (( verbosity > 1 )) && echo "Last line was a 5/15/25/..., so this line should be 60 seconds later"
            expected_line_epoch=$(( last_line_epoch + 60 ))
        else
            if (( this_line_epoch == (last_line_epoch + 60) )); then
                (( verbosity > 1 )) && echo "This line is 60 sesconds after the last"
                expected_line_epoch=$(( last_line_epoch + 60 ))   ### This a 5/15/... minute packet
            else
                (( verbosity > 1 )) && echo "this line should be 120 seconds after the last line"
                expected_line_epoch=$(( last_line_epoch + 120 ))
            fi
        fi

        if (( this_line_epoch ==  expected_line_epoch )) ; then
            (( verbosity > 1 )) && printf "%s %s: %3d spots as expected\n" ${spot_count_line_list[1]} ${spot_count_line_list[2]} ${spot_count_line_list[0]}
            (( ++contiguous_cycles_count ))
        else
            printf "%s %s: has %3d spots. After %d contiguous spot reports found gap of %2d WSPR cycles. The last cycle reported '%s'\n" \
                    ${spot_count_line_list[1]} ${spot_count_line_list[2]} ${spot_count_line_list[0]} $contiguous_cycles_count $(( ( (this_line_epoch - last_line_epoch) / 120) - 1 )) "$last_line"
            contiguous_cycles_count=0
        fi
        last_line="$spot_count_line"
        last_line_epoch=$this_line_epoch
    done <  <(cat /tmp/wspr-spots-counts.log)

    spot_count_line_list=( $(tail -n 1 /tmp/wspr-spots-counts.log) )
    printf "%s %s: is the most recent cycle which has %3d spots. It ends a run of %d contiguous wspr cycles\n" \
               ${spot_count_line_list[1]} ${spot_count_line_list[2]} ${spot_count_line_list[0]} $contiguous_cycles_count
}

### ka9q-web-control was written for ON5KQ, but it may be useful elsewhere
function ka9q-web-control-usage() {
    echo "$0:
    -h   => print this help message
    -a   => kill all running ka9q-web servers and start them again
    -z   => kill all running ka9q-web servers
    -s   => show status of running ka9q-web servers"
}

function ka9q-web-control() {
    if [[ -z "$@" || "$@" == "-h" ]] ; then
        ka9q-web-control-usage
        return 0
    fi

    if [[ "${1-}" == "-s" ]]; then
        ps aux | grep ka9q-web | grep -v 'grep\|ka9q-web-control'
        return 0
    fi

    sudo killall ka9q-web

    if [[ "${1-}" == "-k" ]]; then
        echo "Killed all ka9q-web services"
        return 0
    fi

    /usr/local/sbin/ka9q-web -m hf.local  -p 8081 -n ON5KQ-JO10os_2x60m-terminated_dipole_1m-high >& ~/wsprdaemon.ka9q-web-8081.log &
    /usr/local/sbin/ka9q-web -m hf1.local -p 8082 -n ON5KQ-JO10os_2x60m-terminated_dipole_1m-high >& ~/wsprdaemon.ka9q-web-8082.log &

    echo "Spawned two ka9q-web services, one on port 8081 and port 8082"
    return 0
}

### So we don't run through all the 'wd -Z' startupi, installation and validation code which might crash
declare WD_CONF_PATH=~/wsprdaemon/wsprdaemon.conf
function wd-killall() {
    local rc
    local rc1

    local verbosity=${verbosity-0}

    ### Stop all services: the FT decoders and their recorders and uploaders first (below), so nothing is left decoding
    ### while WD and radiod stop
    local services_list=(  wsprdaemon )

    source ${WD_CONF_PATH}
    #
    ### These services have been depricated, so stop and disable them
    local legacy_ft_services_list=( ft4-decoded ft8-decoded )
    local legacy_service
    for legacy_service in ${legacy_ft_services_list[@]} ; do
        local full_legacy_service="${legacy_service}.service"
        if ! sudo systemctl is-active --quiet ${full_legacy_service} ; then
            (( verbosity )) && echo "The deprecaded legacy service ${full_legacy_service} is not active, so there is nothing to do"
        else
            echo "Stopping the active depecaded legacy service ${full_legacy_service}"
            sudo systemctl stop ${full_legacy_service}
        fi
        if systemctl list-unit-files | awk -v svc="${full_legacy_service}" '$1 == svc && $2 == "enabled"' | grep -q .; then
            echo "Disabling the legacy service ${full_legacy_service}"
            sudo systemctl disable ${full_legacy_service}
            sudo systemctl daemon-reload
        fi
    done
 
    if [[  ${KA9Q_RUNS_ONLY_REMOTELY-no} == "yes" ]]; then
        echo "Stop only WD.  Don't stop KA9Q jobs since KA9Q_RUNS_ONLY_REMOTELY='yes'"
    else
        local base_name=$(wd-get_ka9q-conf_file-base_name)
        services_list=( ft4-decode ft4-decode@* ft8-decode ft8-decode@* ft4-record ft8-record pskreporter@ft4 pskreporter@ft8 pskreporter@wspr
                        wsprdaemon radiod@${base_name}.service hfdl )
    fi

    local service_name
    for   service_name in ${services_list[@]}; do 
        (( verbosity > 1 )) && echo "Stopping service '${service_name}"
        local service_clean=${service_name%.service}
        if ! sudo systemctl is-active --quiet  ${service_clean} ; then 
            (( verbosity > 1 )) && echo "Service '${service_name}' is not active, so no need to stop it"
        else
            sudo systemctl stop ${service_clean} --no-pager >& /tmp/wd-killall.log
            rc=$? ; if (( rc )); then
                printf "'ERROR: systemctl stop ${service_clean} => $rc:\n$(</tmp/wd-killall.log)\n"
            else
                (( verbosity )) && echo "Successful systemctl stop ${service_clean} => $rc"
            fi
        fi
    done

    ### FT decoder processes left behind by their services: WD's wd-ft-decode and its jt9s, and ka9q-radio's decode_ft8
    if [[  ${KA9Q_RUNS_ONLY_REMOTELY-no} != "yes" ]]; then
        local ft_pid_list=( $(pgrep -f '^(/bin/bash )?/usr/local/bin/wd-ft-decode |^/usr/local/bin/wd-jt9 |^/usr/local/bin/decode_ft8 ') )
        if (( ${#ft_pid_list[@]} )); then
            echo "Killing ${#ft_pid_list[@]} FT decoder processes left running: ${ft_pid_list[*]}"
            sudo kill ${ft_pid_list[@]} 2> /dev/null
        fi
    fi

    ### Stop all the daemons wpawned by WD/  They store their PID in *.pid files
    local wd_temp_root_dir="${WSPRDAEMON_TMP_DIR-/dev/shm/wsprdaemon}"
    local pid_file_list=( $(find ${wd_temp_root_dir} ${WSPRDAEMON_ROOT_DIR=~/wsprdaemon} -name '*.pid') )
    if (( ! ${#pid_file_list[@]} )); then
        (( verbosity > 1 )) && printf "Found no pid files\n"
    else
        (( verbosity )) && printf "Found ${#pid_file_list[@]} pid files to kill\n" ###: '${pid_file_list[*]}'\n"

        local pid_file
        for pid_file in ${pid_file_list[@]}; do
            local pid_val=$(< ${pid_file})
            (( verbosity > 1 )) && printf "Killing PID ${pid_val} found in  pid file '${pid_file}'\n"
            sudo kill ${pid_val} >& /tmp/wd-killall.log
            rc=$?
            if (( rc )); then
                ### This commonly occurs for wav_recording_daemon.pid files, they are children of an already killed decoding_dameon
                (( verbosity )) && printf "INFO: failed to kill PID ${pid_val} found in pid file '${pid_file}'\n"
            fi
            rm ${pid_file}
            rc1=$?
            if (( rc1 )); then
                ### This commonly occurs for wav_recording_daemon.pid files, they are children of an already killed decoding_dameon
                (( verbosity )) && printf "INFO: failed to rm '${pid_file}'\n"
            fi
            if (( ! rc && ! rc1 )); then
                (( verbosity > 1 )) && printf "Killed PID  ${pid_val} found in pid file '${pid_file}' and deleted that file\n" 
            fi
        done
   fi

    #ps aux | grep 'wsprdaemon.sh\|pcmrecord\|ka9q-web\|radiod\|psk\|/usr/local/bin/jt-decode\|wd-record' | grep -v 'journalctl\|grep'  > /tmp/wd-killall.log
    local killall_pattern='wsprdaemon.sh\|\wsprdaemon\/ka9q-radio\/pcmrecord'
    if [[ "${KA9Q_WEB_SYSTEMD-no}" != "yes" ]]; then
        killall_pattern+='\|/usr/local/sbin/ka9q-web'     ### WD-spawned ka9q-web.  Left running, it holds the port and the next WD's ka9q-web loops every 6 s (N8UR 2026-09-06)
    fi
    ps aux | grep "${killall_pattern}" | grep -v 'journalctl\|grep'  > /tmp/wd-killall.log
    local zombie_pid_list=( $( awk '{print $2}'  /tmp/wd-killall.log ) )
    if (( ${#zombie_pid_list[@]} )); then
        printf "Killing ${#zombie_pid_list[@]} zombie PIDs: ${zombie_pid_list[*]}\n$(</tmp/wd-killall.log)\n"
        sudo kill ${zombie_pid_list[@]}
    fi
    local mutex_dir_list=( $(find /dev/shm/wsprdaemon -type d -name '*mutex.lock') )
    if (( ${#mutex_dir_list[@]} )) ; then
        echo "Purging ${#mutex_dir_list[@]} zombie *mutex.lock directories"
        rmdir ${mutex_dir_list[@]}
    fi
    local active_cpus_files_list=( $(find ${wd_temp_root_dir} -type f -name active_cpus_count) )
    if (( ${#active_cpus_files_list[@]} )); then
        (( verbosity > 1 )) && printf "Found ${#active_cpus_files_list[@]} active_cpus_count files\n"
        local active_cpu_file
        for active_cpu_file in ${active_cpus_files_list[@]}; do
            local cpu_count=$(<${active_cpu_file})
            if (( cpu_count )); then
                 (( verbosity )) && printf "Found active_cpu_file=${active_cpu_file} has count ${cpu_count}, so flush that file\n"
                 rm ${active_cpu_file}
             else
                 (( verbosity > 1 )) && printf "Found active_cpu_file=${active_cpu_file} has count ${cpu_count}, so leave it alone\n"
            fi
        done
    fi

    ### Clear the CPU affinity settings if they have been set by WD.  Only on the Orange Pi 5 for now.
    local wd_cgroups_root_path="/sys/fs/cgroup"
    local wd_cgroups_list=( wsprdaemon radiod linux )
    local cgroup_name
    for cgroup_name in ${wd_cgroups_list[@]} ; do
        local cgroup_path="${wd_cgroups_root_path}/${cgroup_name}"
        if [[ -d ${cgroup_path} ]]; then
            (( verbosity )) && printf "Disabling Cgroup ${cgroup_path}\n"
            local rc
            sudo rmdir  ${cgroup_path} >& /dev/null
            rc=$? ; if (( rc )); then
                local proc_list=( $(<  ${cgroup_path}/cgroup.procs ) )
                local proc_pid
                for proc_pid in ${proc_list[@]}; do
                    (( verbosity )) && echo "Killing zombie command '$(ps -o comm= $proc_pid)' "
                    sudo kill  $proc_pid >& /dev/null
                done
            fi
        else
            (( verbosity ))  && printf "Cgroup ${cgroup_path} is not configured\n"
        fi
    done
    return 0      ### Without this the exit status is that of the '(( verbosity )) && ...' above, i.e. 1 at verbosity 0, which breaks 'wdz && git pull && wda'
}

function wd-flush-hashtables() {
    wc $(find /dev/shm/wsprdaemon/ -type f -name hashtable.txt) | sort -n
    read -p "Flush these hashtable.txt files? => "
    find /dev/shm/wsprdaemon/ -type f -name hashtable.txt -delete
}

alias wdtw="test_get_wav_file_list"
function test_get_wav_file_list() {
    local sleep_seconds_between_creating_new_wav_files=${1-2}   ### defult to 2 seconds

    local testing_receiver_name="KA9Q_0"
    local testing_receiver_band="20"
    #local testing_frequency=$( get_wspr_band_freq_hz ${testing_receiver_band} )
    local testing_frequency="14095600"
    local testing_modes="W2:F2:W5"
    #local testing_wav_recording_dir=$(get_recording_dir_path ${testing_receiver_name} ${testing_receiver_band})
    local testing_wav_recording_dir=" /dev/shm/wsprdaemon/recording.d/KA9Q_0"
    local template_wav_file="${testing_wav_recording_dir%/*}/one-minute.wav"

    if [[ ! -f ${template_wav_file} ]]; then
        echo "WARNING: Can't find '${template_wav_file}', so 'cp  ~/one-minute.wav ${template_wav_file}'"
        cp ~/one-minute.wav ${template_wav_file}
    fi

    local skip_reads=0

    while true; do
        local testing__epoch

        local test_wav_file_name_list=( $( find ${testing_wav_recording_dir} -type f -name "20*Z_${testing_frequency}_usb.wav"  | sort ) )
        if (( ${#test_wav_file_name_list[@]} == 0 )); then
            local current_epoch=$(date +%s)
            local first_wav_file_epoch=$(( current_epoch - (current_epoch % 3600) ))
            testing_epoch=${first_wav_file_epoch}
        else
            local most_recent_file_name=${test_wav_file_name_list[-1]##*/}
            local most_recent_file_date_format="${most_recent_file_name:0:8} ${most_recent_file_name:9:2}:${most_recent_file_name:11:2}:${most_recent_file_name:13:2}"
            local most_recent_epoch=$(date -d "${most_recent_file_date_format}" +%s)
            testing_epoch=$(( most_recent_epoch + 60 ))
        fi

        local test_wav_file_time=$(date -d @${testing_epoch} "+%Y%m%dT%H%M%S")
        local test_wav_file_name="${testing_wav_recording_dir}/${test_wav_file_time}Z_${testing_frequency}_usb.wav"
        cp ${template_wav_file} ${test_wav_file_name}

        if (( skip_reads ==  0 )); then
            echo ""
            read -p "For testing_epoch ${testing_epoch} created ${test_wav_file_name}. Create the next one minute file? => "
            if [[ "${REPLY}" =~ ^[0-9]+$ ]]; then
                if (( REPLY > 0 )); then
                    skip_reads=$(( ${REPLY} - 1 ))
                fi
            fi
        else
            (( --skip_reads ))
            echo -n "Sleeping ${sleep_seconds_between_creating_new_wav_files} seconds.. "
            sleep ${sleep_seconds_between_creating_new_wav_files}
        fi



    done
}


function wd-restart-web() {
    local ka9q_web_pid_file=~/wsprdaemon/ka9q_web_daemon.pid

    if [[ ! -f  ${ka9q_web_pid_file} ]]; then
        echo "There is no file ${ka9q_web_pid_file}, so KA9Q-web server is not running"
        return 0
    fi
    local ka9q_web_pid=$(<  ${ka9q_web_pid_file} )
    echo "Killing the KA9Q-web server with PID = ${ka9q_web_pid}"
    kill ${ka9q_web_pid}
    rc=$?
    if [[ ${rc} -ne 0 ]]; then
        echo "WARNING: found stale pid  ${ka9q_web_pid} in ${ka9q_web_pid_file} since  kill ${ka9q_web_pid} => ${rc}"
    fi
    rm ${ka9q_web_pid_file}
    echo "Starting the  KA9Q-web server (and any other WD services which aren't running)"
    ~/wsprdaemon/wsprdaemon.sh -A
}


### On a Wsprsonde gateway, this function simplifies opening a terminal session to the WS through a USB connection
function wd2ws(){
    if [[ ! -e /dev/ttyACM0 ]]; then
        echo "ERROR: There is no USB cable on this server which is connected to a Wsprsonde"
        return 1
    fi
    read -p "Connect to the WS8 by running 'screen'? (exit 'screen' by typing <CONTROL-A> + '\') => "
    sudo screen /dev/ttyACM0
}

function wd-desnap() {
    local snaps_list=()
    local rc

    while snaps_list=( $( snap list | awk '!/Name/{print $1}') ) && [[ ${#snaps_list[@]} -gt 0 ]]; do
        local desnap_failures=0
        local snap
        for snap in ${snaps_list[@]} ; do
            #read -p "Execute 'sudo snap remove --purge ${snap}' [Y,n]? => "
            sudo snap remove --purge ${snap} >& /tmp/wd-desnap.log
            rc=$?
            if [[ ${rc} -ne 0 ]]; then
                echo "ERROR:'sudo snap remove --purge ${snap}' => ${rc}"
                (( ++desnap_failures))
            else
                echo "Removed ${snap}"
            fi
        done
        read -p "Finished de-smap pass with ${desnap_failures} failures.  Proceed [Y,n]? =>"
    done
    sudo rm -rf /var/cache/snapd/
    sudo apt autoremove --purge snapd gnome-software-plugin-snap
    rm -fr ~/snap
    sudo apt autoremove --purge snapd gnome-software-plugin-
}

export LC_TIME="C"          ### Forces user session 'date' to display 24 hour time

if [[ -d /dev/shm/wsprdaemon ]]; then
    WD_TMP_ROOT=/dev/shm/wsprdaemon
else
    WD_TMP_ROOT=/tmp/wsprdaemon
fi


### Each element containsa the function name, a comma-seperated list of its aliases, and a description of it
### ie.e:  "FUNCTION_NAME:  ALIAS1,ALIAS2.,... DESCRIOTION"
declare WD_BASH_HELP_LIST=()
declare WD_BASH_HELP_HEADER_LINE_LIST=( "FUNCTION"  "ALIAS(es)"  "DESCRIPTION" )

function wd-print-help-strings() {
    local function_name_field_width=${#WD_BASH_HELP_HEADER_LINE_LIST[0]}
    local alias_name_field_width=${#WD_BASH_HELP_HEADER_LINE_LIST[1]}
    local help_string_index

    ### Calculate the width of fields which will be printed
    for (( help_string_index=0; help_string_index < ${#WD_BASH_HELP_LIST[@]}; ++ help_string_index )) do
        local help_string="${WD_BASH_HELP_LIST[${help_string_index}]}"
        local help_string_list=( ${help_string} )

        local function_name="${help_string_list[0]}"
        if [[ ${#function_name} -gt ${function_name_field_width} ]]; then
            function_name_field_width=${#function_name}
        fi
        local alias_names="${help_string_list[1]}"
        if [[ ${#alias_names} -gt ${alias_name_field_width} ]]; then
            alias_name_field_width=${#alias_names}
        fi
    done

    printf "%${function_name_field_width}s  %${alias_name_field_width}s  %s\n" \
               "${WD_BASH_HELP_HEADER_LINE_LIST[0]}" "${WD_BASH_HELP_HEADER_LINE_LIST[1]}" "${WD_BASH_HELP_HEADER_LINE_LIST[2]}"

    for (( help_string_index=0; help_string_index < ${#WD_BASH_HELP_LIST[@]}; ++ help_string_index )) do
        local help_string="${WD_BASH_HELP_LIST[${help_string_index}]}"
        local help_string_list=( ${help_string} )

        local function_name="${help_string_list[0]}"
        local alias_names="${help_string_list[1]}"
        local description_string="${help_string_list[@]:2}"

        printf "%${function_name_field_width}s  %${alias_name_field_width}s  %s\n" "${function_name}" "${alias_names}" "${description_string}"
    done
}

WD_BASH_HELP_LIST=( "wd_logger  wdl  Emulate WD's wd_logger() function when these functions are run from the linux cmd line" )

### When this file is sourced by wsprdaemon.sh this function will be redefined in wd-utils.sh to the full wd_logger() used by WD
function wd_logger(){
    local verbosity_printing_level=${1:-1}
    local printout_string="${2:-Missing printout string}"

    if [[ ${verbosity_printing_level} -ge ${verbosity:-1} ]]; then
        echo -e "${printout_string}"
    fi
}

WD_BASH_HELP_LIST+=( "wd-machine_name - Executes 'uname -m'" )
function wd-machine-name() {
    echo $(uname -m)
}

WD_BASH_HELP_LIST+=( "wd-flush-empty-dirs - Flush any empty directories under CWD" )
function wd-flush-empty-dirs() {
    for dir in * ;  do 
        if [[ -d $dir ]]; then
            file_count=$(find $dir -type f | wc -l)
            if [[ $file_count -eq 0 ]]; then 
                echo "Deleting empty directory $dir"
                rm -r $dir
            fi
        fi
    done
}

WD_BASH_HELP_LIST+=( "wd-find-functions - Find all bash functions in \$1 and print all usage of them" )
function wd-find-functions() {
   local bash_file=$1

   local functions_list=( $( awk '/^function/{print $2}' ${bash_file} | sed 's/(.*//' ) )

   echo "Found ${#functions_list[@]} functions: '${functions_list[*]}'"

   for function in ${functions_list[@]} ; do
       grep -w ${function}  ${bash_file}
       read
   done
}

### To speed debugging of the WD upload daemons, kill just the watchdog and upload daemons.  You can then 'wda' and watch the upload log as the WD uploader starts to work immediately
function wd-kill-uploader()
{
    kill $( cat ~/wsprdaemon/watchdog_daemon.pid  ~/wsprdaemon/uploads.d/wsprnet.d/spots.d/upload_to_wsprnet_daemon.pid )
}
alias wd-k=wd-kill-uploader
alias wdk=wd-kill-uploader

### Watch the Wsprnet.log upload log file
function wd-watch-wsprnet-upload-log()
{
    local wsprnet_uploads_log_file_path=~/wsprdaemon/uploads/wsprnet/spots/upload_to_wsprnet_daemon.log
    if ! [[ -f ${wsprnet_uploads_log_file_path} ]]; then
         wsprnet_uploads_log_file_path=~/wsprdaemon/uploads.d/wsprnet.d/spots.d/upload_to_wsprnet_daemon.log
    fi
    echo "To view the full log file execute the command: ' less ${wsprnet_uploads_log_file_path} '"
    tail -F ${wsprnet_uploads_log_file_path}
}
alias wdln=wd-watch-wsprnet-upload-log

### Watch the Wsprnet.log upload log file
function wd-watch-wsprdaemon-upload-log()
{
    local view_mode="${1-tail}"

    local wsprdaemon_uploads_log_file_path=~/wsprdaemon/uploads/wsprdaemon/upload_to_wsprdaemon_daemon.log
    if ! [[ -f ${wsprdaemon_uploads_log_file_path} ]]; then
         wsprdaemon_uploads_log_file_path=~/wspsrdaemon/uploads.d/wsprdaemon.d/upload_to_wsprdaemon_daemon.log
    fi
    echo "To view the log file with 'less': add any argument"
    if [[ "${view_mode}" == "tail" ]]; then
        tail -F ${wsprdaemon_uploads_log_file_path}
    else
        less ${wsprdaemon_uploads_log_file_path}
    fi
}
alias wdld=wd-watch-wsprdaemon-upload-log

### Gets status for the Kiwis listed in WD.conf
function wd-find-kiwis()
{
    local local_lan_list=( $(ip addr | grep -w inet | grep -v 'inet 127' | awk '{print $2}' | sed 's;\.[0-9]*/;.0/;g') )

    [[ ${verbosity-0} -gt 0 ]] && echo "Searching the ${#local_lan_list[@]} LANs for Kiwis:  ${local_lan_list[@]}"
    for lan in ${local_lan_list[@]} ; do
        echo "Searching LAN ${lan} for Kiwis"
        local kiwi_ip_list=( $( nmap --open -p 8073 ${lan} | awk '/Nmap scan report for/{print $5}' ) )
        if [[ ${#kiwi_ip_list[@]} -eq 0 ]]; then
            echo "Found no Kiwis"
        else
            echo "Found ${#kiwi_ip_list[@]} Kiwis: ${kiwi_ip_list[*]/%/  }"
        fi
    done
}

function wd-kiwi-status()
{
    local wsprconf_file=~/wsprdaemon/wsprdaemon.conf;
    if [[ ! -f ${wsprconf_file} ]]; then
        echo "ERROR: can't find config file ${wsprconf_file}";
        return;
    fi;
    source ${wsprconf_file};
    local kiwi_list=();
    for rx_line in "${RECEIVER_LIST[@]}";
    do
        local rx_fields=(${rx_line});
        local rx_name=${rx_fields[0]};
        local rx_ip_port=${rx_fields[1]};
        if [[ ${rx_name} =~ AUDIO* ]] || [[ ${rx_name} =~ MERG* ]] || [[ ${rx_name} =~ RTL* ]]; then
            [[ ${verbosity-0} -ge 1 ]] && echo "skipping rx ${rx_name}";
        else
            local rx_ip=${rx_ip_port/:*/};
            if ! ping -c 1 -W 1 ${rx_ip} > /dev/null; then
                echo "WARNING: the Kiwi '${rx_name}' defined in ${wsprconf_file} doesn't respond to pings";
            else
                kiwi_list+=(${rx_name},${rx_ip_port});
                [[ ${verbosity-0} -ge 1 ]] && echo "Added ${rx_name} => ${rx_ip}";
            fi;
        fi;
    done;
    [[ ${verbosity-0} -ge 1 ]] && echo "kiwi_list[@] => ${kiwi_list[@]}";
    [[ ${verbosity-0} -ge 1 ]] && echo;
    local users;
    local users_max;
    local gps_good;
    local users_total=0;
    local wsprdaemon_total=0;
    local channels_total=0;
    for kiwi_info in "${kiwi_list[@]}";
    do
        local kiwi_name=${kiwi_info%,*};
        local kiwi_ip=${kiwi_info#*,};
        local kiwi_status="$(curl -s http://${kiwi_ip}/status)";
        users=$(sed -n '/users=\(.*\)/s//\1/p' <<< ${kiwi_status});
        users_max=$(sed -n '/users_max=\(.*\)/s//\1/p' <<< ${kiwi_status});
        channels_total=$((channels_total + users_max));
        gps_good=$(sed -n '/gps_good=\(.*\)/s//\1/p' <<< ${kiwi_status});
        [[ ${verbosity-0} -ge 1 ]] && printf "%15s: %d channels, %d users, %2d GPS satellites tracked\n" "${kiwi_name}" "${users_max}" "${users}" "${gps_good}";
        curl -s http://${kiwi_ip}/users | sed -n 's/"i":\([0-9]*\)/\n\1/g ; s/},{//g ; p' | sed '/\[{/d; s/}]//; s/,/     /g' > /tmp/curl.log;
        while read user_line; do
            [[ ${verbosity-0} -ge 1 ]] && echo "Checking '${user_line:1}' on user line ${user_line}"
            if grep --color=auto -q wsprdaemon <<< "${user_line}"; then
                [[ ${verbosity-0} -ge 1 ]] && echo "skipping WD user";
                wsprdaemon_total=$((wsprdaemon_total + 1));
            else
                [[ ${verbosity-0} -ge 1 ]] && echo "Checking '${user_line:1}' on user line ${user_line}"
                if [[ -n "${user_line:1}" ]]; then
                    eval declare $(echo "${user_line:1}" | sed '/"\([^"]*\)":/s//\1=/g');
                    [[ $? -ne 0 ]] && continue
                    local listener_channel="${user_line:0:1}";
                    local listener_name="$(echo "$n" | sed 's/%20/ /g')";
                    local listener_location="$(echo "$g" | sed 's/%20//g')";
                    local listener_freq="$f";
                    local listener_mode="$m";
                    local listener_ip="$a";
                    local listener_time="$t";
                    printf "%12s:%s %20s %8s %3s %s %20s %s\n" "${kiwi_name}" "${listener_channel}" "'${listener_name}'" "${listener_freq}" "${listener_mode}" "${listener_time}" "${listener_ip}" "${listener_location}";
                    users_total=$((users_total + 1));
                fi;
            fi;
        done < /tmp/curl.log;
    done;
    printf "Found %d Kiwis configured for %d total rx channels, %d total listeners, %d wsprdaemon channels\n" "${#kiwi_list[@]}" $channels_total $users_total $wsprdaemon_total
}

### 'git commit'
function wd-gc()
{
    if [[ -z "$1" ]]; then
        echo "usage:  wd-gc \"commit message\""
        return
    fi
    git config --global user.email "rob@robinett.us"
    git config --global user.name "Rob"
    git commit -m "$1" -a
}

### 'git push'
alias wd-gp='git push'

#### Get the most recent commit
alias wd-gl='git log | head -n 12'

### Setup a WD server 'git cloned' with https, so it can checkin to github with ssh
function wd-gc-ssh() {
    git remote set-url origin git@github.com:rrobinett/wsprdaemon.git 
    git config --global user.email "rob@robinett.us"
    git config --global user.name "Rob Robinett"
}

function wdgl() {
    git log
}

function wdgs() {
    git status
}

### Add sourcing this file to ~/.bash_aliases so that these bash aliases and functions are defined in every bash session
function wd-rci() 
{
    if [[ ! -f ~/.bash_aliases ]] || ! grep '/bash-aliases' ~/.bash_aliases > /dev/null ; then
        echo "
if [[ -f ~/wsprdaemon/bash-aliases ]]; then
    source ~/wsprdaemon/bash-aliases
fi
" >> ~/.bash_aliases
         echo "A reference to '~/wsprdaemon/bash-aliases' has been added to ' ~/.bash_aliases'"
    else
         [[ ${verbosity-0} -gt 1 ]] && echo "bash-aliases has already been installed in ~/.bash_aliases"
    fi
}
wd-rci
alias wdrci='wd-rci'

### Reload the local bash aliases and funtions defined in its ~/.bash_aliases, which will include this file after 'rci' has been executed
alias wd-rcc='source ~/.bash_aliases'
alias wd-rcc='wd-rcc'

### Common usages of Linux commands
alias l='ls -CF'
alias ll='ls -l'
alias lrt='ls -lrt'
alias la='ls -A'

alias pd=pushd
alias d=dirs

alias ag='alias | grep'

alias h=history
alias hg='history | grep'
alias j=jobs

alias cdw='cd ~/wsprdaemon/'
alias cdk='cd ~/wsprdaemon/ka9q-radio'
alias cdr='cd /etc/radio'
alias cdrf='cd /run/ft8'
alias cds='cd /etc/systemd/system'
alias pdw='pushd ~/wsprdaemon'
alias cdu='cd /dev/shm/wsprdaemon/uploads.d/'
alias cdww='cd ~/wsprdaemon/wav-archive/'

### Start 'vi' using WD's .vimrc
function vi_fixup() {
    local REPO_VIMRC="$HOME/wsprdaemon/.vimrc"
    local USER_VIMRC="$HOME/.vimrc"

    # If ~/.vimrc exists and is not the correct symlink
    if [ -e "$USER_VIMRC" ] && [ ! -L "$USER_VIMRC" ]; then
        echo "Backing up existing .vimrc to .vimrc.backup"
        mv "$USER_VIMRC" "$USER_VIMRC.backup"
    fi

    # Create symlink if it doesn't exist or points elsewhere
    if [ ! -L "$USER_VIMRC" ] || [ "$(readlink "$USER_VIMRC")" != "$REPO_VIMRC" ]; then
        ln -sf "$REPO_VIMRC" "$USER_VIMRC"
        echo "Symlinked .vimrc -> $REPO_VIMRC"
    fi

    # Run the real vi with all arguments
    /usr/bin/vi "$@"
}

alias vi=vi_fixup
alias vib='vi ~/wsprdaemon/bash-aliases'
alias vibb='vi ~/.bash_aliases'
alias viw='vi ~/wsprdaemon/wsprdaemon.sh'
alias vic='vi ~/wsprdaemon/wsprdaemon.conf'
alias vir='vi ~/wsprdaemon/radiod@rx888-wsprdaemon.conf'
alias virr='vi /etc/radio/radiod@rx888-wsprdaemon.conf'

alias tf='tail -F -n 40'
alias tfd='tf decoding_daemon.log'
alias tfr='tf wav_recording_daemon.log'
alias tfw='tf ~/wsprdaemon/watchdog_daemon.log'

alias g='git'
alias gc='git commit'
alias gd='git diff'
alias gs='git status | grep -B 100 "Untracked files"'
alias gl='git log | head'

### Get pub file for a copy/past to remote server's .ssh/authorized_keys file
alias catss='cat ~/.ssh/*pub'

### Helps cleanup WD source files
function find_unused_vars() {
    local sh_file=$1

    local var
    local vars=$(cat ${sh_file} | sed -n '/^declare /s///p' | sed 's/=.*//')
    for var in ${vars}; do
        local lines=$(grep -v '^declare' ${sh_file} | grep ${var})
        if [[ -z "${lines}" ]]; then
            echo "Found no usage of variable ${var}"
        fi
    done
}

### Prepends the date to each response from 'ping'
function pinge() 
{
   if [[ -z "$1" ]]; then
       echo "usage:  pinge DEST"
       return 1
   fi
   local dest=$1
   local pong

   ping ${dest} | while read pong; do echo "$(date): ${pong}"; done
}


function setup-ubuntu() {
    local remote_ip=$1

    if [[ -z "${remote_ip}" ]]; then
        echo "usage: ${FUNCNAME[0]} USER@IPADDRESS"
        return 0
    fi
    local url_list=( ${remote_ip/@/ } )
    if [[ ${#url_list[@]} -ne 2 ]]; then
        echo "ERROR: URL '${remote_ip}' doesn't contain a 'USER@IP'"
    fi
    scp -p ~/.ssh/authorized_keys ${remote_ip}:.ssh/
    scp -p ~/.vimrc ~/.bash_aliases ${remote_ip}:
    ### In an interactive login session:  run visudo to setup NOPASSWD:, install vim, disable VNC encryption so RealVNC clients can login
    ssh ${remote_ip} "echo 'sudo visudo
    sudo apt update 
    sudo apt install vim -y
    sudo apt install git -y
    sudo apt install sox -y
    gsettings set org.gnome.Vino require-encryption false
    ' > setup-ubuntu.sh; chmod +x setup-ubuntu.sh"
    echo
    echo "Run './setup-ubuntu.sh' to finish this installation"
    echo
    ssh ${remote_ip} 
}

# Get status of all active kiwis
function kiwi-stats(){
    local wsprconf_file=~/wsprdaemon/wsprdaemon.conf

    if [[ ! -f ${wsprconf_file} ]] ; then
        echo "ERROR: can't find config file ${wsprconf_file}"
        return
    fi
    source ${wsprconf_file}

    ## Construct a list of configured and present Kiwis
    local kiwi_list=()
    for rx_line in "${RECEIVER_LIST[@]}"; do
        local rx_fields=(${rx_line})
        local rx_name=${rx_fields[0]}
        local rx_ip_port=${rx_fields[1]}
        if [[ ${rx_name} =~ AUDIO* ]] || [[ ${rx_name} =~ MERG* ]] || [[ ${rx_name} =~ RTL* ]] ; then
            [[ ${verbosity-0} -ge 1 ]] && echo "skipping rx ${rx_name}"
        else
            local rx_ip=${rx_ip_port/:*/}
            if ! ping -c 1 -W 1 ${rx_ip} > /dev/null ; then
                echo "WARNING: the Kiwi '${rx_name}' defined in ${wsprconf_file} doesn't respond to pings"
            else
                kiwi_list+=(${rx_name},${rx_ip_port})
                [[ ${verbosity-0} -ge 1 ]] && echo "Added ${rx_name} => ${rx_ip}"
            fi
        fi
    done
    [[ ${verbosity-0} -ge 1 ]] && echo "kiwi_list[@] => ${kiwi_list[@]}"
    [[ ${verbosity-0} -ge 1 ]] && echo

    local users
    local users_max
    local gps_good
    local users_total=0
    local wsprdaemon_total=0
    local channels_total=0
    for kiwi_info in "${kiwi_list[@]}" ; do
        local kiwi_name=${kiwi_info%,*}
        local kiwi_ip=${kiwi_info#*,}
        local kiwi_status="$(curl -s http://${kiwi_ip}/status)"
        users=$(sed -n '/users=\(.*\)/s//\1/p' <<< ${kiwi_status})
        users_max=$(sed -n '/users_max=\(.*\)/s//\1/p' <<< ${kiwi_status})
        channels_total=$((channels_total + users_max))
        gps_good=$(sed -n '/gps_good=\(.*\)/s//\1/p' <<< ${kiwi_status})
        [[ ${verbosity-0} -ge 1 ]] && printf "%15s: %d channels, %d users, %2d GPS satellites tracked\n" "${kiwi_name}" "${users_max}" "${users}" "${gps_good}"

        ### I was unable to parse the curl output when it was assigned to a bash variable. After some frustration I am using the hack of
        ### piping curl's output to a /tmp file and reading and parsing it from there

        curl -s http://${kiwi_ip}/users | sed -n 's/"i":\([0-9]*\)/\n\1/g ; s/},{//g ; p' | sed '/\[{/d; s/}]//; s/,/     /g' > /tmp/curl.log
        while read user_line; do
            if grep -q wsprdaemon <<< "${user_line}" ; then
                [[ ${verbosity-0} -ge 1 ]] && echo "skipping WD user"
                wsprdaemon_total=$((wsprdaemon_total + 1))
            else
                if [[ -n "${user_line:1}" ]]; then
                    eval declare $(echo "${user_line:1}" | sed '/"\([^"]*\)":/s//\1=/g')
                    local listener_channel="${user_line:0:1}"
                    local listener_name="$(echo "$n" | sed 's/%20/ /g')"
                    local listener_location="$(echo "$g" | sed 's/%20//g')"
                    local listener_freq="$f"
                    local listener_mode="$m"
                    local listener_ip="$a"
                    local listener_time="$t"
                    printf "%12s:%s %15s %8s %3s %s %20s %s\n" "${kiwi_name}" "${listener_channel}" "'${listener_name}'" "${listener_freq}" "${listener_mode}" "${listener_time}" "${listener_ip}" "${listener_location}"
                    users_total=$((users_total + 1))
                fi
            fi
        done < /tmp/curl.log
    done
    printf "Found %d Kiwis configured for %d total rx channels, %d total listeners, %d wsprdaemon channels\n" "${#kiwi_list[@]}" $channels_total  $users_total $wsprdaemon_total
}

### Aliases which call WD
alias wd='~/wsprdaemon/wsprdaemon.sh'
alias wdl='wd -l'
alias wdle='wd -l e'
alias wda='wd -a'       ### Install/enable wsprdaemon.service if needed and 'systemctl start' it.  WD must run as the service (CPU affinity drop-ins, Restart=always), never as 'wd -A' from a shell.
alias wd-journal='journalctl -u wsprdaemon -n 100 -f'   ### Follow the stdout/stderr of the running service (what a shell-started 'wd -A' used to print to the terminal)
alias wdj='wd-journal'
### These three are plain aliases again: wsprdaemon.sh now answers -b/-t/-u before its own 'source' block, so they
### cost ~0.1 s rather than the ~60 s of CPU-tuning setup they used to pay.  See the comment there.
alias wd-decode-health='wd -b'   ### Bands whose decodes have fallen permanently behind, decodes killed by their timeout, and cycles purged before being decoded
alias wdb='wd-decode-health'
alias wd-time='wd -t'    ### Is the clock synchronised?  chronyc tracking + sources + time-sync.log
alias wdt='wd-time'
alias wd-usb='wd -u'     ### RX888s on the USB bus, their hub ports, and which ports WD can power cycle (uhubctl)
alias wdu='wd-usb'
alias wd-usb-cycle='wd -U'   ### wd-usb-cycle SERIAL|HUB:PORT|all  => power cycle that RX888 / port
alias wd-usb-retrain='wd -R'   ### An RX888 sitting at 480 Mb/s on a USB 3 jack: cut both halves of the jack and let it train again, no cable touched
alias wdr='wd-usb-retrain'
alias wds='wd -s'
alias wdz='wd-killall'  ### Quickly stops all WD-associated services and kills all WD daemons and zombies
alias wdzz='(cd; rm -r /dev/shm/wsprdaemon/*)'
alias wdc='wd -c'       ### Apply the clock ceilings in wsprdaemon.conf WITHOUT restarting WD or radiod.
                        ### Editing WD_CPU_FREQ_* alone changes nothing; this is what makes it take effect.
alias wdv='wd -V'
alias wdvv='(cd ~/wsprdaemon; echo "$(< wd_version.txt)-$(git rev-list --count HEAD)" )'
alias wdd='wd -d'       ### Increment the verbosity level of all running daemons in CWD
alias wddd='wd -D'      ### Decrement

function wd-merged() {
    local merged_file_list=( $(find ${WD_TMP_ROOT}/ -name merged.log  | sort -t / -k 7nr ) )
    local merged_file_count=${#merged_file_list[@]}
    if [[ ${#merged_file_list[@]} -eq 0 ]]; then
        echo "There are no 'merged.log' files"
        return 0
    fi
    echo "Found ${#merged_file_list[@]} 'merged.log' files:"
    local i
    for  (( i = 0; i < ${merged_file_count}; ++i )); do
        local file_line_list=( $(ls -l ${merged_file_list[${i}]}) )
        local merge_log_line_count=$( wc -l < ${merged_file_list[${i}]} )
        printf "%2d:  %-70s %s %2s %s %6s bytes = %4d lines\n" ${i} ${file_line_list[-1]} ${file_line_list[-4]}  ${file_line_list[-3]}  ${file_line_list[-2]}  ${file_line_list[-5]} ${merge_log_line_count}
    done
    if [[ "${1}" == "-k" ]]; then
         wd-merged-kfs -n
    fi
    return 0
}

function wd-merged-kfs() {
    local merged_file_list=( $(find ${WD_TMP_ROOT}/ -name merged.log | sort -t / -k 7nr) )      ### sort by ascending band freqeuncy
    local merged_file_count=${#merged_file_list[@]}
    if [[ ${merged_file_count} -eq 0 ]]; then
        echo "There are no 'merged.log' files"
        return 0
     fi
     [[ "${1}" != "-n" ]] && echo "Found ${merged_file_count} 'merged.log' files:"
     local default_index=0
     local i
     for  (( i = 0; i < ${merged_file_count}; ++i )); do
         local file_path_list=( ${merged_file_list[${i}]//\// } )
         if [[ "${file_path_list[-2]}" == "20" ]]; then
             default_index=${i}
         fi
         [[ "${1}" != "-n" ]] && printf "%2s: %s\n" ${i}   ${merged_file_list[${i}]}
     done

     local merge_log_file
     read -p "Select merged file to examine [0-${merged_file_count}], default = ${default_index} ? => "
     if [[ -z "${REPLY}" ]]; then
         REPLY=${default_index}
     fi
     local merged_log_file=${merged_file_list[${REPLY}]}
     echo "Examining  merged_log_file  ${merged_log_file} "
     sed 's/KA9Q_AIRSPYKA9Q_SDRPLAY/KA9Q_AIRSPY KA9Q_SDRPLAY/;s/TOTAL.*//'  ${merged_log_file} | awk '{printf "%s %s %s %s %s %s %10s %8s %10s:   %10s %10s %10s %10s      %10s %10s %10s\n", $1, $2, $3, $4, $5, $6, $7, $8, $9, $16, $12, $10, $11, $15, $14, $13}' | less
}


declare WD_WAV_ARCHIVE_ROOT_DIR=~/wsprdaemon/wav-archive    ### No "" so ~ is expanded
### 11/2/22 - This doesn't work anymore, but it could be modifed 
function wd-grape-info(){
    local grape_date_dirs_list=( $( find ${WD_WAV_ARCHIVE_ROOT_DIR} -mindepth 1 -maxdepth 1 -type d | sort) )
    wd_logger 1 "Found ${#grape_date_dirs_list[@]} date dirs: ${grape_date_dirs_list[*]##*/}"
    local date_dir
    for date_dir in ${grape_date_dirs_list[@]}; do
        local file_count=$( find ${date_dir} -type f | wc -l )
        printf "Date %s contains %5d files: " ${date_dir##*/} ${file_count}
        case ${file_count} in
            0)
                read -N 1 -p "There are no files in the date ${date_dir##*/} tree, so delete that tree? [Yn] => "
                if [[ ${REPLY:-y} == "y" ]]; then
                    echo "\nDeleting directory tree ${date_dir}"
                    rm -r ${date_dir}
                fi
                ;;
            21)
                echo "has been purged of all but the sox.log, 24h.wav and the 'has been uploaded' files"
                ;;
            14421)
                echo "and is fill with 14400 flac files, the sox, wav and 'has been uploaded' files"
                ;;
            *)
                local date=$(printf "%(%Y%m%d)T")
                if [[ "${date_dir##*/}" == "${date}" ]]; then
                    echo "Skipping more checks for files being created today"
                else
                    read -p "has invalid number ${file_count} of files. List the non-flac files? => "
                    if [[ ${REPLY:-n} == "y" ]]; then
                        find ${date_dir} -type f ! -name '*.flac' | less
                    fi
                fi
                ;;
        esac
    done
}
alias wdgi='wd-grape-info'

function wd-wav-clean() {
    local wav_archive_root_dir_list=( $(find ${WD_WAV_ARCHIVE_ROOT_DIR} -mindepth 1 -maxdepth 1 -type d | sort) )
    local wav_archive_root_dir
    for wav_archive_root_dir in ${wav_archive_root_dir_list[@]} ; do
        local flat_file_list=( $(find ${wav_archive_root_dir} -type f | sort) )  
        if [[ ${#flat_file_list[@]} -eq 0 ]]; then
            read -p "wav file root dir ${wav_archive_root_dir} contains only directory files, so delete it? [Yn] => "
            if [[ "${REPLY:-y}" == "y" ]]; then
                rm -r ${wav_archive_root_dir}
            fi
        else
            printf "Found %6d files under %s\n"  ${#flat_file_list[@]}  ${wav_archive_root_dir}
        fi
    done
}

### Human-readable byte count (falls back to raw bytes if numfmt is missing).
function wd-hrb() { numfmt --to=iec --suffix=B "${1:-0}" 2>/dev/null || echo "${1:-0}B"; }

### Echo the value of VAR from wsprdaemon.conf (tolerates 'declare ', quotes, trailing comment), or nothing.
function wd-conf-value() {
    local var=$1 conf="${2:-${HOME}/wsprdaemon/wsprdaemon.conf}" line val
    [[ -r ${conf} ]] || return 0
    line=$( command grep -E "^[[:space:]]*(declare[[:space:]]+(-[a-zA-Z]+[[:space:]]+)?)?${var}=" "${conf}" | tail -1 )
    [[ -z ${line} ]] && return 0
    val=${line#*${var}=}; val=${val%%#*}; val=${val//[\"\' ]/}
    echo "${val}"
}

### HELP: wd-grape-purge-uploaded (alias wdgpu) - reclaim disk from PSWS-delivered wav-archive trees.
### HELP: A reporter dir <DATE>/<REPORTER>_<GRID> holding the 'pswsnetwork_upload_completed' marker has been fully
### HELP: uploaded, so its raw *.wv work files, 24_hour_10sps_iq.wav, sox.log and marker are all purgeable.
### HELP:
### HELP: By design raw files are KEPT while the volume has room; they are purged OLDEST-FIRST only once the
### HELP: filesystem holding the archive reaches the fill threshold, and only enough to drop back under it.
### HELP: This keeps space free for the current day's recording (a full disk = missing minutes = silence-padded,
### HELP: partial PSWS uploads).  Undelivered dirs (no marker, e.g. today's recording) are never touched.
### HELP:
### HELP: It also reports the filesystem's free space and raises a DEADLOCK ALERT when there is not room for one
### HELP: day's recording even after purging every delivered tree (add disk / drop bands / lower retention).
### HELP:
### HELP: Usage: wd-grape-purge-uploaded [-r] [-a]      (alias wdgpu)
### HELP:   (default) DRY RUN: report filesystem space and what WOULD be purged; change nothing.
### HELP:   -r        REAL RUN: actually delete and prune emptied <DATE> trees.
### HELP:   -a        Purge ALL delivered trees, ignoring the fill threshold (full reclaim).
### HELP: Config (export in your shell/profile to override the defaults):
### HELP:   WD_ARCHIVE_FILL_PERCENT (default 75)  purge delivered trees once the volume is >= this % full.
### HELP:   WD_ARCHIVE_MIN_FREE_GB  (default 0)   GB one day of recording needs for the deadlock check;
### HELP:                                         0 => auto-estimate from the largest <DATE> tree present.
function wd-grape-purge-uploaded() {
    local marker_name="pswsnetwork_upload_completed"
    local do_delete="no" purge_all="no" arg
    for arg in "$@"; do
        case "${arg}" in
            -r)       do_delete="yes" ;;
            -a)       purge_all="yes" ;;
            -ra|-ar)  do_delete="yes"; purge_all="yes" ;;
            -h|--help)
                cat <<'EOF'
wd-grape-purge-uploaded [-r] [-a]   (alias wdgpu) - reclaim disk from PSWS-delivered wav-archive trees.

A reporter dir <DATE>/<REPORTER>_<GRID> holding the 'pswsnetwork_upload_completed' marker is fully
uploaded, so its raw *.wv work files, 24_hour_10sps_iq.wav, sox.log and marker are purgeable.
Raw files are KEPT while there is room; they are purged OLDEST-FIRST once the volume reaches the fill
threshold, and only enough to drop back under it, so space stays free for the current day's recording.
Undelivered dirs (no marker, e.g. today's recording) are never touched.

  (default)  DRY RUN: report free space and what WOULD be purged; change nothing.
  -r         REAL RUN: delete and prune emptied <DATE> trees.
  -a         Purge ALL delivered trees, ignoring the fill threshold (full reclaim).

Config (export to override):
  WD_ARCHIVE_FILL_PERCENT (default 75)  purge once the volume is >= this % full.
  WD_ARCHIVE_MIN_FREE_GB  (default 0)   GB one day of recording needs; 0 => auto from largest <DATE> tree.
Returns 2 when even purging every delivered tree cannot free one day's space (deadlock alert).
EOF
                return 0 ;;
            *) echo "usage: wd-grape-purge-uploaded [-r] [-a]  (see 'wdgpu -h')"; return 1 ;;
        esac
    done

    if [[ ! -d ${WD_WAV_ARCHIVE_ROOT_DIR} ]]; then
        echo "No wav-archive dir '${WD_WAV_ARCHIVE_ROOT_DIR}', nothing to purge."
        return 1
    fi

    ### Config precedence: environment var > wsprdaemon.conf > built-in default (matches the wd-cleanup.sh cron).
    local _cv
    _cv="${WD_ARCHIVE_FILL_PERCENT:-$(wd-conf-value WD_ARCHIVE_FILL_PERCENT)}"; local -i fill_percent=${_cv:-75}
    _cv="${WD_ARCHIVE_MIN_FREE_GB:-$(wd-conf-value WD_ARCHIVE_MIN_FREE_GB)}";   local -i min_free_gb=${_cv:-0}

    ### Stats for the volume that physically holds the archive (df follows to its real mountpoint).
    local -a df_fields
    df_fields=( $(df -P -B1 "${WD_WAV_ARCHIVE_ROOT_DIR}" | awk 'NR==2{print $2, $3, $4, $5, $6}') )
    local -i fs_total=${df_fields[0]} fs_used=${df_fields[1]} fs_avail=${df_fields[2]}
    local -i fs_usep=${df_fields[3]%\%}
    local fs_mount=${df_fields[4]}

    ### One day's space requirement for the room / deadlock checks.
    local -i one_day_bytes=0
    local estimate_src
    if (( min_free_gb > 0 )); then
        one_day_bytes=$(( min_free_gb * 1000000000 ))
        estimate_src="WD_ARCHIVE_MIN_FREE_GB=${min_free_gb}"
    else
        local d db
        for d in $(find "${WD_WAV_ARCHIVE_ROOT_DIR}" -mindepth 1 -maxdepth 1 -type d); do
            db=$( du -sb "${d}" 2>/dev/null | cut -f1 )
            (( db > one_day_bytes )) && one_day_bytes=${db}
        done
        estimate_src="auto: largest <DATE> tree"
    fi

    ### Delivered (uploaded) reporter trees, OLDEST FIRST (path sort => YYYYMMDD date sort).
    local -a marker_list
    mapfile -t marker_list < <(find "${WD_WAV_ARCHIVE_ROOT_DIR}" -type f -name "${marker_name}" | sort)

    printf "=== wd-grape-purge-uploaded %s ===\n" "$( [[ ${do_delete} == yes ]] && echo "REAL RUN" || echo "DRY RUN (pass -r to delete)" )"
    printf "Volume %s holding %s:\n" "${fs_mount}" "${WD_WAV_ARCHIVE_ROOT_DIR}"
    printf "  size %s   used %s (%d%%)   free %s   fill-threshold %d%%\n" \
        "$(wd-hrb ${fs_total})" "$(wd-hrb ${fs_used})" "${fs_usep}" "$(wd-hrb ${fs_avail})" "${fill_percent}"
    printf "  one day of recording needs ~%s (%s)\n" "$(wd-hrb ${one_day_bytes})" "${estimate_src}"
    printf "  room for another full day right now? %s\n" \
        "$( (( fs_avail >= one_day_bytes )) && echo "YES ($(wd-hrb ${fs_avail}) free)" || echo "*** NO -- only $(wd-hrb ${fs_avail}) free ***" )"

    ### Total reclaimable across every delivered tree.
    local -i delivered_bytes=0 dbytes
    local m reporter_dir
    for m in "${marker_list[@]}"; do
        dbytes=$( du -sb "${m%/*}" 2>/dev/null | cut -f1 )
        (( delivered_bytes += dbytes ))
    done

    ### DEADLOCK ALERT: even purging everything delivered still can't buy one day.
    local -i avail_after_all=$(( fs_avail + delivered_bytes ))
    local -i deadlock=0
    if (( avail_after_all < one_day_bytes )); then
        deadlock=1
        printf "\n*** DEADLOCK ALERT ***\n"
        printf "    Purging ALL %d delivered tree(s) would free %s, leaving %s free -- still LESS than one day (~%s).\n" \
            "${#marker_list[@]}" "$(wd-hrb ${delivered_bytes})" "$(wd-hrb ${avail_after_all})" "$(wd-hrb ${one_day_bytes})"
        printf "    This site will keep producing partial (silence-padded) PSWS uploads until you add disk,\n"
        printf "    reduce recorded bands/receivers, or point wav-archive at a larger volume.\n"
    fi

    if (( ${#marker_list[@]} == 0 )); then
        printf "\nNo delivered reporter trees present to purge.\n"
        return $(( deadlock ? 2 : 0 ))
    fi

    ### Below threshold and not forcing -a: keep everything (raw files retained while there's room).
    if [[ ${purge_all} == no ]] && (( fs_usep < fill_percent )); then
        printf "\n%d%% full is below the %d%% threshold: keeping all %d delivered tree(s), %s reclaimable when needed.\n" \
            "${fs_usep}" "${fill_percent}" "${#marker_list[@]}" "$(wd-hrb ${delivered_bytes})"
        return $(( deadlock ? 2 : 0 ))
    fi

    if [[ ${purge_all} == yes ]]; then
        printf "\nPurging ALL %d delivered tree(s) (-a), oldest first:\n" "${#marker_list[@]}"
    else
        printf "\n%d%% full >= %d%% threshold: purging oldest delivered tree(s) until under threshold:\n" "${fs_usep}" "${fill_percent}"
    fi

    local -i freed=0 purged=0 used_now=${fs_used} cur_usep
    for m in "${marker_list[@]}"; do
        reporter_dir="${m%/*}"                                        ### <DATE>/<REPORTER>_<GRID>
        dbytes=$( du -sb "${reporter_dir}" 2>/dev/null | cut -f1 )
        printf "  %9s  %s\n" "$(wd-hrb ${dbytes})" "${reporter_dir#${WD_WAV_ARCHIVE_ROOT_DIR}/}"
        [[ ${do_delete} == yes ]] && rm -rf "${reporter_dir}"
        (( used_now -= dbytes, freed += dbytes, purged++ ))
        if [[ ${purge_all} == no ]]; then
            ### Re-measure after a real delete; project from running total during a dry run.
            if [[ ${do_delete} == yes ]]; then
                cur_usep=$( df -P "${WD_WAV_ARCHIVE_ROOT_DIR}" | awk 'NR==2{gsub(/%/,"",$5);print $5}' )
            else
                cur_usep=$(( fs_total > 0 ? used_now * 100 / fs_total : 0 ))
            fi
            (( cur_usep < fill_percent )) && break
        fi
    done

    if [[ ${do_delete} == yes ]]; then
        while [[ -n $( find "${WD_WAV_ARCHIVE_ROOT_DIR}" -mindepth 1 -type d -empty -print -quit ) ]]; do
            find "${WD_WAV_ARCHIVE_ROOT_DIR}" -mindepth 1 -type d -empty -delete
        done
    fi

    printf "=== %s %s across %d of %d delivered tree(s) ===\n" \
        "$( [[ ${do_delete} == yes ]] && echo "Freed" || echo "Would free" )" \
        "$(wd-hrb ${freed})" "${purged}" "${#marker_list[@]}"
    [[ ${do_delete} != yes ]] && echo "Re-run with -r to delete (add -a to purge ALL delivered, ignoring the threshold)."
    return $(( deadlock ? 2 : 0 ))
}
alias wdgpu='wd-grape-purge-uploaded'

### At sites which are archiving wav files, this gets some info
alias wd-wav-archive='df -h ~ ; du -sh ~/wsprdaemon/wav-archive/ ; ls -lt ~/wsprdaemon/wav-archive/ | head  -n 3; ls -lt ~/wsprdaemon/wav-archive/ | tail -n 2'

### HELP:  On systems where there ar WWV/CHU channels defined, but GRAPE_PSWS_ID is not defined in WD.conf, the .wv files will be stored in a date...directory nameed ...@NOT_DEFINED
### HELP:  Once GRAPE_PSWS_ID is defined, this command searches for files in the @NOT_DEFINED/... tree and moves them to @${GRAPE_PSWS_ID}/...
function wd-wav-archive-fix(){
    source ~/wsprdaemon/wsprdaemon.conf
    if [[ -z "${GRAPE_PSWS_ID-}" ]]; then
        echo "This server is not configured with a GRAPE_PSWS_ID in the wsprdaemon.conf file, so can't fix the directories"
        return 0
    fi
    if ! [[ ${GRAPE_PSWS_ID} =~ _ ]]; then
        echo "GRAPE_PSWS_ID='${GRAPE_PSWS_ID}' does not match the expected format 'Snnnn_nnn'"
        return 0
    fi
    local date_dir
    for date_dir in $(find -L ${WD_WAV_ARCHIVE_ROOT_DIR} -mindepth 1 -maxdepth 1 -type d | sort ); do 
        #echo "date_dir = ${date_dir}"
        local reporter_dir
        for reporter_dir in $(find ${date_dir} -mindepth 1 -maxdepth 1 -type d | sort ); do
            #echo "reporter_dir = ${reporter_dir}"
            local receiver_dir
            for receiver_dir in $(find ${reporter_dir} -mindepth 1 -maxdepth 1 -type d | sort ); do
                echo  "Examining  ${receiver_dir} "
                if [[ ${receiver_dir} =~ "@NOT_DEFINED" ]]; then
                    local new_root_dir="${receiver_dir/NOT_DEFINED/${GRAPE_PSWS_ID}}"
                    read -p "Move .wv files under ${receiver_dir} to corresponding directores under ${new_root_dir} ? => "
                    local path
                    (
                    cd ${receiver_dir}
                    find . -mindepth 1 -type f -print0 | while IFS= read -r -d '' path; do
                        local src="${path}"
                        local dst="${new_root_dir}/${path}"
                        mkdir -p "$(dirname "${dst}")"
                        mv "${src}" "${dst}"
                     done
                    )
                    rm -r ${receiver_dir}
                elif ! [[ ${receiver_dir} =~ "@" ]]; then
                   read -p  "Fix receiver directory ${receiver_dir} by renaming it to ${receiver_dir}@${GRAPE_PSWS_ID}? => "
                   mv ${receiver_dir} ${receiver_dir}@${GRAPE_PSWS_ID}
                fi
            done
        done
    done
}

function wd-sftp-psws(){
    local rc
    local start_epoch 
    local end_epoch
    local runtime_seconds

    source ~/wsprdaemon/wsprdaemon.conf
    
    if [[ -z "${GRAPE_PSWS_ID-}" ]]; then
        if [[ -z "${PSWS_STATION_ID-}" || -z "${PSWS_DEVICE_ID-}" ]]; then
            wd_logger 1 "This WD server is not configured to upload to the HamSCI GRAPE server"
            return 0
        fi
        GRAPE_PSWS_ID="${PSWS_STATION_ID}_${PSWS_DEVICE_ID}"
        wd_logger 1 "The GRAPE_PSWS_ID=${GRAPE_PSWS_ID} is being derived from PSWS_STATION_ID=${PSWS_STATION_ID} and PSWS_DEVICE_ID=${PSWS_DEVICE_ID}"
    fi
    if ! [[ ${GRAPE_PSWS_ID} =~ _ ]]; then
        echo "GRAPE_PSWS_ID='${GRAPE_PSWS_ID}' does not match the expected format 'Snnnn_nnn'"
        return 0
    fi
    local tmpfile=$(mktemp)
    trap "rm -f '$tmpfile'" RETURN  # cleanup 

    local psws_url="${PSWS_URL-pswsnetwork.eng.ua.edu}"

    local nc_timeout=${PSWS_NC_TIMEOPUT_SECONDS-2}
    echo "Testing Internnet connectivity to the PSWS server by running 'nc -vz -w ${nc_timeout} ${psws_url} 22'"
    start_epoch=${EPOCHSECONDS}
    nc -vz -w ${nc_timeout} ${psws_url} 22 >& ${tmpfile}
    rc=$?
    end_epoch=${EPOCHSECONDS}
    runtime_seconds=$(( end_epoch - start_epoch ))
    if (( rc )); then
        printf "After %2d seconds, Failed to run 'nc -vz -w ${psws_url} 22' => ${rc}:\n$(< ${tmpfile} )\n" "${runtime_seconds}" 
        rm -f ${tmpfile}
        return 1
    fi
    printf "After %2d seconds, Successfully ran 'nc -vz ${psws_url} 22'\n" "${runtime_seconds}"

    if [[ ! -d ~/.ssh ]]; then
        printf "The ~/.ssh directory which contains ssh keys doesn't exist.  Run 'ssh-keygen -t 25519 -N '' to create them"
        rm -f ${tmpfile}
        return 1
    fi
    local public_key_file_path_list=( $( find ~/.ssh -name '*.pub') )
    if (( ${#public_key_file_path_list[@]} == 0 )); then
        printf "This station ${GRAPE_PSWS_ID%_*} has no ssh public key.  Run 'ssh-keygen -t 25519 -N \"\"' to create it\n"
        rm -f ${tmpfile}
        return 1
    fi
    local public_key_file_path="${public_key_file_path_list[0]}"
    (( ${verbosity-0} )) && printf "This station ${GRAPE_PSWS_ID%_*}'s public key is:\n $(< ${public_key_file_path})\n" 

    local psws_timeout=$(( nc_timeout * 10 ))    ### Default is to wait up to 20 seconds to execute a simple sftp connection to the PSWS server
    echo "Testing sftp autologin with 'sftp -o ConnectTimeout=${psws_timeout}  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -b /dev/null ${GRAPE_PSWS_ID%_*}@${psws_url}'"
    local start_epoch=${EPOCHSECONDS}
    sftp -o ConnectTimeout=${psws_timeout} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
            -b /dev/null ${GRAPE_PSWS_ID%_*}@${psws_url} >& ${tmpfile}
    rc=$? 
    end_epoch=${EPOCHSECONDS}
    runtime_seconds=$(( end_epoch - start_epoch ))
    if (( rc == 0 )); then
        printf "After %2d seconds, Successfully ran 'sftp -o ConnectTimeout=${psws_timeout}  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -b /dev/null ${GRAPE_PSWS_ID%_*}@${psws_url}'\n" "${runtime_seconds}"
    else
        printf "After %2d seconds, the Internet connection is open, but failed to run 'sftp -o ConnectTimeout=${psws_timeout}  -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -b /dev/null ${GRAPE_PSWS_ID%_*}@${psws_url}' => ${rc}:\n$(< ${tmpfile} )\n"
        printf "Verify that this station '${GRAPE_PSWS_ID%_*}'s public key has been registered with the PSWS server:\n $(< ${public_key_file_path})\n"
    fi
    rm -f ${tmpfile}
    return 0
}
alias wdssp=wd-sftp-psws

function wd-overloads()
{
: <<"comment_out"
    local log_file_list=( $(find /dev/shm/wsprdaemon/ -name wav_status.log | sort ) )
    #printf "Searching ${#log_file_list[@]} 'wav_status.log' files for WAV overload events\n%-40s YYMMDD_HHMM      MIN_SAMPLE MAX_SAMPLE Count_of MIN/MAX\n" "RECIEVER/BAND/log_file"

    if [[ ${#log_file_list[@]} -eq 0 ]]; then
        echo "Can't find any 'wav_status.log' files, so this must be an older version of WD"
    else
       true  #awk '$2 == -1.0 || $3 == 1.0 { printf( "%-40s %s\n", substr(FILENAME,33) ":", $0) } ' ${log_file_list[@]}
    fi
comment_out

    log_file_list=( $(find /dev/shm/wsprdaemon/ \( -name kiwi_ovs.log -o -name adc_overloads.log \) -printf "%p\n" | sort ) )
    if [[ ${#log_file_list[@]} -eq 0 ]]; then
        echo "Can't find any 'kiwi_ovs.log or ad_overloads.log' files"
        return
    fi

    if [[ $# -ne 0 && "$1" == "-f" ]]; then
        read -p "Press <ENTER> to flush all of the ${#log_file_list[@]} overload..log files => "
        rm ${log_file_list[@]}
        return
    fi

    ### Get a list of the Kiwis
    local kiwi_list=()
    local kiwi_root_path_list=()

    local file_path
    for file_path in ${log_file_list[@]} ; do
        local file_path_list=( ${file_path//\// } )         ### put each directory name in an element of an array
        local kiwi_name=${file_path_list[-3]-}              ### two directories up from the log file is the directory with the name of the kiwi
        if [[ ! " ${kiwi_list[*]} " =~ " ${kiwi_name} " ]]; then
            kiwi_list+=("${kiwi_name}")
           local temp_path_list
           temp_path_list=(${file_path//\// })
           local kiwi_root_path=${temp_path_list[@]:0:$(( ${#temp_path_list[@]} - 2 ))}    ### chop off BAND/kiwi_ovs.log
           kiwi_root_path="/${kiwi_root_path// /\/}"
           #echo "kiwi_root_path = ${kiwi_root_path}"
           kiwi_root_path_list+=(${kiwi_root_path})
           #read -p "Next Kiwi path => "
        fi
    done
    printf "Found there are ${#kiwi_list[@]} Wsprdaemon SDRs: ${kiwi_list[*]}\n\n"

    for kiwi_root_path in ${kiwi_root_path_list[@]} ; do
        #echo "Searching under ${kiwi_root_path}"
        local kiwi_name=${kiwi_root_path##*/}
        log_file_list=( $(find ${kiwi_root_path} \( -name kiwi_ovs.log -o -name adc_overloads.log \) -printf "%p\n" | sort ) )

        local newest_log_file=$(ls -1t ${log_file_list[@]} | head -1)
        local most_ovs_line=$(sort -k 3n ${newest_log_file} | awk '$2 != $3' | tail -n 1)
        local ov_report_count=$( wc -l < ${newest_log_file})
        local ov_non_zero_count=$(awk '$3 != 0' ${newest_log_file} | wc -l)

        local most_ovs_line_list=( ${most_ovs_line} )
        if [[ ${most_ovs_line_list[2]-0} -eq 0 ]]; then
            printf "SDR ${kiwi_name} reports no overload events\n\n"
            continue
        fi

        printf "For SDR '${kiwi_name}' found in '${newest_log_file}', the newest of the ${#log_file_list[@]} '*.log' files:\n"
        printf "There are %6d total wspr cycles in the log file, of which %6d cycles report overload events\n"  ${ov_report_count}  ${ov_non_zero_count}
        printf "This line reports the most OVs:\n${most_ovs_line}\n" 

        read -p "Press 'l + <RETURN>' to execute 'less ${newest_log_file}', then search for ${most_ovs_line_list[0]} => " user_input
        if [[ -n "${user_input}" ]]; then
            less ${newest_log_file}
        fi
        printf "\n"
    done
}
alias wd-ov='wd-overloads'
alias wdov='wd-ov'

function wd-query()
{
    if [[ $# -lt 1 ]]; then
        echo "usage:  wd-query {-r | -t} ID [HOURS_TO_SEARCH]       -r => search for ID of reporter     -t => search for ID of transmit beacon"
        return 0
    fi
    local query_target_field="rx_id"
    case $1 in
        -r) query_target_field="rx_id";shift ;;
        -t) query_target_field="tx_call";shift ;;
    esac
    if [[ $# -lt 1 ]]; then
        echo "ERROR: missing ID"
        echo "usage:  wd-query {-r | -t} ID [HOURS_TO_SEARCH]       -r => search for ID of reporter     -t => search for ID of transmit beacon"
        return 1
    fi
    local target_val=$1
    local query_hours=${2-1}
    local query_minutes=$((query_hours * 60))

    PGPASSWORD=JTWSPR2008 psql -U wdread -h wsprdaemon.org -p 5432 -d tutorial -c \
        "select time,tx_call,tx_grid,rx_id,receiver,rx_grid,freq,\"tx_dBm\",\"SNR\",drift,mode,metric as \"S width (mHz)\" from wsprdaemon_spots_s where ${query_target_field}='${target_val}' and mode >0 and time > now() - interval '${query_minutes} minutes' order by (time,rx_id,receiver) desc;"
}
alias wd-q='wd-query'
alias wdq='wd-query'

alias wd-syslog='sudo tail -F /var/log/syslog'
alias wd-syslogl='sudo less /var/log/syslog'
alias wdsl='wd-syslog'
alias wd-wd-rec='watch "ps aux | grep wd-rec | grep -v grep | sort -k 14,14n -k 15r"'
alias wdwd='wd-wd-rec'
function wd-wd-rec-flush()
{
    local wd_record_pids=( $(ps aux | grep wd-rec | grep -v grep | awk '{print $2}') )
    if [[ ${#wd_record_pids[@]} -gt 0 ]]; then
        echo "Flushing ${#wd_record_pids[@]} wd-record jobs"
        kill ${wd_record_pids[@]}
    fi
}

function wd-freqs(){
    local rec_root_dir="/dev/shm/wsprdaemon/recording.d"
    local rec_rx_root_dir_list=( $( find ${rec_root_dir}/* -type d -prune ) )
    local rec_bands_list=( $( find ${rec_root_dir} -mindepth 2 -maxdepth 2 -type d ! -name kiwi_status.d -printf "%f\n" | sort -u ) )

    local first_print="yes"
    for band in ${rec_bands_list[@]}; do
        if [[ ${first_print} == "no" ]]; then
            echo
        fi
        first_print=no
        for rx_root_dir in ${rec_rx_root_dir_list[@]} ; do
            local rx_name=${rx_root_dir##*/}
            echo -n "${rx_name}: "
            local rx_band_dir=${rx_root_dir}/${band}
            if [[ -d ${rx_band_dir} ]]; then
                printf "%s" "$( grep "^23.*AI6" ${rx_band_dir}/decoding_daemon.log | tail -n 1)"
            fi
            echo
        done | sort
    done
}

function wd-wavs() {
    local rx_band=$1

    if [[ -z "${rx_band}" ]]; then
        echo "usage: wd-wavs BAND"
        return 0
    fi

    for rx_dir in */${band} ; do
        ls -lt ${rx_dir}/${rx_band}/*.wav | head -1
    done
    return 0

    local rec_root_dir="/dev/shm/wsprdaemon/recording.d"
    local rec_rx_root_dir_list=( $( find ${rec_root_dir}/* -type d -prune ) )
    local rec_bands_list=( $( find ${rec_root_dir} -mindepth 2 -maxdepth 2 -type d ! -name kiwi_status.d -printf "%f\n" | sort -u ) )

    local first_print="yes"
    for band in ${rec_bands_list[@]}; do
        echo =n ${band}
    done
}

### Monitor and control the KA9Q-radio service
###
function wd-get_ka9q-conf_file-base_name() {
    source ~/wsprdaemon/wsprdaemon.conf
    local conf_base_name

    if [[ -n "${KA9Q_CONF_NAME-}" ]]; then
        conf_base_name="${KA9Q_CONF_NAME##*/}"          ### remove leading path if present
        conf_base_name="${conf_base_name%.conf}"        ### remove trailing '.conf' if present
        if [[ ! -f /etc/radio/radiod@${conf_base_name}.conf ]]; then
            echo "ERROR: the KA9Q-radio conf file ${KA9Q_CONF_NAME} defined in wsprdaemon.conf doesn't exist in /etc/radio/" 1>&2
            echo ""
            return 1
        fi
        [[ ${verbosity-0} -ge 1 ]] && echo "Using conf name extracted from the line KA9Q_CONF_NAME='${KA9Q_CONF_NAME}' found in  ~/wsprdaemon/wsprdaemon.conf" 1>&2
        echo "${conf_base_name}"
        return 0
    fi

    ### KA9Q_CONF_NAME wasnt defined in wsprdaemon.conf.  See if there is one and only one
    local conf_file_list=( $(find /etc/radio -maxdepth 1 -type f -name 'radiod@*.conf' -o -type d -name '*.conf.d') )
    case ${#conf_file_list[@]} in
        0) 
            echo "ERROR: There is no KA9Q_CONF_NAME='...' line found in  ~/wsprdaemon/wsprdaemon.conf and there are no radiod@...conf files in /etc/radio" 1>&2
            echo ""
            return 1
            ;;
        1)
            conf_base_name=${conf_file_list[0]##*/radiod@}         ### remove leading path if present
            conf_base_name="${conf_base_name%.conf}"               ### remove trailing '.conf' if present
            conf_base_name="${conf_base_name%.conf.d}"             ### remove trailing '.conf' if present
            [[ ${verbosity-0} -ge 1 ]] && echo "There is no KA9Q_CONF_NAME='${KA9Q_CONF_NAME}' found in  ~/wsprdaemon/wsprdaemon.conf. But since there is only one one radiod@..conf file '${conf_file_list[0]}', assume it is the active radiod configuration" 1>&2
            echo "${conf_base_name}"
            return 0
            ;;
        *)
            echo "ERROR: There is no KA9Q_CONF_NAME='...' line in ~/wsprdaemon/wsprdaemon.conf and there are more than one /etc/radio/radiod@...conf file. So remove all but the active file:" 1>&2
            echo "       ${conf_file_list[*]}" 1>&2
            echo ""
            return 1
            ;;
    esac
    ### Should never get here
    echo ""
    return 1
}

function wd-get-ka9q-conf-file-path () {
    local base_name=$(wd-get_ka9q-conf_file-base_name)
    local conf_file_path="/etc/radio/radiod@${base_name}.conf"
    if [[ ! -f ${conf_file_path} ]]; then
        echo "ERROR: can't find expected conf file: ${conf_file_path}"
        return 1
    fi
    echo ${conf_file_path}
    return 0
}

#alias wd-rl='sudo journalctl -u radiod@rx888-wsprdaemon.service'     ### show the syslog entries for the radiod service.  add -f to watch new log lines appear
function wd-rl() {
    local rl_args=${@--n 80}
    local base_name=$(wd-get_ka9q-conf_file-base_name)
    sudo journalctl -u radiod@${base_name} -f ${rl_args}
}
alias wdrl='wd-rl'

function wd-radiod-action(){
   local action=$1

   local ka9q_base_name=$(wd-get_ka9q-conf_file-base_name)
   if [[ -z "${ka9q_base_name}" ]]; then
       echo "ERROR: can't find a config file for KA9Q-radio"
       return 1
   fi

   local ka9q_services_list=( radiod@${ka9q_base_name} ft4-record ft8-record ft4-decode ft8-decode  pskreporter@ft4  pskreporter@ft8  pskreporter@wspr )

   for service in ${ka9q_services_list[@]}; do
       if [[ "${action}"  == "status" ]]; then
           if sudo systemctl is-active --quiet ${service}.service; then
               printf "Service %20s is     active\n" "${service}"
           else
               printf "Service %20s is not active\n" "${service}"
           fi
       else
           ### should be 'start' or 'stop'
           sudo systemctl ${action} ${service}.service
       fi
   done

}

alias wd-ra='wd-radiod-action start'  ### show it's status
alias wdra='wd-ra'

alias wd-rz='wd-radiod-action stop'  ### show it's status
alias wdrz='wd-rz'

alias wd-rs='wd-radiod-action status'  ### show it's status
alias wdrs='wd-rs'

function wd-radiod-conf-edit() {
   local conf_file_path

   if ! conf_file_path=$(wd-get-ka9q-conf-file-path); then
       echo "ERROR: ${conf_file_path}"
       return 1;
   fi
   vi ${conf_file_path}
}
alias wdrv='wd-radiod-conf-edit'

### WD systemctl 
alias wd-ss='sudo systemctl'
alias wdss=wd-ss

alias wd-wd-start='sudo systemctl start wsprdaemon.service'
alias wdwa='wd-wd-start'

alias wd-wd-stop='sudo systemctl stop wsprdaemon.service'
alias wdwz='wd-wd-stop'

alias wd-wd-status='sudo systemctl status wsprdaemon.service'
alias wdws='wd-wd-status'

### Checks that the radiod config file is set with the desired low = 1300, high = 1700 and fix them if they were set to 100, 5000 by WD 3.1.4
function wd-radiod-bw-check() {
    local running_radiod_conf_file=$( sudo systemctl status | grep -v awk | awk '/\/etc\/radio\/radiod.*conf/{print $NF}' )
    if [[ -z "${running_radiod_conf_file}" ]]; then
        wd_logger 1 "radiod is not configured to run on this server"
        return 0
    fi
    local rx_audio_low=$( awk '/^low =/{print $3;exit}' ${running_radiod_conf_file})     ### Assume that the first occurence of '^low' and '^high' is in the [WSPR] section
    local rx_audio_high=$( awk '/^high =/{print $3;exit}' ${running_radiod_conf_file})
    wd_logger 2 "In ${running_radiod_conf_file}: low = ${rx_audio_low}, high = ${rx_audio_high}"

    if [[ -z "${rx_audio_low}" || -z "${rx_audio_high}" ]]; then
        wd_logger 1 "ERROR: can't find the expected low and/or high settings in  ${running_radiod_conf_file}"
        return 1
    fi
    local rx_needs_restart="no"
    if [[ "${rx_audio_low}" != "1300" ]]; then
        wd_logger 1 "WARNING: found low = ${rx_audio_low}, so changing it to the desired value of 1300"
        sed -i "0, /^low =/{s/low = ${rx_audio_low}/low = 1300/}"  ${running_radiod_conf_file}      ### Only change the first 'low = ' line in the conf file
        rx_needs_restart="yes"
    fi
    if [[ "${rx_audio_high}" != "1700" ]]; then
        wd_logger 1 "WARNING: found high = ${rx_audio_high}, so changing it to the desired value of 1700"
        sed -i "0, /^high/{s/high = ${rx_audio_high}/high = 1700/}"  ${running_radiod_conf_file}
        rx_needs_restart="yes"
    fi
    if [[ ${rx_needs_restart} == "no" ]]; then
        wd_logger 2 "No changes needed"
    else
        wd_logger 1 "Restarting the radiod service"
        local radiod_service_name=${running_radiod_conf_file##*/}
        radiod_service_name=${radiod_service_name/.conf/.service}
        sudo systemctl restart ${radiod_service_name}
    fi
    return 0
}

function wd-9qstats() {
    local root_dir="/dev/shm/wsprdaemon/posting.d/MERG_6_Q"
    local band_dir_list=( $(find ${root_dir} -maxdepth 1 -type d -not -wholename ${root_dir}) )
    local band_list=( ${band_dir_list[@]##*/} )

    echo "Band  Total  Q Missed       SNR    Kiwi Missed       SNR  SNR Diff"
    for band_dir in ${band_list[@]}  ; do
        local merged_log_file
        merged_log_file=${root_dir}/${band_dir}/merged.log
        if [[ ! -f ${merged_log_file} ]]; then
            echo "${band_dir}: No    merged.log in ${root_dir}/${band_dir}"
        else
            local spots_info_list=( $(awk '! /FREQUENCY/{++spots_count};\
                                     $10 == "*" {++q_missed_count};\
                                     $11 == "*" {++kiwi_missed_count};\
                                     ! /FREQUENCY/ && $10 != "*" && $11 != "*" { q_spots_sum += $10; kiwi_spots_sum += $11; ++sum_count} ;\
                                     END {printf "%5d %5d %4.2f%% %f %5d  %4.2f%% %f   %f\n", \
                                                 spots_count,\
                                                 q_missed_count,  (q_missed_count/spots_count)*100, q_spots_sum /sum_count,\
                                                 kiwi_missed_count, (kiwi_missed_count/spots_count)*100, kiwi_spots_sum/sum_count,\ 
                                                (kiwi_spots_sum/sum_count) - (q_spots_sum /sum_count) }'  ${merged_log_file}) )
            #echo "spots_info_list[] = '${spots_info_list[*]}'"
            local spots_count=${spots_info_list[0]}
            local q_missed_count=${spots_info_list[1]}
            local q_missed_percent=${spots_info_list[2]}
            local q_average=${spots_info_list[3]}
            local kiwi_missed_count=${spots_info_list[4]}
            local kiwi_missed_percent=${spots_info_list[5]}
            local kiwi_average=${spots_info_list[6]}
            local spots_diff_average=${spots_info_list[7]}

            printf "%4d: %5d   %4d %6s %5.1f        %4d %6s %5.1f   %4.2f\n" \
                ${band_dir} ${spots_count} ${q_missed_count} ${q_missed_percent} ${q_average} ${kiwi_missed_count} ${kiwi_missed_percent} ${kiwi_average} ${spots_diff_average}
        fi
    done
}

function wd-radiod-control()
{
    local band=${1-20}
    local tuning_freq=${KA9Q_wspr_band_freq_list[${band}]}
    local mc_url

    local radiod_conf_base_name=$(wd-get_ka9q-conf_file-base_name)
    local radio_conf_file_name="/etc/radio/radiod@${radiod_conf_base_name}.conf"

    if [[ ! -f "${radio_conf_file_name}" ]]; then
        echo "Can't find the radiod configuration file '${radio_conf_file_name}'"
        return 1
    else
        local global_section_status_url="$(grep -i -A 20 '^\[.*global' "${radio_conf_file_name}" | grep "^status" | head -1 | cut -d ' ' -f 3)"
        if [[ -z "${global_section_status_url}" ]]; then
            echo "Can't find [global] status = URL in '${radio_conf_file_name}'"
            return 2
        else
            echo "Found [global] status = ${global_section_status_url} in '${radio_conf_file_name}'"
            control ${global_section_status_url} -s ${tuning_freq}
        fi
    fi
    return 0
}
alias wd9c=wd-radiod-control
alias wdrc=wd-radiod-control

function wd-rc-gain() {
    local root_dir="/dev/shm/wsprdaemon/recording.d/KA9Q_0"

    if [[ ! -d "${root_dir}" ]]; then
        echo "${root_dir} doesn't exist"
        return0
    fi
}
alias wdrg='wd-rc-gain'

function wd-9m()
{
    local header_printed="no"
    local band
    for band in $( IFS=$'\n'; echo "${!KA9Q_wspr_band_freq_list[*]}" | sort -n ); do
        local metadump_out=$(metadump -n -c 1 -s ${KA9Q_wspr_band_freq_list[${band}]} hf.local)
        local rf_gain=$(echo "${metadump_out}" | awk '/rf gain/{print $(NF - 1) }')
        local if_power=$(echo "${metadump_out}" | awk '/IF pwr/{print $(NF - 1) }')
        local noise_level=$(echo "${metadump_out}" | awk '/N0 /{print $(NF - 1) }')
        local agc_enable=$(echo "${metadump_out}" | awk '/agc /{print $(NF - 0) }')
        local baseband_pwr=$(echo "${metadump_out}" | awk '/baseband pwr/{print $(NF - 1) }')
        local output_level=$(echo "${metadump_out}" | awk '/output level/{print $(NF - 1) }')
        local headroom=$(echo "${metadump_out}" | awk '/headroom/{print $(NF - 1) }')
        local agc_gain=$(echo "${metadump_out}" | awk '/) gain/{print $(NF - 1) }')

        if [[ ${header_printed} == "no" ]]; then
            printf "RF Gain = %4.1f, AGC %s, AGC gain %4.1f\n" ${rf_gain} ${agc_enable} ${agc_gain}
            printf "Band  IF_power  Noise_level  Baseband_power Output_level  Headroom\n" 
            header_printed="yes"
        fi
        ##     band    if_power noise_level    baseband_pwr   output_level  headroom
        printf "%3d      %6.1f      %6.1f          %6.1f       %6.1f    %6.1f\n" ${band} "${if_power}" "${noise_level}" "${baseband_pwr}" "${output_level}" "${headroom}"
    done
}
alias wd9m=wd-9m

### tmux alias
function tm() {
    local tm_arg=${1-}
    local tm_list=( $(tmux ls 2> /dev/null | awk -F : '{print $1}') )
    local tm_list_count=${#tm_list[@]}

    if [[ -z "${tm_arg}" ]]; then
        if [[ ${tm_list_count} -eq 0 ]]; then
            tmux
            return 0
        fi
        if [[ ${tm_list_count} -eq 1 ]]; then
            tmux a -t ${tm_list[0]}
            return 0
        fi
    fi

    printf "There are ${tm_list_count} active tmux sessions:\n"
    readarray -t tm_list <  <(tmux ls)
    local i
    for (( i = 0; i < ${#tm_list[@]}; ++i )); do
        local tm_session="${tm_list[${i}]}"
        printf "#%s\n" "${tm_session}"
    done
    if [[ "${tm_arg}" == "-l" ]]; then
        return 0
    fi
    local tm_default_id=$(echo "${tm_list[0]}"  | awk -F : '{print $1}')
    read -p "Select tmux session id # (default = ${tm_default_id})? => "
    if [[ -z "${REPLY}" ]]; then
        REPLY="${tm_default_id}"
    fi
    for (( i = 0; i < ${#tm_list[@]}; ++i )); do
        local tm_session="${tm_list[${i}]}"
        local tm_session_id=$( echo "${tm_session}" | awk -F : '{print $1}' )
        if [[ "${tm_session_id}" == "${REPLY}" ]]; then
            tmux a -t ${tm_session_id} 
            return 0
        fi
    done
    echo "${REPLY} is not a valid id"
    return 0
}

function wd-wait-for_wspr-gap(){
   printf "Starting at %(%M-%S)T and waiting for the next gap in the WSPR-2 packets\n" -1
   while printf -v minute "%(%M)T\n" -1 && printf -v sec "%(%S)T\n" -1 && [[ $(( minute % 2 )) -ne 1 || 10#${sec} -lt 55 ]]; do 
       [[ ${verbosity-0} -gt 0 ]] &&printf '%(%M-%S)T\n' -1
       sleep 1; 
   done
}
alias wd-wait='wd-wait-for_wspr-gap'

alias wd-nf="awk '{printf \"%2d: %s\n\", NF, \$0}'"

### git aliases

### Change a ~/wsprdaemon from https:..(git read only) to git@github.com (git read/write) so you can check in WD source code changes
function  wd-git-to-ssh(){
    local current_url=$(git remote get-url origin)
    if ! [[ "${current_url}" =~ https ]]; then
        echo "Nothing to do since current_url='${current_url}' doesn't contain the string 'https'"
        return 0
    fi
    local repo_name=${current_url/*github.com\//}
    local new_repo_url="git@github.com:${repo_name}"
    read -p "Change git's url from '${current_url}' to '${new_repo_url}'? => "
    git config --global user.email "rob@robinett.us"
    git config --global user.name "Rob Robinett"
    git remote set-url origin ${new_repo_url}
    if ! [[ -d ~/.ssh ]]; then
        echo "There is no local ssh public key file.  Runnning 'ssh-keygen to generate one.  Press <CR> to accept all of its defaults:"
        ssh-keygen
    fi
    local public_key_file=$(find ~/.ssh -type f -name '*pub')
    if [[ -z "${public_key_file}" ]]; then
        echo "There is no local ssh public key file.  Runnning 'ssh-keygen to generate one.  Press <CR> to accept all of its defaults:"
        ssh-keygen
    fi
    public_key_file=$(find ~/.ssh -type f -name '*pub')
    if [[ -z "${public_key_file}" ]]; then
        echo "There is still no local ssh public key file, this has failed"
        return 1
    fi
    echo "Copy and paste this ssh public key to the github.com account:"
    echo "$(< ${public_key_file})"
    return 0
}

alias wd-get-my-public_ip='echo "$(curl ifconfig.me 2> /dev/null)"'

alias wd-watch-wavs >& /dev/null && unalias wd-watch-wavs
declare WSPR_BAND_TO_TUNING_FREQ=(   2200:136000  630:474200  160:1836600  80:3568600  80eu:3592600   60:5287200 60eu:5364700  40:7038600  30:10138700    22:13553900   20:14095600 \
                                     17:18104600 15:21094600  12:24924600 10:28124600    8:40680000   6:50293000   4:70091000 2:144489000 70:432300000  23:1296500000  29:560308000)
function wd-watch-wavs-scan() {
    local band
    for band in $(find -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort -rV); do
        (( ${verbosity-0} )) && read -p "Watch band ${band} => "
        (cd $band; wd-watch-wavs )
    done
}

function wd-watch-wavs() {
    local band_dir=${PWD##*/}
    local band_freq=""

    local band_info=""
    for band_info in ${WSPR_BAND_TO_TUNING_FREQ[@]}; do
        local band_info_list=(${band_info/:/ } )
        local band_meters=${band_info_list[0]}
        local band_tuning_freq=${band_info_list[1]}
        if [[ ${band_meters} == ${band_dir} ]]; then
            band_freq=${band_tuning_freq}
            break
        fi
    done
    if [[ -z "${band_freq}" ]]; then
        echo "Can't find a band which matches this PWD=${PWD}"
        return 0
    fi

    watch "ls -lt *.wav ../*Z_${band_freq}*"
}
alias wdww='wd-watch-wavs'

alias wdpss='ps aux | grep "wd-record\|pcmrecord" | grep -v grep'

function wd-ssh-to-wdclient() {
    local rac_id=${1:-usage}

    if [[ "${rac_id}" == "usage" ]]; then
        echo "usage:   wd-ssh-to-wdclient (or wdssr) RAD_ID"
        return 0
    fi
    if [[ ${#rac_id} -eq 1 ]]; then
        rac_id="0${rac}"
    fi
    ssh -p 358${rac_id} wsprdaemon@wsprdaemon.org
}
alias wdssr=wd-ssh-to-wdclient

###  Reloads these functions and aliases into the users running bash (must be an alias)
alias wd-bash_aliases='source ~/wsprdaemon/bash-aliases'
alias wdba='wd-bash_aliases'

function wd-help()
{
    echo "Programs which print information about Wsprdaemon (WD) system:
Command       Can also be executed by    Description
wd-help       wd-h wd-                   Prints this help message
wd-start      wd-a wda                   Start or keep running WD
wd-stop       wd-z wdz                   Stop WD if it is running
wd-status     wd-s wds                   Show the status of a running WD 
wd-journal    wdj                        Follow the stdout/stderr of wsprdaemon.service (journalctl -u wsprdaemon -f)
wd-decode-health wdb                    Show bands whose decodes have fallen permanently behind, and any cycles lost to a decoder timeout or a purge
wd-time       wdt                        Show whether the clock is synchronised (chronyc tracking + sources)
wd-usb        wdu                        Show the RX888s on the USB bus, their hub ports and which WD can power cycle
wd-usb-cycle  wd -U SERIAL|HUB:PORT|all  Power cycle an RX888 / hub port through uhubctl
wd-upload-wn  wd-un wdln                 Watch the 'upload to wsprnet' log
wd-upload-wd  wd-uw wdlw                 Watch the 'upload to wsprdaemon' log
wd-errors     wd-e wdle                  Each 10 seconds, search all the log files and print newly logged 'ERROR' lines
wd-overloads  wd-ov wdov                 Prints all overload events recorded in the wav and ADC log files 
wd-query      wd-q wdq                   Query the wsprdaemon database for spots
wd-gc                                    Execute git commit
wd-gp                                    Execute git push
wq-gl                                    Execute git log
wd-gc-ssh                                Execute git remote set-ur + git config
"
}
alias wd-='wd-help'
alias wd-h='wd-help'

if [[ ! -f ~/.tmux.conf ]]; then
    echo "set -g mouse on" > ~/.tmux.conf
fi

alias wdg='wd -g'

###  Runs du and sorts output by name rather than ???   Useful in the ~/wsprdaemon/wav-archive directory
function wddu() {
    local files="$@"

    du -s ${files}  | awk '{print $2 ":  " $1}'
}

### Copy a 2 minute wav file to /tmp
function wd-wav-capture() {
    local wav_file
    echo -n "$(date): Waiting for the 2 minute long wav file to appear after the next even 2 minutes: "
    while ! wav_file=$(ls 2?????_*.wav 2> /dev/null ); do 
         echo -n "."
         sleep .25
     done
     cp ${wav_file} /tmp/
     echo 
     echo "$(date): Captured /tmp/${wav_file}"
}

function wd-ft-logs() {
   for d in 4 8; do
       local ft_log_file_name="/mnt/ka9q-radio/ft${d}/ft${d}.log"
       echo "FT${d}: ${ft_log_file_name} size if $(stat --format=%s ${ft_log_file_name}) bytes"
       local ft_log_archived_file_names="$( find ${ft_log_file_name%/*} -type f -name ${ft_log_file_name##*/}.* )"
       if [[ -n "${ft_log_archived_file_names}" ]]; then
           echo -n "Archives: "; echo ${ft_log_archived_file_names}
       fi
       head -4  ${ft_log_file_name}
       echo "++++++++++++++"
       tail -4  ${ft_log_file_name}
       echo 
   done
   df /mnt/ka9q-radio | grep "^tmp"
   if [[ -n "${1-}" ]]; then
       if [[ ! -f /mnt/ka9q-radio/ft4/ft4.log ]]; then
           echo "There is no /mnt/ka9q-radio/ft4/ft4.log"
       else
           read -p "$(date): Run 'tail -f /mnt/ka9q-radio/ft4/ft4.log'? => "
           tail -f "/mnt/ka9q-radio/ft4/ft4.log"
       fi
   fi
}

### On MacOS, disable the caps lock key on a Mac keyboard
function wd-no-capslock() {
   hidutil property --set '{"UserKeyMapping":[{"HIDKeyboardModifierMappingSrc":0x700000039,"HIDKeyboardModifierMappingDst":0}]}'
}

alias cdb='cd ~/wsprdaemon/ka9q-web'
alias cdu='cd /srv/wd_data/uploads/wsprdaemon'
alias r='reset'