wsprdaemon.sh

#!/bin/bash
### The previous line signals to the vim editor that it should use its 'bash' editing mode when editing this file

###  Wsprdaemon:   A robust  decoding and reporting system for  WSPR 

###    Copyright (C) 2020-2024  Robert S. Robinett
###
###    This program is free software: you can redistribute it and/or modify
###    it under the terms of the GNU General Public License as published by
###    the Free Software Foundation, either version 3 of the License, or
###    (at your option) any later version.
###
###    This program is distributed in the hope that it will be useful,
###    but WITHOUT ANY WARRANTY; without even the implied warranty of
###    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
###    GNU General Public License for more details.
###
###    You should have received a copy of the GNU General Public License
###    along with this program.  If not, see <https://www.gnu.org/licenses/>.

[[ ${v:-no} == "yes" ]] && echo "wsprdaemon.sh Copyright (C) 2020-2024  Robert S. Robinett
This program comes with ABSOLUTELY NO WARRANTY; for details type './wsprdaemon.sh -h'
This is free software, and you are welcome to redistribute it under certain conditions.  execute'./wsprdaemon.sh -h' for details.
wsprdaemon depends heavily upon the 'wsprd' program and other technologies developed by Joe Taylor K1JT and others, to whom we are grateful.
Goto https://physics.princeton.edu/pulsar/K1JT/wsjtx.html to learn more about WSJT-x
"

### This bash script logs WSPR spots from one or more Kiwi
### It differs from the autowspr mode built in to the Kiwi by:
### 1) Processing the uncompressed audio .wav file through the 'wsprd' utility program supplied as part of the WSJT-x distribution
###    The latest 'wsprd' includes algorithmic improvements over the version included in the Kiwi
### 2) Executing 'wsprd -d', a deep search mode which sometimes detects 10% more signals in the .wav file
### 3) By executing on a more powerful CPU than the single core ARM in the Beaglebone, many more signals are extracted on busy WSPR bands,'
###    e.g. 20M during daylight hours
###
###  This script depends extensively upon the 'kiwirecorder.py' utility developed by John Seamons, the Kiwi author
###  I owe him much thanks for his encouragement and support 
###  Feel free to email me with questions or problems at:  rob@robinett.us
###  This script was originally developed on Mac OSX, but this version 0.1 has been tested only on the Raspberry Pi 3b+
###  On the 3b+ I am easily running 6 simultaneous WSPR decode sessions and expect to be able to run 12 sessions covering all the 
###  LF/MF/HF WSPR bands on one Pi
###
###  Rob Robinett AI6VN   rob@robinett.us    July 1, 2018
###
###  This software is provided for free but with no guarantees of its usefullness, functionality or reliability
###  You are free to make and distribute copies and modifications as long as you include this disclaimer
###  I welcome feedback about its performance and functionality

shopt -s -o nounset          ### bash stops with error if undeclared variable is referenced

#declare VERSION=3.0.9                ### Refine search for WSPR packet wav files
#declare VERSION=3.1.0               ### Add support for KA9Q-radio and RX888
#declare VERSION=3.1.1               ### Add support for KA9Q-radio and RX888
#declare VERSION=3.1.2                 ### Add support for fixed AGC level to KA9Q receivers
                                       ### Add support for WSPR ansd WWV IQ file recording into compressed files in a series of tar archives
#declare VERSION=3.1.3                 ### Add support for WSPR-2 spectral spreading reports 
#declare VERSION=3.1.4                 ### Add support for the GRAPE system
#declare VERSION=3.1.5                 ### Revert to having Kiwi and RX888 do narrow audio filtering, not sox
#declare VERSION=3.1.6                 ### Install 4/19/24 KA9Q-radio
#declare VERSION=3.1.7                 ### Fix to handle changes in respones by wsprnet.org to uploads with duplicates
#declare VERSION=3.2.0                 ### Add all weprd and jt9 binaries.  Installs on Pi 5
#declare VERSION=3.2.1                 ### Increment version number so wspr.rocks will report which WD sites have been upgraded
#declare VERSION=3.2.2                 ### Adds ka9q-web and FT4/8 reporting 
#declare VERSION=3.2.3                 ### Merge 3.2.0 branch into master, a second time

if [[ $USER == "root" ]]; then
    echo "ERROR: This command '$0' should NOT be run as user 'root' or non-root users will experience file permissions problems"
    exit 1
fi

declare SUDOUERS_FILE="/etc/sudoers.d/${USER}"
if [[ -f ${SUDOUERS_FILE} ]]; then
    (( ${verbosity-0} )) && echo "User ${USER} is already a member of the 'sudo' group"
else
    if echo "${USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/${USER} > /dev/null; then
        echo "Updated user '${USER}' to auto-sudo"
    else
        echo "User ${USER} need to be a member of the 'sudo' group.  This change requires that you enter the 'root' user password:"
        if ! su -c "/usr/sbin/usermod -aG sudo ${USER}; echo '${USER} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/${USER}"; then
            echo  "ERROR: failed to set up this user ${USER} for sudo access"
            exit 1
        fi
        echo "${USER} now has sudo access"
    fi
fi

### Dont wrap 'bc' since it is used in ways incompatible with being a bash function
function wrap_bc() {
    local lineno=${BASH_LINENO[0]}
    local src=${BASH_SOURCE[1]}
    local output
    local errfile
    errfile=$(mktemp)

    # Run the real bc, capturing stderr to a temp file
    output=$(/usr/bin/bc "$@" 2>"$errfile")
    local status=$?

    if [[ -s "$errfile" ]]; then
        printf 'bc error at %s:%d\n' "$src" "$lineno" >&2
        cat "$errfile" >&2
    fi

    rm -f "$errfile"
    return $status
}   
### These need to be defined first
declare -r WSPRDAEMON_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
declare -r VERSION="$(cd ${WSPRDAEMON_ROOT_DIR}; echo "$(< wd_version.txt)-$(git rev-list --count HEAD)")"

declare -r RUNNING_IN_DIR=${PWD}        ### Used by the '-d' and '-D' commands so they know where to look for *.pid files
################# Check that our recordings go to a tmpfs (i.e. RAM disk) file system ################
declare WSPRDAEMON_TMP_DIR=/dev/shm/wsprdaemon
mkdir -p /dev/shm/wsprdaemon
if [[ -n "${WSPRDAEMON_TMP_DIR-}" && -d ${WSPRDAEMON_TMP_DIR} ]] ; then
    true
    ### The user has configured a TMP dir
    #wd_logger 2 "Using user configured TMP dir ${WSPRDAEMON_TMP_DIR}"
elif df /tmp/wspr-captures > /dev/null 2>&1; then
    ### Legacy name for /tmp file system.  Leave it alone
    WSPRDAEMON_TMP_DIR=/tmp/wspr-captures
elif df /tmp/wsprdaemon > /dev/null 2>&1; then
    WSPRDAEMON_TMP_DIR=/tmp/wsprdaemon
fi

cd ${WSPRDAEMON_ROOT_DIR}

### '-X [identify]': which RX888 serial each radiod@ opens, and walking the operator through confirming them by unplugging
### one radio at a time.  It runs with WD stopped, so it must not set anything up or re-pin anything first.
if [[ ${1-} == "-X" ]] && (( $# <= 2 )); then      ### outside the one-argument fast paths below: it takes an optional word
    set +o nounset
    source ${WSPRDAEMON_ROOT_DIR}/wsprdaemon.conf 2>/dev/null       ### RECEIVER_LIST, and any KA9Q_RADIOD_CONF_DIR override
    function wd_logger() { (( ${1:-2} <= 1 )) && echo -e "${2:-}" ; return 0 ; }
    source ${WSPRDAEMON_ROOT_DIR}/wd-usb-power.sh
    source ${WSPRDAEMON_ROOT_DIR}/wd-rx888-assign.sh
    LC_ALL=C wd_rx888_assign_cmd "${2:-}"
    exit $?
fi

### Read-only status reports, answered before the sourcing below and then gone.
### Each of these prints from one file and needs nothing else, but they used to take ~60 s because
### sourcing ka9q-utils.sh runs its bare 'ka9q-setup' AT SOURCE TIME, which calls wd_cpu_tuning ->
### wd-cpu-apply.sh (twice) and restarts wd-resctrl, wd-irq-affinity and wd-cpu-freq.  A bash -x
### profile of 'wd -b' on ON5KQ put 41.3 s of 64 s in wd-cpu-tuning.sh.  Asking a host how its
### decoders, its clock or its USB bus are doing must not re-apply its CPU plan and bounce those
### units first, so answer here and exit.
### Only when the flag is the whole command line: anything else falls through and is handled by the
### getopts loop below exactly as before, so 'wd -b' keeps working in every combination.
if (( $# == 1 )); then
    ### '-V' prints one string that is already in hand: VERSION is assigned at the top of this file and
    ### needs neither a site's conf nor any of the sourcing below.  It was left to the getopts loop, so
    ### it paid the full ~64 s anyway -- measured at ON5KQ 2026-09-18, where a bare 'wdv' also re-pinned
    ### both running radiods and moved 64 processes between cores.  An operator who asks WD its version
    ### and watches nothing happen for a minute concludes the machine is wedged and power-cycles it,
    ### which is exactly what that site was doing to itself.  Same text as the 'V)' case below.
    if [[ $1 == "-V" ]]; then
        echo "Version = ${VERSION}"
        exit 0
    fi
    ### '-c' applies the clock ceilings in wsprdaemon.conf, and touches nothing else.
    ### Editing WD_CPU_FREQ_RADIOD_MHZ/_OTHER_MHZ does nothing on its own: the value has to be written
    ### into ${WD_CPU_PLAN_CONF} and wd-cpu-freq restarted before a core's scaling_max_freq moves.
    ### Until the '-V' fast path above, EVERY wd command re-ran the whole CPU tuning at source time, so
    ### 'wdv' applied clock changes by accident and at least one operator's workflow depended on it
    ### (ON5KQ, 2026-09-29: "after changing the wsprdaemon.conf file just the command wdv doesn't change
    ### anything").  His fallback was 'wdz' then 'wda', a full stop and start of WD and radiod, which on
    ### that host means re-seating RX888 cables by hand.  Changing a clock ceiling should never cost
    ### anyone a radio, so give it its own command: rewrite the plan file, restart the one unit that
    ### reads it, and leave radiod and the decoders running.
    if [[ $1 == "-c" ]]; then
        set +o nounset
        source ${WSPRDAEMON_ROOT_DIR}/wsprdaemon.conf 2>/dev/null
        ### wd-cpu-tuning.sh logs through wd_logger(), which lives in wd-utils.sh -- and sourcing THAT
        ### file turns 'nounset' back on and installs an ERR trap, which is precisely what this fast path
        ### switched off.  Supply the one function it needs instead of dragging in the rest: level 1 is
        ### what an operator should see (wd_cpu_tuning_log demotes everything that is not an ERROR or a
        ### WARNING to level 2), and the file log is written by wd_cpu_tuning_log itself either way.
        function wd_logger() { (( ${1:-2} <= 1 )) && echo -e "${2:-}" ; return 0 ; }
        source ${WSPRDAEMON_ROOT_DIR}/wd-cpu-tuning.sh
        if [[ "${WD_CPU_TUNING}" != "yes" ]]; then
            echo "WD_CPU_TUNING is not \"yes\", so this host's clocks come from CPU_CORE_KHZ in wsprdaemon.conf rather than from WD.  Nothing applied."
            exit 0
        fi
        wd_cpu_tuning_write_plan_conf
        if ! sudo systemctl restart wd-cpu-freq ; then
            echo "ERROR: wd-cpu-freq failed to restart; the old ceilings are still in force"
            sudo systemctl status wd-cpu-freq --no-pager 2>&1 | tail -5
            exit 1
        fi
        echo "Applied the clock policy from wsprdaemon.conf.  radiod and the decoders were left running."
        echo "Ceilings now in force:"
        for wd_freq_file in /sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_max_freq ; do
            [[ -r ${wd_freq_file} ]] || continue
            wd_freq_cpu=${wd_freq_file#/sys/devices/system/cpu/cpu}
            printf '%s %s\n' "${wd_freq_cpu%%/*}" "$(( $(< ${wd_freq_file}) / 1000 ))"
        done | sort -n | awk '{grp[$2]=grp[$2]","$1} END {for (mhz in grp) {sub(/^,/,"",grp[mhz]); printf "  %5s MHz : cpus %s\n", mhz, grp[mhz]}}'
        echo "A change made only here is temporary in the useful sense: the next WD start rewrites it from wsprdaemon.conf."
        exit 0
    fi
    declare -A WD_STATUS_REPORTS=(
        ["-b"]="wd-decode-health.sh wd_decode_health_show"
        ["-t"]="wd-time-sync.sh wd_time_sync_show"
        ["-u"]="wd-usb-power.sh wd_usb_power_show"
    )
    ### '-R' re-trains an RX888 that came up on the USB 2 half of its USB 3 jack.  An operator reaches for
    ### this when a radio is already missing, so it must not first spend ~49 s re-applying the CPU plan --
    ### and re-pinning the very radiod it is about to stop.  Same stub-wd_logger trick as '-c' above.
    if [[ $1 == "-R" ]]; then
        set +o nounset
        source ${WSPRDAEMON_ROOT_DIR}/wsprdaemon.conf 2>/dev/null
        function wd_logger() { (( ${1:-2} <= 1 )) && echo -e "${2:-}" ; return 0 ; }
        source ${WSPRDAEMON_ROOT_DIR}/wd-usb-power.sh
        LC_ALL=C wd_usb_power_retrain_cmd
        exit $?
    fi
    if [[ -n "${WD_STATUS_REPORTS[$1]-}" ]]; then
        declare wd_report_file wd_report_func
        read -r wd_report_file wd_report_func <<< "${WD_STATUS_REPORTS[$1]}"
        ### These reports, and a site's wsprdaemon.conf, are written to be read by a normal shell.
        ### Only the full load below declares everything 'nounset' needs, so drop it for this path.
        set +o nounset
        source ${WSPRDAEMON_ROOT_DIR}/wsprdaemon.conf 2>/dev/null   ### only for a site which overrides a log path or KA9Q_RADIOD_CONF_DIR
        source ${WSPRDAEMON_ROOT_DIR}/${wd_report_file}
        LC_ALL=C ${wd_report_func}      ### LC_ALL=C or sorted columns and globs collate differently per locale
        exit $?
    fi
fi

### Stopping WD must not re-apply the CPU plan first.  'ExecStop=wsprdaemon.sh -Z' is not one of the
### fast paths above, so it falls through to the sourcing below, where the bare 'ka9q-setup' in
### ka9q-utils.sh calls wd_cpu_tuning AT SOURCE TIME.  Measured at ON5KQ 2026-10-02: a 'wdz' spent
### 05:25:46 -> 05:26:28 enabling the three tuning units, running wd-cpu-apply.sh and re-pinning BOTH
### still-running radiods before it killed a single daemon, then overran the unit's 90 s
### TimeoutStopSec and was SIGKILLed -- the same on Sep 29 (x2), Sep 30 and Oct 1.  Re-tuning a host
### that is shutting down is pointless, and it perturbs the very radiods -Z is about to stop.  The
### operator sees it as 'wdz takes 2-3 minutes'.  wd_cpu_tuning() returns early when this is set.
[[ " $* " == *" -Z "* ]] && declare -x WD_SKIP_CPU_TUNING="yes"
### Nor may it set anything else up.  The sourcing below also runs ka9q-utils.sh's ka9q-setup, which rewrites radiod's conf
### and restarts radiod when it changed and (re)starts the FT services, and wd-setup.sh's start of an ubersdr build.  At
### W7WKR-1 on 2026-10-19 a 'git pull' then 'wdz' did both inside the stop: the ubersdr build started, radiod was
### restarted at 11:10:17 PDT and its RX888 delivered no data from then on (radiod "No rx888 data for 5 seconds" and a
### libusb abort, every 30 s), and the FT decoders were running again behind the stop.  WD_STOPPING makes both skip.
if [[ " $* " == *" -Z "* || " $* " == *" -z "* ]]; then
    declare -x WD_STOPPING="yes" WD_SKIP_CPU_TUNING="yes"
fi

source ${WSPRDAEMON_ROOT_DIR}/bash-aliases       ### Set up WD aliases for all users
source ${WSPRDAEMON_ROOT_DIR}/wd-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/config-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/wd-setup.sh
source ${WSPRDAEMON_ROOT_DIR}/wd-cpu-tuning.sh   ### MUST precede ka9q-utils.sh: that file runs
                                                  ### ka9q-setup at source time, which calls wd_cpu_tuning
source ${WSPRDAEMON_ROOT_DIR}/ka9q-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/noise-graphing.sh
source ${WSPRDAEMON_ROOT_DIR}/atsc.sh
source ${WSPRDAEMON_ROOT_DIR}/ppm.sh
source ${WSPRDAEMON_ROOT_DIR}/kiwi-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/recording.sh
source ${WSPRDAEMON_ROOT_DIR}/decoding.sh
source ${WSPRDAEMON_ROOT_DIR}/posting.sh
source ${WSPRDAEMON_ROOT_DIR}/upload-client-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/job-management.sh
source ${WSPRDAEMON_ROOT_DIR}/usage.sh
source ${WSPRDAEMON_ROOT_DIR}/noise-graphs-daemon.sh
source ${WSPRDAEMON_ROOT_DIR}/wav-archive.sh
source ${WSPRDAEMON_ROOT_DIR}/grape-utils.sh
source ${WSPRDAEMON_ROOT_DIR}/wd-drops.sh
source ${WSPRDAEMON_ROOT_DIR}/wd-ka9q-web-guard.sh   ### Restarts a ka9q-web left spinning by an abandoned browser session
source ${WSPRDAEMON_ROOT_DIR}/wd-decode-backlog.sh   ### FT8/FT4/WSPR decode backlog monitor
source ${WSPRDAEMON_ROOT_DIR}/wd-decode-health.sh    ### Records every decode which didn't finish inside its cycle
source ${WSPRDAEMON_ROOT_DIR}/watchdog.sh         ### Should come last

[[ -z "$*" ]] && usage

while getopts :aAbczZsg:hij:l:pvVw:dDr:RtuU: opt ; do
    case $opt in
        l)
            log_file_viewing  $OPTARG
            ;;
        r)
            spawn_wav_recording ${OPTARG//,/ }
            ;;
        A)
            ### '-A' is what systemd runs at boot and on 'systemctl start', but the 900 s start limit was only ever written by
            ### '-a' (start_systemctl_daemon), so a host never started by hand kept systemd's 90 s.  K6FOD 2026-10-17: the first
            ### start after a reboot was still setting up the ka9q FT services at 90 s and was killed ("Failed to start
            ### wsprdaemon.service"); systemd's restart then succeeded in 53 s.  Writing it here covers every later start.
            wd_ensure_service_start_timeout
            if [[ ${WD_STARTUP_DELAY_SECS-0} -gt 0 ]]; then
                echo "Wsprdaemon is delaying startup for ${WD_STARTUP_DELAY_SECS} seconds"
                wd_sleep ${WD_STARTUP_DELAY_SECS}
            fi
            wd_cleanup_setup
            watchdog_cmd a
            ;;
        a)
            start_systemctl_daemon
            ;;
        Z)
            wd_kill_all    ## silently kill everything
            ;;
        z)
            stop_systemctl_daemon
            ;;
        s)
            jobs_cmd     s
            watchdog_cmd s
            ;;
        i)
            list_devices 
            ;;
        b)
            wd_decode_health_show
            ;;
        t)
            wd_time_sync_show
            ;;
        u)
            wd_usb_power_show
            ;;
        U)
            wd_usb_power_cycle_cmd $OPTARG
            ;;
        R)
            wd_usb_power_retrain_cmd
            ;;
        w)
            watchdog_cmd $OPTARG
            ;;
        j)
            jobs_cmd $OPTARG
            ;;
        p)
            plot_noise
            ;;
        h)
            usage
            ;;
        g)
            grape_menu -$OPTARG
            echo "Finished running 'grape_menu -$OPTARG' => $?"
            ;;
        v)
            ((verbosity++))
            [[ $verbosity -ge 4 ]] && echo "Verbosity = ${verbosity}"
            ;;
        V)
            echo "Version = ${VERSION}"
            ;;
        c)
            ### Handled above, before any sourcing, but only when '-c' is the whole command line.  Reaching
            ### here means it was combined with something else, and by now the source-time CPU tuning has
            ### already run -- so say what happened rather than silently doing nothing.
            echo "'-c' applies the clock policy and must be given on its own: run 'wd -c'." 1>&2
            ;;
        d)
            increment_verbosity
            ;;
        D)
            decrement_verbosity
            ;;
        \?)
            echo "Invalid option: -$OPTARG" 1>&2
            ;;
        : )
            echo "Invalid option: -$OPTARG requires an argument" 1>&2
            ;;
    esac
done