add a watchdog to keep pifmadv and ffmpeg alive

music-api.py

@@ -4,6 +4,7 @@ import time
 from datetime import datetime, timezone
 import random
 import threading
+import psutil
 import subprocess
 import hashlib
 import sqlite3
@@ -389,6 +390,47 @@ def _pump_decoder_to_wrapper(decoder_stdout, stop_event):
 def write_silence_ms(ms: int = SILENCE_MS_AFTER_TRACK):
     write_silence(ms / 1000.0)
 
+def is_zombie(proc):
+    try:
+        return proc.status() == psutil.STATUS_ZOMBIE
+    except psutil.NoSuchProcess:
+        return True
+
+def restart_radio_chain():
+    print("[WATCHDOG] Restarting radio chain")
+
+    # Kill ffmpeg
+    for proc in psutil.process_iter(['name', 'cmdline']):
+        try:
+            cmd = proc.info['cmdline'] or []
+            if proc.info['name'] == 'ffmpeg' or (cmd and 'ffmpeg' in cmd[0]):
+                print(f"[WATCHDOG] Killing ffmpeg PID {proc.pid}")
+                proc.kill()
+        except Exception:
+            pass
+
+    # Kill pifmadv
+    for proc in psutil.process_iter(['name', 'cmdline']):
+        try:
+            cmd = proc.info['cmdline'] or []
+            if proc.info['name'] == 'pifmadv' or (cmd and 'pifmadv' in cmd[0]):
+                print(f"[WATCHDOG] Killing pifmadv PID {proc.pid}")
+                proc.kill()
+        except Exception:
+            pass
+
+    time.sleep(0.5)
+
+    print("[WATCHDOG] Restarting pifmadv")
+    start_pifm()
+
+    # Your addition: broadcast station ID
+    id_file = pick_station_id_file()
+    if id_file:
+        print(f"[WATCHDOG] Broadcasting recovery ID: {id_file}")
+        stream_file(id_file, allow_skip=False)
+        write_silence(0.25)
+
 # -----------------------------
 # Robust stream_file implementation
 # -----------------------------
@@ -409,7 +451,10 @@ def stream_file(path: str, allow_skip=True) -> bool:
     decoder_args = [
         "ffmpeg", "-nostdin", "-v", "error",
         "-i", path,
-        "-f", "s16le", "-ar", str(SAMPLE_RATE), "-ac", str(CHANNELS),
+        "-f", "s16le", # raw PCM
+        "-af", "acompressor=threshold=-18dB:ratio=4:attack=5:release=100,alimiter=limit=0.0dB", # compress just like a real radio station, because this is a real radio station
+        "-ar", str(SAMPLE_RATE),
+        "-ac", str(CHANNELS),
         "pipe:1"
     ]
 
@@ -676,6 +721,42 @@ def worker():
             print("[WORKER] Exception in worker loop:", e)
             time.sleep(1)
 
+restart_lock = threading.Lock()
+
+def watchdog():
+    while True:
+        time.sleep(5)
+
+        pifm_state = "missing"
+        ffmpeg_state = "missing"
+
+        for proc in psutil.process_iter(['name', 'cmdline', 'status']):
+            try:
+                cmd = proc.info['cmdline'] or []
+                name = proc.info['name']
+
+                if name == "pifmadv" or (cmd and "pifmadv" in cmd[0]):
+                    if proc.status() == psutil.STATUS_ZOMBIE:
+                        pifm_state = "zombie"
+                    else:
+                        pifm_state = "ok"
+
+                if name == "ffmpeg" or (cmd and "ffmpeg" in cmd[0]):
+                    # Only count ffmpeg processes that belong to your pipeline
+                    if any(x in cmd for x in ["-f", "wav", "-i", "-"]):
+                        if proc.status() == psutil.STATUS_ZOMBIE:
+                            ffmpeg_state = "zombie"
+                        else:
+                            ffmpeg_state = "ok"
+
+            except Exception:
+                continue
+
+        if pifm_state != "ok" or ffmpeg_state != "ok":
+            with restart_lock:
+                print(f"[WATCHDOG] pifmadv={pifm_state}, ffmpeg={ffmpeg_state}")
+                restart_radio_chain()
+
 # -----------------------------
 # API Endpoints
 # -----------------------------
@@ -767,7 +848,16 @@ if __name__ == "__main__":
 
     discover_tracks()
     start_pifm()
+
+    # play station ID at startup so the listener(s) know we're up
+    id_file = pick_station_id_file()
+    if id_file:
+        set_rds("RT HushPupPi Station ID")
+        stream_file(id_file, allow_skip=False)
+        write_silence(0.25)
+
     threading.Thread(target=worker, daemon=True).start()
+    threading.Thread(target=watchdog, daemon=True).start()
     # Reduce Flask logging noise in production
     import logging
     logging.getLogger("werkzeug").setLevel(logging.ERROR)