#!/usr/bin/env python3 """Streaming receiver: decode chirpmodem frames as the samples arrive. Live from an RTL-SDR (runs rtl_sdr itself, or reads its output on stdin), or a capture file replayed as fast as the receiver can go, which measures how much faster than real time it is: ./stream_rx.py --rtl --sf 12 --gain 49.6 # live, 223.580 MHz rtl_sdr -f 223330000 -s 1000000 -g 40 - | ./stream_rx.py - --format cu8 --fs 1e6 --lo 223.33e6 ./stream_rx.py ota.cu8 --format cu8 --fs 2.4e6 --lo 223.33e6 --sf 12 # replay How it works, block by block (0.1 s of samples at a time): 1. rate conversion (shift the signal to 0 Hz, FIR decimate, fractional resample) carrying filter state across blocks: iq_io.StreamConverter 2. decimate to the chip rate and run the preamble detector on the new windows only; a candidate is accepted once the windows on both sides of it are in (so the strongest window of each preamble wins) 3. each candidate waits until its header has arrived, is decoded, then waits for the rest of the frame (length from the header) and is decoded in full; old samples are dropped once no candidate needs them Effort (--effort 0..2, see frame.EFFORT_MAX) caps the retries per frame. With --auto-effort (default for live input) the receiver lowers it when decoding takes more than 70% of real time and raises it again under 30%. Every --status seconds (live) it prints the load: processing time divided by stream time, split into front end (steps 1-2) and decoding (step 3). """ import argparse, os, queue, signal, subprocess, sys, threading, time from collections import deque import numpy as np from phy import PhyParams, payload_layout from frame import receive_frame, frame_chips, header_codec, EFFORT_MAX, MODE_COHERENT import iq_io import rx from rx_file import frame_report, level_check # Symbols of slack in 'candidate inside the last decoded frame': a frame can # start exactly where the previous one ended, whose last pilot is an up-chirp # too and can pull the detector's peak a window early. BUSY_SLACK = 8 class StreamReceiver: """Feed modem-rate samples (p.fs) with push(); frames come out via ``out``.""" def __init__(self, p, freq, max_bytes=255, effort=EFFORT_MAX, auto_effort=False, excise=True, det_thresh=4.0, verbose=False, out=print, t_offset=0.0, clock=None): self.p, self.freq = p, freq self.M, self.osr = p.M, p.osr self.L = p.n_preamble - 1 self.effort, self.max_effort, self.auto = effort, effort, auto_effort self.excise, self.det_thresh, self.verbose = excise, det_thresh, verbose self.say, self.t_offset, self.clock = out, t_offset, clock self.max_chips = frame_chips(p, max_bytes, coherent=False) n_hdr = header_codec(p).n_sym self.hdr_chips = p.payload_start + len(payload_layout(n_hdr, p.pilot_every)[0]) * p.M self.dec_margin = 16 # chips of look-ahead the decimator needs self.x = np.zeros(0, complex) # modem-rate samples, x[0] = chip x0 self.x0 = 0 # always a multiple of M self.r = np.zeros(0, complex) # chip rate, r[0] = chip x0 self.k_next = 0 # first window whose metric isn't final yet self.metric = {} # window -> metric (recent only) self.Eu = {} # window -> zero-padded dechirped power self.cu = np.conj(rx.base_chirp_chips(p.M)) self.pending = [] # candidates waiting for samples self.busy_until = 0 # chip where the last decoded frame ended self.frames, self.n_fail, self.n_false = [], 0, 0 self.t_front = self.t_dec = 0.0 self.load_hist = deque(maxlen=50) # (stream s, decode s) per block # --------------------------------------------------------------- input @property def chips_in(self): return self.x0 + len(self.x) // self.osr def push(self, y): """Add modem-rate samples; runs detection and any decodes now possible.""" t0 = time.perf_counter() self.x = np.concatenate([self.x, y]) self._decimate() self._detect() t1 = time.perf_counter() self._service() t2 = time.perf_counter() self._trim() self.t_front += t1 - t0 self.t_dec += t2 - t1 self._adapt(len(y) / self.p.fs, t2 - t0) def _decimate(self): n_have = self.x0 + len(self.r) n_can = self.chips_in - self.dec_margin if n_can <= n_have: return lo = max(0, (n_have - self.x0) * self.osr - self.dec_margin * self.osr) hi = min(len(self.x), (n_can - self.x0) * self.osr + self.dec_margin * self.osr) pos = (np.arange(n_have, n_can) - self.x0) * float(self.osr) - lo self.r = np.concatenate([self.r, rx.resample_at(self.x[lo:hi], pos, cutoff=1.0 / self.osr)]) # --------------------------------------------------------------- detection def _detect(self): M, L, n_pre = self.M, self.L, self.p.n_preamble nw = (self.x0 + len(self.r)) // M # complete windows (absolute) k_last = nw - L # metric needs windows k..k+L-1 k_final = k_last - n_pre # neighbourhood complete up to here if k_final < self.k_next: return # each window's spectrum is computed once (rx.preamble_metric's # consistency metric, evaluated incrementally) w0 = max(self.x0 // M, self.k_next - n_pre) # include left neighbours new = [w for w in range(w0, nw) if w not in self.Eu] if new: W = np.stack([self.r[w * M - self.x0:(w + 1) * M - self.x0] for w in new]) E = np.abs(np.fft.fft(W * self.cu, n=rx.P_DET * M, axis=1)) ** 2 for w, e in zip(new, E): self.Eu[w] = e for k in range(w0, k_last + 1): if k in self.metric: continue rows = np.stack([self.Eu[k + i] for i in range(L)]) B = int(np.argmax(rows.sum(axis=0))) ratio = rows[:, B] / rows.mean(axis=1) self.metric[k] = float(np.exp(np.mean(np.log(ratio + 1e-12)))) for k in range(self.k_next, k_final + 1): v = self.metric.get(k, 0.0) if v < self.det_thresh: continue nb = [self.metric.get(j, 0.0) for j in range(k - n_pre, k + n_pre + 1)] if v < max(nb) or any(self.metric.get(j, 0.0) == v for j in range(k - n_pre, k)): continue # not the local maximum (first of ties) c0 = (k - 3) * M if c0 + BUSY_SLACK * M < self.busy_until: continue # a pilot inside a decoded frame self.pending.append(dict(k=k, c0=c0, need=c0 + 6 * M + self.hdr_chips + 8 * M)) self.k_next = k_final + 1 for j in [j for j in self.metric if j < self.k_next - 2 * n_pre]: del self.metric[j] for j in [j for j in self.Eu if j < self.k_next - 2 * n_pre]: del self.Eu[j] # --------------------------------------------------------------- decoding def _service(self): M = self.M still = [] hold_until = -1 # an earlier candidate that may contain later ones is unresolved for c in sorted(self.pending, key=lambda c: c["c0"]): if c["c0"] < self.x0 or c["c0"] + BUSY_SLACK * M < self.busy_until: continue if c["c0"] < hold_until: # Inside the span of an earlier frame still being received: its # pilots are up-chirps and can look like a preamble. Wait until # that frame is decoded (busy_until then drops this one). still.append(c) continue if self.chips_in < c["need"]: still.append(c) hold_until = c["c0"] + self.max_chips continue win = self.x[(c["c0"] - self.x0) * self.osr:] res = receive_frame(win, self.p, excise=self.excise, search_chips=6 * M, effort=self.effort) if res.sync is None: self.n_false += 1 continue start = c["c0"] + res.sync.tau0 if start + M / 2 < self.busy_until: # synced onto the tail of the frame just decoded (pilots are up-chirps): # the BUSY_SLACK allowance let it through; it is not a real frame self.n_false += 1 continue reason = (res.info or {}).get("reason") if reason == "truncated": # wait for the rest if res.header is None: need = int(start) + self.hdr_chips + 16 else: need = int(start) + frame_chips(self.p, res.header.length, res.header.mode == MODE_COHERENT) + 16 c["need"] = max(need, c["need"] + M) still.append(c) hold_until = c["c0"] + self.max_chips continue t = start / self.p.bw + self.t_offset if res.stage == "ok": f, lines = frame_report(res, t, self.freq) if self.clock: lines[0] = time.strftime("%H:%M:%S ", time.localtime()) + lines[0] self.frames.append(f) for ln in lines: self.say(ln) self.busy_until = start + res.info["frame_chips"] else: self.n_fail += 1 if self.verbose: self.say(f"[{t:9.3f} s] preamble found, {res.stage.replace('_', ' ')} " f"(offset {res.sync.f_hz:+.1f} Hz)") self.pending = still def _trim(self): """Drop samples nobody needs: keep the oldest pending candidate, the detector's overlap, and the decimator's look-back.""" M = self.M keep = (self.k_next - self.p.n_preamble - 2) * M if self.pending: keep = min(keep, min(c["c0"] for c in self.pending)) oldest = self.chips_in - (self.max_chips + 16 * M) # never hold more than this keep = max(keep, oldest) self.pending = [c for c in self.pending if c["c0"] >= keep] keep = (keep // M) * M if keep > self.x0: d = keep - self.x0 self.x = self.x[d * self.osr:] self.r = self.r[d:] self.x0 = keep # --------------------------------------------------------------- budget def _adapt(self, stream_s, proc_s): self.load_hist.append((stream_s, proc_s)) if not self.auto or len(self.load_hist) < 20: return load = sum(b for _, b in self.load_hist) / sum(a for a, _ in self.load_hist) if load > 0.7 and self.effort > 0: self.effort -= 1 self.say(f"(load {load:.0%}: effort lowered to {self.effort})") self.load_hist.clear() elif load < 0.3 and self.effort < self.max_effort: self.effort += 1 self.say(f"(load {load:.0%}: effort raised to {self.effort})") self.load_hist.clear() def flush(self): """End of input: try what's pending once more (it reports as cut off).""" for c in self.pending: if c["c0"] >= self.x0: c["need"] = 0 self._service() class Reader(threading.Thread): """Drains the input into a queue so the SDR never stalls while a frame is being decoded (a pipe holds only milliseconds of samples; a stalled rtl_sdr drops USB transfers silently). ``backlog`` is in bytes.""" def __init__(self, src, nbytes): super().__init__(daemon=True) self.src, self.nbytes = src, nbytes self.q = queue.Queue() self.backlog = 0 self.lock = threading.Lock() def run(self): while True: b = self.src.read(self.nbytes) if not b: self.q.put(None) return with self.lock: self.backlog += len(b) self.q.put(b) def get(self): b = self.q.get() if b is not None: with self.lock: self.backlog -= len(b) return b def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("input", nargs="?", help="capture file (SigMF or raw), or - for stdin") ap.add_argument("--rtl", action="store_true", help="run rtl_sdr and decode live") ap.add_argument("--gain", type=float, default=40.0, help="rtl_sdr tuner gain, dB") ap.add_argument("--rtl-device", default="0") ap.add_argument("--rx-if", type=float, default=250e3, help="signal offset above the RTL LO") ap.add_argument("--format", help="raw datatype (cu8 for rtl_sdr)") ap.add_argument("--fs", type=float, help="input sample rate (live default 1e6)") ap.add_argument("--lo", type=float, help="input center (LO) frequency, Hz") ap.add_argument("--freq", type=float, default=223.58e6, help="signal center, Hz") ap.add_argument("--sf", type=int, default=9) ap.add_argument("--bw", type=float, default=62.5e3) ap.add_argument("--max-bytes", type=int, default=255) ap.add_argument("--effort", type=int, default=EFFORT_MAX, choices=range(EFFORT_MAX + 1)) ap.add_argument("--auto-effort", action="store_true", help="lower/raise effort with the load (always on for live input)") ap.add_argument("--no-excise", action="store_true") ap.add_argument("--block", type=float, default=0.1, help="seconds per block") ap.add_argument("--status", type=float, default=30.0, help="live load report interval, s") ap.add_argument("-v", "--verbose", action="store_true") a = ap.parse_args() live = a.rtl or a.input == "-" proc = None if a.rtl: fs = a.fs or 1e6 lo = a.freq - a.rx_if fmt = "cu8" cmd = ["rtl_sdr", "-d", a.rtl_device, "-f", str(int(round(lo))), "-s", str(int(fs)), "-g", repr(a.gain), "-"] print("$ " + " ".join(cmd), file=sys.stderr, flush=True) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) src = proc.stdout elif a.input == "-": if not (a.format and a.fs and a.lo): sys.exit("stdin needs --format, --fs and --lo") fmt, fs, lo, src = a.format, a.fs, a.lo, sys.stdin.buffer elif a.input: if a.format: if a.fs is None or a.lo is None: sys.exit("raw input needs --fs and --lo") fmt, fs, lo, path = a.format, a.fs, a.lo, a.input else: m = iq_io.read_sigmf_meta(a.input) fmt, fs, lo, path = m["fmt"], m["fs"], m["fc"], m["data_path"] src = open(path, "rb") else: sys.exit("give a capture file, - for stdin, or --rtl") p = PhyParams(sf=a.sf, bw=a.bw, osr=2, fc=a.freq) if_off = a.freq - lo if abs(if_off) + a.bw / 2 > fs / 2: sys.exit(f"signal at {if_off / 1e3:+.1f} kHz from the LO is outside {fs / 1e6:g} MS/s") sc = iq_io.StreamConverter(fs, p.fs, f_shift=if_off, passband=a.bw / 2 + 20e3) srx = StreamReceiver(p, a.freq, a.max_bytes, effort=a.effort, auto_effort=a.auto_effort or live, excise=not a.no_excise, verbose=a.verbose, out=lambda s: print(s, flush=True), t_offset=sc.delay_s, clock=live) print(f"SF{a.sf}, {a.bw / 1e3:g} kHz at {a.freq / 1e6:.6f} MHz; input {fs / 1e6:g} MS/s " f"{fmt}, LO {lo / 1e6:.6f} MHz; effort {a.effort}" + (" (auto)" if srx.auto else ""), flush=True) bps = iq_io.bytes_per_sample(fmt) nblk = int(a.block * fs) t_wall0 = time.perf_counter() t_conv = 0.0 n_in = 0 first = True last_status = 0.0 reader = Reader(src, nblk * bps) if live else None if reader: reader.start() max_backlog = 0.0 try: while True: buf = reader.get() if reader else src.read(nblk * bps) if not buf: break if reader: max_backlog = max(max_backlog, reader.backlog / bps / fs) raw = np.frombuffer(buf[:len(buf) // bps * bps], dtype=iq_io._check(fmt)[0]) if first and fmt in ("cu8", "ci8", "ci16_le"): print(level_check(raw, fmt)[2], flush=True) first = False t0 = time.perf_counter() y = sc.process(iq_io.to_complex(raw, fmt)) t_conv += time.perf_counter() - t0 n_in += len(raw) // 2 srx.push(y) st = n_in / fs if live and st - last_status >= a.status: last_status = st print(f" [{st:7.0f} s] load {(t_conv + srx.t_front + srx.t_dec) / st:.0%} " f"(front end {(t_conv + srx.t_front) / st:.0%}, decoding " f"{srx.t_dec / st:.0%}); {len(srx.frames)} decoded, effort {srx.effort}; " f"max backlog {max_backlog:.1f} s", flush=True) max_backlog = 0.0 except KeyboardInterrupt: pass finally: if proc and proc.poll() is None: proc.send_signal(signal.SIGINT) try: proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() srx.flush() st = n_in / fs wall = time.perf_counter() - t_wall0 busy = t_conv + srx.t_front + srx.t_dec print(f"{len(srx.frames)} frame(s) decoded, {srx.n_fail} failed after sync; " f"{st:.1f} s of signal") if st > 0 and busy > 0: print(f"processing: {busy:.2f} s = {st / busy:.1f}x real time " f"(rate conversion {t_conv:.2f} s, detection {srx.t_front:.2f} s, " f"decoding {srx.t_dec:.2f} s)" + ("" if live else f"; wall {wall:.1f} s")) if __name__ == "__main__": main()