You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
392 lines
18 KiB
392 lines
18 KiB
#!/usr/bin/env python3 |
|
"""Streaming receiver: decode chirpmodem frames as the samples arrive. |
|
|
|
Live from an RTL-SDR (runs rtl_sdr itself, or reads its output on stdin), |
|
or a capture file replayed as fast as the receiver can go, which measures |
|
how much faster than real time it is: |
|
|
|
./stream_rx.py --rtl --sf 12 --gain 49.6 # live, 223.580 MHz |
|
rtl_sdr -f 223330000 -s 1000000 -g 40 - | ./stream_rx.py - --format cu8 --fs 1e6 --lo 223.33e6 |
|
./stream_rx.py ota.cu8 --format cu8 --fs 2.4e6 --lo 223.33e6 --sf 12 # replay |
|
|
|
How it works, block by block (0.1 s of samples at a time): |
|
1. rate conversion (shift the signal to 0 Hz, FIR decimate, fractional |
|
resample) carrying filter state across blocks: iq_io.StreamConverter |
|
2. decimate to the chip rate and run the preamble detector on the new |
|
windows only; a candidate is accepted once the windows on both sides |
|
of it are in (so the strongest window of each preamble wins) |
|
3. each candidate waits until its header has arrived, is decoded, then |
|
waits for the rest of the frame (length from the header) and is decoded |
|
in full; old samples are dropped once no candidate needs them |
|
|
|
Effort (--effort 0..2, see frame.EFFORT_MAX) caps the retries per frame. |
|
With --auto-effort (default for live input) the receiver lowers it when |
|
decoding takes more than 70% of real time and raises it again under 30%. |
|
|
|
Every --status seconds (live) it prints the load: processing time divided |
|
by stream time, split into front end (steps 1-2) and decoding (step 3). |
|
""" |
|
import argparse, os, queue, signal, subprocess, sys, threading, time |
|
from collections import deque |
|
import numpy as np |
|
from phy import PhyParams, payload_layout |
|
from frame import receive_frame, frame_chips, header_codec, EFFORT_MAX, MODE_COHERENT |
|
import iq_io |
|
import rx |
|
from rx_file import frame_report, level_check |
|
|
|
# Symbols of slack in 'candidate inside the last decoded frame': a frame can |
|
# start exactly where the previous one ended, whose last pilot is an up-chirp |
|
# too and can pull the detector's peak a window early. |
|
BUSY_SLACK = 8 |
|
|
|
|
|
class StreamReceiver: |
|
"""Feed modem-rate samples (p.fs) with push(); frames come out via ``out``.""" |
|
|
|
def __init__(self, p, freq, max_bytes=255, effort=EFFORT_MAX, auto_effort=False, |
|
excise=True, det_thresh=4.0, verbose=False, out=print, t_offset=0.0, |
|
clock=None): |
|
self.p, self.freq = p, freq |
|
self.M, self.osr = p.M, p.osr |
|
self.L = p.n_preamble - 1 |
|
self.effort, self.max_effort, self.auto = effort, effort, auto_effort |
|
self.excise, self.det_thresh, self.verbose = excise, det_thresh, verbose |
|
self.say, self.t_offset, self.clock = out, t_offset, clock |
|
self.max_chips = frame_chips(p, max_bytes, coherent=False) |
|
n_hdr = header_codec(p).n_sym |
|
self.hdr_chips = p.payload_start + len(payload_layout(n_hdr, p.pilot_every)[0]) * p.M |
|
self.dec_margin = 16 # chips of look-ahead the decimator needs |
|
self.x = np.zeros(0, complex) # modem-rate samples, x[0] = chip x0 |
|
self.x0 = 0 # always a multiple of M |
|
self.r = np.zeros(0, complex) # chip rate, r[0] = chip x0 |
|
self.k_next = 0 # first window whose metric isn't final yet |
|
self.metric = {} # window -> metric (recent only) |
|
self.Eu = {} # window -> zero-padded dechirped power |
|
self.cu = np.conj(rx.base_chirp_chips(p.M)) |
|
self.pending = [] # candidates waiting for samples |
|
self.busy_until = 0 # chip where the last decoded frame ended |
|
self.frames, self.n_fail, self.n_false = [], 0, 0 |
|
self.t_front = self.t_dec = 0.0 |
|
self.load_hist = deque(maxlen=50) # (stream s, decode s) per block |
|
|
|
# --------------------------------------------------------------- input |
|
@property |
|
def chips_in(self): |
|
return self.x0 + len(self.x) // self.osr |
|
|
|
def push(self, y): |
|
"""Add modem-rate samples; runs detection and any decodes now possible.""" |
|
t0 = time.perf_counter() |
|
self.x = np.concatenate([self.x, y]) |
|
self._decimate() |
|
self._detect() |
|
t1 = time.perf_counter() |
|
self._service() |
|
t2 = time.perf_counter() |
|
self._trim() |
|
self.t_front += t1 - t0 |
|
self.t_dec += t2 - t1 |
|
self._adapt(len(y) / self.p.fs, t2 - t0) |
|
|
|
def _decimate(self): |
|
n_have = self.x0 + len(self.r) |
|
n_can = self.chips_in - self.dec_margin |
|
if n_can <= n_have: |
|
return |
|
lo = max(0, (n_have - self.x0) * self.osr - self.dec_margin * self.osr) |
|
hi = min(len(self.x), (n_can - self.x0) * self.osr + self.dec_margin * self.osr) |
|
pos = (np.arange(n_have, n_can) - self.x0) * float(self.osr) - lo |
|
self.r = np.concatenate([self.r, rx.resample_at(self.x[lo:hi], pos, |
|
cutoff=1.0 / self.osr)]) |
|
|
|
# --------------------------------------------------------------- detection |
|
def _detect(self): |
|
M, L, n_pre = self.M, self.L, self.p.n_preamble |
|
nw = (self.x0 + len(self.r)) // M # complete windows (absolute) |
|
k_last = nw - L # metric needs windows k..k+L-1 |
|
k_final = k_last - n_pre # neighbourhood complete up to here |
|
if k_final < self.k_next: |
|
return |
|
# each window's spectrum is computed once (rx.preamble_metric's |
|
# consistency metric, evaluated incrementally) |
|
w0 = max(self.x0 // M, self.k_next - n_pre) # include left neighbours |
|
new = [w for w in range(w0, nw) if w not in self.Eu] |
|
if new: |
|
W = np.stack([self.r[w * M - self.x0:(w + 1) * M - self.x0] for w in new]) |
|
E = np.abs(np.fft.fft(W * self.cu, n=rx.P_DET * M, axis=1)) ** 2 |
|
for w, e in zip(new, E): |
|
self.Eu[w] = e |
|
for k in range(w0, k_last + 1): |
|
if k in self.metric: |
|
continue |
|
rows = np.stack([self.Eu[k + i] for i in range(L)]) |
|
B = int(np.argmax(rows.sum(axis=0))) |
|
ratio = rows[:, B] / rows.mean(axis=1) |
|
self.metric[k] = float(np.exp(np.mean(np.log(ratio + 1e-12)))) |
|
for k in range(self.k_next, k_final + 1): |
|
v = self.metric.get(k, 0.0) |
|
if v < self.det_thresh: |
|
continue |
|
nb = [self.metric.get(j, 0.0) for j in range(k - n_pre, k + n_pre + 1)] |
|
if v < max(nb) or any(self.metric.get(j, 0.0) == v for j in range(k - n_pre, k)): |
|
continue # not the local maximum (first of ties) |
|
c0 = (k - 3) * M |
|
if c0 + BUSY_SLACK * M < self.busy_until: |
|
continue # a pilot inside a decoded frame |
|
self.pending.append(dict(k=k, c0=c0, need=c0 + 6 * M + self.hdr_chips + 8 * M)) |
|
self.k_next = k_final + 1 |
|
for j in [j for j in self.metric if j < self.k_next - 2 * n_pre]: |
|
del self.metric[j] |
|
for j in [j for j in self.Eu if j < self.k_next - 2 * n_pre]: |
|
del self.Eu[j] |
|
|
|
# --------------------------------------------------------------- decoding |
|
def _service(self): |
|
M = self.M |
|
still = [] |
|
hold_until = -1 # an earlier candidate that may contain later ones is unresolved |
|
for c in sorted(self.pending, key=lambda c: c["c0"]): |
|
if c["c0"] < self.x0 or c["c0"] + BUSY_SLACK * M < self.busy_until: |
|
continue |
|
if c["c0"] < hold_until: |
|
# Inside the span of an earlier frame still being received: its |
|
# pilots are up-chirps and can look like a preamble. Wait until |
|
# that frame is decoded (busy_until then drops this one). |
|
still.append(c) |
|
continue |
|
if self.chips_in < c["need"]: |
|
still.append(c) |
|
hold_until = c["c0"] + self.max_chips |
|
continue |
|
win = self.x[(c["c0"] - self.x0) * self.osr:] |
|
res = receive_frame(win, self.p, excise=self.excise, search_chips=6 * M, |
|
effort=self.effort) |
|
if res.sync is None: |
|
self.n_false += 1 |
|
continue |
|
start = c["c0"] + res.sync.tau0 |
|
if start + M / 2 < self.busy_until: |
|
# synced onto the tail of the frame just decoded (pilots are up-chirps): |
|
# the BUSY_SLACK allowance let it through; it is not a real frame |
|
self.n_false += 1 |
|
continue |
|
reason = (res.info or {}).get("reason") |
|
if reason == "truncated": # wait for the rest |
|
if res.header is None: |
|
need = int(start) + self.hdr_chips + 16 |
|
else: |
|
need = int(start) + frame_chips(self.p, res.header.length, |
|
res.header.mode == MODE_COHERENT) + 16 |
|
c["need"] = max(need, c["need"] + M) |
|
still.append(c) |
|
hold_until = c["c0"] + self.max_chips |
|
continue |
|
t = start / self.p.bw + self.t_offset |
|
if res.stage == "ok": |
|
f, lines = frame_report(res, t, self.freq) |
|
if self.clock: |
|
lines[0] = time.strftime("%H:%M:%S ", time.localtime()) + lines[0] |
|
self.frames.append(f) |
|
for ln in lines: |
|
self.say(ln) |
|
self.busy_until = start + res.info["frame_chips"] |
|
else: |
|
self.n_fail += 1 |
|
if self.verbose: |
|
self.say(f"[{t:9.3f} s] preamble found, {res.stage.replace('_', ' ')} " |
|
f"(offset {res.sync.f_hz:+.1f} Hz)") |
|
self.pending = still |
|
|
|
def _trim(self): |
|
"""Drop samples nobody needs: keep the oldest pending candidate, the |
|
detector's overlap, and the decimator's look-back.""" |
|
M = self.M |
|
keep = (self.k_next - self.p.n_preamble - 2) * M |
|
if self.pending: |
|
keep = min(keep, min(c["c0"] for c in self.pending)) |
|
oldest = self.chips_in - (self.max_chips + 16 * M) # never hold more than this |
|
keep = max(keep, oldest) |
|
self.pending = [c for c in self.pending if c["c0"] >= keep] |
|
keep = (keep // M) * M |
|
if keep > self.x0: |
|
d = keep - self.x0 |
|
self.x = self.x[d * self.osr:] |
|
self.r = self.r[d:] |
|
self.x0 = keep |
|
|
|
# --------------------------------------------------------------- budget |
|
def _adapt(self, stream_s, proc_s): |
|
self.load_hist.append((stream_s, proc_s)) |
|
if not self.auto or len(self.load_hist) < 20: |
|
return |
|
load = sum(b for _, b in self.load_hist) / sum(a for a, _ in self.load_hist) |
|
if load > 0.7 and self.effort > 0: |
|
self.effort -= 1 |
|
self.say(f"(load {load:.0%}: effort lowered to {self.effort})") |
|
self.load_hist.clear() |
|
elif load < 0.3 and self.effort < self.max_effort: |
|
self.effort += 1 |
|
self.say(f"(load {load:.0%}: effort raised to {self.effort})") |
|
self.load_hist.clear() |
|
|
|
def flush(self): |
|
"""End of input: try what's pending once more (it reports as cut off).""" |
|
for c in self.pending: |
|
if c["c0"] >= self.x0: |
|
c["need"] = 0 |
|
self._service() |
|
|
|
|
|
class Reader(threading.Thread): |
|
"""Drains the input into a queue so the SDR never stalls while a frame is |
|
being decoded (a pipe holds only milliseconds of samples; a stalled |
|
rtl_sdr drops USB transfers silently). ``backlog`` is in bytes.""" |
|
|
|
def __init__(self, src, nbytes): |
|
super().__init__(daemon=True) |
|
self.src, self.nbytes = src, nbytes |
|
self.q = queue.Queue() |
|
self.backlog = 0 |
|
self.lock = threading.Lock() |
|
|
|
def run(self): |
|
while True: |
|
b = self.src.read(self.nbytes) |
|
if not b: |
|
self.q.put(None) |
|
return |
|
with self.lock: |
|
self.backlog += len(b) |
|
self.q.put(b) |
|
|
|
def get(self): |
|
b = self.q.get() |
|
if b is not None: |
|
with self.lock: |
|
self.backlog -= len(b) |
|
return b |
|
|
|
|
|
def main(): |
|
ap = argparse.ArgumentParser(description=__doc__, |
|
formatter_class=argparse.RawDescriptionHelpFormatter) |
|
ap.add_argument("input", nargs="?", help="capture file (SigMF or raw), or - for stdin") |
|
ap.add_argument("--rtl", action="store_true", help="run rtl_sdr and decode live") |
|
ap.add_argument("--gain", type=float, default=40.0, help="rtl_sdr tuner gain, dB") |
|
ap.add_argument("--rtl-device", default="0") |
|
ap.add_argument("--rx-if", type=float, default=250e3, help="signal offset above the RTL LO") |
|
ap.add_argument("--format", help="raw datatype (cu8 for rtl_sdr)") |
|
ap.add_argument("--fs", type=float, help="input sample rate (live default 1e6)") |
|
ap.add_argument("--lo", type=float, help="input center (LO) frequency, Hz") |
|
ap.add_argument("--freq", type=float, default=223.58e6, help="signal center, Hz") |
|
ap.add_argument("--sf", type=int, default=9) |
|
ap.add_argument("--bw", type=float, default=62.5e3) |
|
ap.add_argument("--max-bytes", type=int, default=255) |
|
ap.add_argument("--effort", type=int, default=EFFORT_MAX, choices=range(EFFORT_MAX + 1)) |
|
ap.add_argument("--auto-effort", action="store_true", |
|
help="lower/raise effort with the load (always on for live input)") |
|
ap.add_argument("--no-excise", action="store_true") |
|
ap.add_argument("--block", type=float, default=0.1, help="seconds per block") |
|
ap.add_argument("--status", type=float, default=30.0, help="live load report interval, s") |
|
ap.add_argument("-v", "--verbose", action="store_true") |
|
a = ap.parse_args() |
|
|
|
live = a.rtl or a.input == "-" |
|
proc = None |
|
if a.rtl: |
|
fs = a.fs or 1e6 |
|
lo = a.freq - a.rx_if |
|
fmt = "cu8" |
|
cmd = ["rtl_sdr", "-d", a.rtl_device, "-f", str(int(round(lo))), "-s", str(int(fs)), |
|
"-g", repr(a.gain), "-"] |
|
print("$ " + " ".join(cmd), file=sys.stderr, flush=True) |
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) |
|
src = proc.stdout |
|
elif a.input == "-": |
|
if not (a.format and a.fs and a.lo): |
|
sys.exit("stdin needs --format, --fs and --lo") |
|
fmt, fs, lo, src = a.format, a.fs, a.lo, sys.stdin.buffer |
|
elif a.input: |
|
if a.format: |
|
if a.fs is None or a.lo is None: |
|
sys.exit("raw input needs --fs and --lo") |
|
fmt, fs, lo, path = a.format, a.fs, a.lo, a.input |
|
else: |
|
m = iq_io.read_sigmf_meta(a.input) |
|
fmt, fs, lo, path = m["fmt"], m["fs"], m["fc"], m["data_path"] |
|
src = open(path, "rb") |
|
else: |
|
sys.exit("give a capture file, - for stdin, or --rtl") |
|
|
|
p = PhyParams(sf=a.sf, bw=a.bw, osr=2, fc=a.freq) |
|
if_off = a.freq - lo |
|
if abs(if_off) + a.bw / 2 > fs / 2: |
|
sys.exit(f"signal at {if_off / 1e3:+.1f} kHz from the LO is outside {fs / 1e6:g} MS/s") |
|
sc = iq_io.StreamConverter(fs, p.fs, f_shift=if_off, passband=a.bw / 2 + 20e3) |
|
srx = StreamReceiver(p, a.freq, a.max_bytes, effort=a.effort, |
|
auto_effort=a.auto_effort or live, excise=not a.no_excise, |
|
verbose=a.verbose, out=lambda s: print(s, flush=True), |
|
t_offset=sc.delay_s, clock=live) |
|
print(f"SF{a.sf}, {a.bw / 1e3:g} kHz at {a.freq / 1e6:.6f} MHz; input {fs / 1e6:g} MS/s " |
|
f"{fmt}, LO {lo / 1e6:.6f} MHz; effort {a.effort}" |
|
+ (" (auto)" if srx.auto else ""), flush=True) |
|
|
|
bps = iq_io.bytes_per_sample(fmt) |
|
nblk = int(a.block * fs) |
|
t_wall0 = time.perf_counter() |
|
t_conv = 0.0 |
|
n_in = 0 |
|
first = True |
|
last_status = 0.0 |
|
reader = Reader(src, nblk * bps) if live else None |
|
if reader: |
|
reader.start() |
|
max_backlog = 0.0 |
|
try: |
|
while True: |
|
buf = reader.get() if reader else src.read(nblk * bps) |
|
if not buf: |
|
break |
|
if reader: |
|
max_backlog = max(max_backlog, reader.backlog / bps / fs) |
|
raw = np.frombuffer(buf[:len(buf) // bps * bps], dtype=iq_io._check(fmt)[0]) |
|
if first and fmt in ("cu8", "ci8", "ci16_le"): |
|
print(level_check(raw, fmt)[2], flush=True) |
|
first = False |
|
t0 = time.perf_counter() |
|
y = sc.process(iq_io.to_complex(raw, fmt)) |
|
t_conv += time.perf_counter() - t0 |
|
n_in += len(raw) // 2 |
|
srx.push(y) |
|
st = n_in / fs |
|
if live and st - last_status >= a.status: |
|
last_status = st |
|
print(f" [{st:7.0f} s] load {(t_conv + srx.t_front + srx.t_dec) / st:.0%} " |
|
f"(front end {(t_conv + srx.t_front) / st:.0%}, decoding " |
|
f"{srx.t_dec / st:.0%}); {len(srx.frames)} decoded, effort {srx.effort}; " |
|
f"max backlog {max_backlog:.1f} s", flush=True) |
|
max_backlog = 0.0 |
|
except KeyboardInterrupt: |
|
pass |
|
finally: |
|
if proc and proc.poll() is None: |
|
proc.send_signal(signal.SIGINT) |
|
try: |
|
proc.wait(timeout=5) |
|
except subprocess.TimeoutExpired: |
|
proc.kill() |
|
srx.flush() |
|
st = n_in / fs |
|
wall = time.perf_counter() - t_wall0 |
|
busy = t_conv + srx.t_front + srx.t_dec |
|
print(f"{len(srx.frames)} frame(s) decoded, {srx.n_fail} failed after sync; " |
|
f"{st:.1f} s of signal") |
|
if st > 0 and busy > 0: |
|
print(f"processing: {busy:.2f} s = {st / busy:.1f}x real time " |
|
f"(rate conversion {t_conv:.2f} s, detection {srx.t_front:.2f} s, " |
|
f"decoding {srx.t_dec:.2f} s)" + ("" if live else f"; wall {wall:.1f} s")) |
|
|
|
|
|
if __name__ == "__main__": |
|
main()
|
|
|