You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

392 lines
18 KiB

#!/usr/bin/env python3
"""Streaming receiver: decode chirpmodem frames as the samples arrive.
Live from an RTL-SDR (runs rtl_sdr itself, or reads its output on stdin),
or a capture file replayed as fast as the receiver can go, which measures
how much faster than real time it is:
./stream_rx.py --rtl --sf 12 --gain 49.6 # live, 223.580 MHz
rtl_sdr -f 223330000 -s 1000000 -g 40 - | ./stream_rx.py - --format cu8 --fs 1e6 --lo 223.33e6
./stream_rx.py ota.cu8 --format cu8 --fs 2.4e6 --lo 223.33e6 --sf 12 # replay
How it works, block by block (0.1 s of samples at a time):
1. rate conversion (shift the signal to 0 Hz, FIR decimate, fractional
resample) carrying filter state across blocks: iq_io.StreamConverter
2. decimate to the chip rate and run the preamble detector on the new
windows only; a candidate is accepted once the windows on both sides
of it are in (so the strongest window of each preamble wins)
3. each candidate waits until its header has arrived, is decoded, then
waits for the rest of the frame (length from the header) and is decoded
in full; old samples are dropped once no candidate needs them
Effort (--effort 0..2, see frame.EFFORT_MAX) caps the retries per frame.
With --auto-effort (default for live input) the receiver lowers it when
decoding takes more than 70% of real time and raises it again under 30%.
Every --status seconds (live) it prints the load: processing time divided
by stream time, split into front end (steps 1-2) and decoding (step 3).
"""
import argparse, os, queue, signal, subprocess, sys, threading, time
from collections import deque
import numpy as np
from phy import PhyParams, payload_layout
from frame import receive_frame, frame_chips, header_codec, EFFORT_MAX, MODE_COHERENT
import iq_io
import rx
from rx_file import frame_report, level_check
# Symbols of slack in 'candidate inside the last decoded frame': a frame can
# start exactly where the previous one ended, whose last pilot is an up-chirp
# too and can pull the detector's peak a window early.
BUSY_SLACK = 8
class StreamReceiver:
"""Feed modem-rate samples (p.fs) with push(); frames come out via ``out``."""
def __init__(self, p, freq, max_bytes=255, effort=EFFORT_MAX, auto_effort=False,
excise=True, det_thresh=4.0, verbose=False, out=print, t_offset=0.0,
clock=None):
self.p, self.freq = p, freq
self.M, self.osr = p.M, p.osr
self.L = p.n_preamble - 1
self.effort, self.max_effort, self.auto = effort, effort, auto_effort
self.excise, self.det_thresh, self.verbose = excise, det_thresh, verbose
self.say, self.t_offset, self.clock = out, t_offset, clock
self.max_chips = frame_chips(p, max_bytes, coherent=False)
n_hdr = header_codec(p).n_sym
self.hdr_chips = p.payload_start + len(payload_layout(n_hdr, p.pilot_every)[0]) * p.M
self.dec_margin = 16 # chips of look-ahead the decimator needs
self.x = np.zeros(0, complex) # modem-rate samples, x[0] = chip x0
self.x0 = 0 # always a multiple of M
self.r = np.zeros(0, complex) # chip rate, r[0] = chip x0
self.k_next = 0 # first window whose metric isn't final yet
self.metric = {} # window -> metric (recent only)
self.Eu = {} # window -> zero-padded dechirped power
self.cu = np.conj(rx.base_chirp_chips(p.M))
self.pending = [] # candidates waiting for samples
self.busy_until = 0 # chip where the last decoded frame ended
self.frames, self.n_fail, self.n_false = [], 0, 0
self.t_front = self.t_dec = 0.0
self.load_hist = deque(maxlen=50) # (stream s, decode s) per block
# --------------------------------------------------------------- input
@property
def chips_in(self):
return self.x0 + len(self.x) // self.osr
def push(self, y):
"""Add modem-rate samples; runs detection and any decodes now possible."""
t0 = time.perf_counter()
self.x = np.concatenate([self.x, y])
self._decimate()
self._detect()
t1 = time.perf_counter()
self._service()
t2 = time.perf_counter()
self._trim()
self.t_front += t1 - t0
self.t_dec += t2 - t1
self._adapt(len(y) / self.p.fs, t2 - t0)
def _decimate(self):
n_have = self.x0 + len(self.r)
n_can = self.chips_in - self.dec_margin
if n_can <= n_have:
return
lo = max(0, (n_have - self.x0) * self.osr - self.dec_margin * self.osr)
hi = min(len(self.x), (n_can - self.x0) * self.osr + self.dec_margin * self.osr)
pos = (np.arange(n_have, n_can) - self.x0) * float(self.osr) - lo
self.r = np.concatenate([self.r, rx.resample_at(self.x[lo:hi], pos,
cutoff=1.0 / self.osr)])
# --------------------------------------------------------------- detection
def _detect(self):
M, L, n_pre = self.M, self.L, self.p.n_preamble
nw = (self.x0 + len(self.r)) // M # complete windows (absolute)
k_last = nw - L # metric needs windows k..k+L-1
k_final = k_last - n_pre # neighbourhood complete up to here
if k_final < self.k_next:
return
# each window's spectrum is computed once (rx.preamble_metric's
# consistency metric, evaluated incrementally)
w0 = max(self.x0 // M, self.k_next - n_pre) # include left neighbours
new = [w for w in range(w0, nw) if w not in self.Eu]
if new:
W = np.stack([self.r[w * M - self.x0:(w + 1) * M - self.x0] for w in new])
E = np.abs(np.fft.fft(W * self.cu, n=rx.P_DET * M, axis=1)) ** 2
for w, e in zip(new, E):
self.Eu[w] = e
for k in range(w0, k_last + 1):
if k in self.metric:
continue
rows = np.stack([self.Eu[k + i] for i in range(L)])
B = int(np.argmax(rows.sum(axis=0)))
ratio = rows[:, B] / rows.mean(axis=1)
self.metric[k] = float(np.exp(np.mean(np.log(ratio + 1e-12))))
for k in range(self.k_next, k_final + 1):
v = self.metric.get(k, 0.0)
if v < self.det_thresh:
continue
nb = [self.metric.get(j, 0.0) for j in range(k - n_pre, k + n_pre + 1)]
if v < max(nb) or any(self.metric.get(j, 0.0) == v for j in range(k - n_pre, k)):
continue # not the local maximum (first of ties)
c0 = (k - 3) * M
if c0 + BUSY_SLACK * M < self.busy_until:
continue # a pilot inside a decoded frame
self.pending.append(dict(k=k, c0=c0, need=c0 + 6 * M + self.hdr_chips + 8 * M))
self.k_next = k_final + 1
for j in [j for j in self.metric if j < self.k_next - 2 * n_pre]:
del self.metric[j]
for j in [j for j in self.Eu if j < self.k_next - 2 * n_pre]:
del self.Eu[j]
# --------------------------------------------------------------- decoding
def _service(self):
M = self.M
still = []
hold_until = -1 # an earlier candidate that may contain later ones is unresolved
for c in sorted(self.pending, key=lambda c: c["c0"]):
if c["c0"] < self.x0 or c["c0"] + BUSY_SLACK * M < self.busy_until:
continue
if c["c0"] < hold_until:
# Inside the span of an earlier frame still being received: its
# pilots are up-chirps and can look like a preamble. Wait until
# that frame is decoded (busy_until then drops this one).
still.append(c)
continue
if self.chips_in < c["need"]:
still.append(c)
hold_until = c["c0"] + self.max_chips
continue
win = self.x[(c["c0"] - self.x0) * self.osr:]
res = receive_frame(win, self.p, excise=self.excise, search_chips=6 * M,
effort=self.effort)
if res.sync is None:
self.n_false += 1
continue
start = c["c0"] + res.sync.tau0
if start + M / 2 < self.busy_until:
# synced onto the tail of the frame just decoded (pilots are up-chirps):
# the BUSY_SLACK allowance let it through; it is not a real frame
self.n_false += 1
continue
reason = (res.info or {}).get("reason")
if reason == "truncated": # wait for the rest
if res.header is None:
need = int(start) + self.hdr_chips + 16
else:
need = int(start) + frame_chips(self.p, res.header.length,
res.header.mode == MODE_COHERENT) + 16
c["need"] = max(need, c["need"] + M)
still.append(c)
hold_until = c["c0"] + self.max_chips
continue
t = start / self.p.bw + self.t_offset
if res.stage == "ok":
f, lines = frame_report(res, t, self.freq)
if self.clock:
lines[0] = time.strftime("%H:%M:%S ", time.localtime()) + lines[0]
self.frames.append(f)
for ln in lines:
self.say(ln)
self.busy_until = start + res.info["frame_chips"]
else:
self.n_fail += 1
if self.verbose:
self.say(f"[{t:9.3f} s] preamble found, {res.stage.replace('_', ' ')} "
f"(offset {res.sync.f_hz:+.1f} Hz)")
self.pending = still
def _trim(self):
"""Drop samples nobody needs: keep the oldest pending candidate, the
detector's overlap, and the decimator's look-back."""
M = self.M
keep = (self.k_next - self.p.n_preamble - 2) * M
if self.pending:
keep = min(keep, min(c["c0"] for c in self.pending))
oldest = self.chips_in - (self.max_chips + 16 * M) # never hold more than this
keep = max(keep, oldest)
self.pending = [c for c in self.pending if c["c0"] >= keep]
keep = (keep // M) * M
if keep > self.x0:
d = keep - self.x0
self.x = self.x[d * self.osr:]
self.r = self.r[d:]
self.x0 = keep
# --------------------------------------------------------------- budget
def _adapt(self, stream_s, proc_s):
self.load_hist.append((stream_s, proc_s))
if not self.auto or len(self.load_hist) < 20:
return
load = sum(b for _, b in self.load_hist) / sum(a for a, _ in self.load_hist)
if load > 0.7 and self.effort > 0:
self.effort -= 1
self.say(f"(load {load:.0%}: effort lowered to {self.effort})")
self.load_hist.clear()
elif load < 0.3 and self.effort < self.max_effort:
self.effort += 1
self.say(f"(load {load:.0%}: effort raised to {self.effort})")
self.load_hist.clear()
def flush(self):
"""End of input: try what's pending once more (it reports as cut off)."""
for c in self.pending:
if c["c0"] >= self.x0:
c["need"] = 0
self._service()
class Reader(threading.Thread):
"""Drains the input into a queue so the SDR never stalls while a frame is
being decoded (a pipe holds only milliseconds of samples; a stalled
rtl_sdr drops USB transfers silently). ``backlog`` is in bytes."""
def __init__(self, src, nbytes):
super().__init__(daemon=True)
self.src, self.nbytes = src, nbytes
self.q = queue.Queue()
self.backlog = 0
self.lock = threading.Lock()
def run(self):
while True:
b = self.src.read(self.nbytes)
if not b:
self.q.put(None)
return
with self.lock:
self.backlog += len(b)
self.q.put(b)
def get(self):
b = self.q.get()
if b is not None:
with self.lock:
self.backlog -= len(b)
return b
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("input", nargs="?", help="capture file (SigMF or raw), or - for stdin")
ap.add_argument("--rtl", action="store_true", help="run rtl_sdr and decode live")
ap.add_argument("--gain", type=float, default=40.0, help="rtl_sdr tuner gain, dB")
ap.add_argument("--rtl-device", default="0")
ap.add_argument("--rx-if", type=float, default=250e3, help="signal offset above the RTL LO")
ap.add_argument("--format", help="raw datatype (cu8 for rtl_sdr)")
ap.add_argument("--fs", type=float, help="input sample rate (live default 1e6)")
ap.add_argument("--lo", type=float, help="input center (LO) frequency, Hz")
ap.add_argument("--freq", type=float, default=223.58e6, help="signal center, Hz")
ap.add_argument("--sf", type=int, default=9)
ap.add_argument("--bw", type=float, default=62.5e3)
ap.add_argument("--max-bytes", type=int, default=255)
ap.add_argument("--effort", type=int, default=EFFORT_MAX, choices=range(EFFORT_MAX + 1))
ap.add_argument("--auto-effort", action="store_true",
help="lower/raise effort with the load (always on for live input)")
ap.add_argument("--no-excise", action="store_true")
ap.add_argument("--block", type=float, default=0.1, help="seconds per block")
ap.add_argument("--status", type=float, default=30.0, help="live load report interval, s")
ap.add_argument("-v", "--verbose", action="store_true")
a = ap.parse_args()
live = a.rtl or a.input == "-"
proc = None
if a.rtl:
fs = a.fs or 1e6
lo = a.freq - a.rx_if
fmt = "cu8"
cmd = ["rtl_sdr", "-d", a.rtl_device, "-f", str(int(round(lo))), "-s", str(int(fs)),
"-g", repr(a.gain), "-"]
print("$ " + " ".join(cmd), file=sys.stderr, flush=True)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
src = proc.stdout
elif a.input == "-":
if not (a.format and a.fs and a.lo):
sys.exit("stdin needs --format, --fs and --lo")
fmt, fs, lo, src = a.format, a.fs, a.lo, sys.stdin.buffer
elif a.input:
if a.format:
if a.fs is None or a.lo is None:
sys.exit("raw input needs --fs and --lo")
fmt, fs, lo, path = a.format, a.fs, a.lo, a.input
else:
m = iq_io.read_sigmf_meta(a.input)
fmt, fs, lo, path = m["fmt"], m["fs"], m["fc"], m["data_path"]
src = open(path, "rb")
else:
sys.exit("give a capture file, - for stdin, or --rtl")
p = PhyParams(sf=a.sf, bw=a.bw, osr=2, fc=a.freq)
if_off = a.freq - lo
if abs(if_off) + a.bw / 2 > fs / 2:
sys.exit(f"signal at {if_off / 1e3:+.1f} kHz from the LO is outside {fs / 1e6:g} MS/s")
sc = iq_io.StreamConverter(fs, p.fs, f_shift=if_off, passband=a.bw / 2 + 20e3)
srx = StreamReceiver(p, a.freq, a.max_bytes, effort=a.effort,
auto_effort=a.auto_effort or live, excise=not a.no_excise,
verbose=a.verbose, out=lambda s: print(s, flush=True),
t_offset=sc.delay_s, clock=live)
print(f"SF{a.sf}, {a.bw / 1e3:g} kHz at {a.freq / 1e6:.6f} MHz; input {fs / 1e6:g} MS/s "
f"{fmt}, LO {lo / 1e6:.6f} MHz; effort {a.effort}"
+ (" (auto)" if srx.auto else ""), flush=True)
bps = iq_io.bytes_per_sample(fmt)
nblk = int(a.block * fs)
t_wall0 = time.perf_counter()
t_conv = 0.0
n_in = 0
first = True
last_status = 0.0
reader = Reader(src, nblk * bps) if live else None
if reader:
reader.start()
max_backlog = 0.0
try:
while True:
buf = reader.get() if reader else src.read(nblk * bps)
if not buf:
break
if reader:
max_backlog = max(max_backlog, reader.backlog / bps / fs)
raw = np.frombuffer(buf[:len(buf) // bps * bps], dtype=iq_io._check(fmt)[0])
if first and fmt in ("cu8", "ci8", "ci16_le"):
print(level_check(raw, fmt)[2], flush=True)
first = False
t0 = time.perf_counter()
y = sc.process(iq_io.to_complex(raw, fmt))
t_conv += time.perf_counter() - t0
n_in += len(raw) // 2
srx.push(y)
st = n_in / fs
if live and st - last_status >= a.status:
last_status = st
print(f" [{st:7.0f} s] load {(t_conv + srx.t_front + srx.t_dec) / st:.0%} "
f"(front end {(t_conv + srx.t_front) / st:.0%}, decoding "
f"{srx.t_dec / st:.0%}); {len(srx.frames)} decoded, effort {srx.effort}; "
f"max backlog {max_backlog:.1f} s", flush=True)
max_backlog = 0.0
except KeyboardInterrupt:
pass
finally:
if proc and proc.poll() is None:
proc.send_signal(signal.SIGINT)
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
srx.flush()
st = n_in / fs
wall = time.perf_counter() - t_wall0
busy = t_conv + srx.t_front + srx.t_dec
print(f"{len(srx.frames)} frame(s) decoded, {srx.n_fail} failed after sync; "
f"{st:.1f} s of signal")
if st > 0 and busy > 0:
print(f"processing: {busy:.2f} s = {st / busy:.1f}x real time "
f"(rate conversion {t_conv:.2f} s, detection {srx.t_front:.2f} s, "
f"decoding {srx.t_dec:.2f} s)" + ("" if live else f"; wall {wall:.1f} s"))
if __name__ == "__main__":
main()