You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
269 lines
10 KiB
269 lines
10 KiB
"""IQ file I/O: SigMF recordings and raw interleaved samples, plus the rate |
|
conversion and frequency shifting that sit between an SDR and the modem. |
|
|
|
Formats (SigMF ``core:datatype`` names; interleaved I then Q): |
|
cf32_le complex float32, full scale 1.0 (GNU Radio, SDR++) |
|
ci16_le complex int16, full scale 32768 (Pluto / libiio) |
|
cu8 complex uint8, center 127.5 (rtl_sdr native) |
|
ci8 complex int8, full scale 128 (HackRF) |
|
|
|
SigMF: ``name.sigmf-meta`` (JSON) + ``name.sigmf-data`` (raw samples). Files |
|
written here declare a ``chirpmodem`` extension namespace carrying the modem |
|
parameters, so ``rx_file.py`` can decode them without being told SF/BW. |
|
""" |
|
import json, os, datetime |
|
import numpy as np |
|
from dsp import resample_at |
|
|
|
SIGMF_VERSION = "1.2.0" |
|
# Largest recording the Pluto has taken as one cyclic TX buffer in testing: |
|
# 5.6 M samples worked, 25 M failed with EFBIG ("File too large"). |
|
PLUTO_MAX_SAMPLES = 6_000_000 |
|
EXT = {"name": "chirpmodem", "version": "0.1.0", "optional": True} |
|
|
|
_RAW = { |
|
"cf32_le": (np.dtype("<f4"), 0.0, 1.0), |
|
"ci16_le": (np.dtype("<i2"), 0.0, 32768.0), |
|
"cu8": (np.dtype("u1"), 127.5, 127.5), |
|
"ci8": (np.dtype("i1"), 0.0, 128.0), |
|
} |
|
|
|
|
|
def _check(fmt): |
|
if fmt not in _RAW: |
|
raise ValueError(f"unsupported datatype {fmt!r}; use one of {sorted(_RAW)}") |
|
return _RAW[fmt] |
|
|
|
|
|
# ------------------------------------------------------------------ raw |
|
def bytes_per_sample(fmt): |
|
return 2 * _check(fmt)[0].itemsize |
|
|
|
|
|
def open_raw(path, fmt): |
|
"""Memory-map a raw file as interleaved scalars (no conversion yet).""" |
|
dt, _, _ = _check(fmt) |
|
return np.memmap(path, dtype=dt, mode="r") |
|
|
|
|
|
def to_complex(raw, fmt): |
|
"""Interleaved scalars -> complex128, normalized to full scale 1.""" |
|
_, off, scale = _check(fmt) |
|
v = (np.asarray(raw, np.float64) - off) / scale |
|
v = v[: len(v) // 2 * 2] |
|
return v[0::2] + 1j * v[1::2] |
|
|
|
|
|
def from_complex(x, fmt): |
|
"""complex -> interleaved scalars in ``fmt`` (clipped at full scale).""" |
|
dt, off, scale = _check(fmt) |
|
v = np.empty(2 * len(x), np.float64) |
|
v[0::2], v[1::2] = x.real, x.imag |
|
if dt.kind == "f": |
|
return v.astype(dt) |
|
info = np.iinfo(dt) |
|
return np.clip(np.round(v * scale + off), info.min, info.max).astype(dt) |
|
|
|
|
|
def read_raw(path, fmt, start=0, count=None): |
|
raw = open_raw(path, fmt) |
|
stop = None if count is None else 2 * (start + count) |
|
return to_complex(raw[2 * start:stop], fmt) |
|
|
|
|
|
def write_raw(path, x, fmt, append=False): |
|
with open(path, "ab" if append else "wb") as f: |
|
from_complex(np.asarray(x), fmt).tofile(f) |
|
|
|
|
|
# ------------------------------------------------------------------ SigMF |
|
def sigmf_paths(path): |
|
"""Accept 'name', 'name.sigmf-meta' or 'name.sigmf-data'.""" |
|
for ext in (".sigmf-meta", ".sigmf-data", ".sigmf"): |
|
if path.endswith(ext): |
|
path = path[: -len(ext)] |
|
return path + ".sigmf-meta", path + ".sigmf-data" |
|
|
|
|
|
def utc_now(): |
|
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" |
|
|
|
|
|
def write_sigmf_meta(path, fmt, fs, fc, description="", annotations=(), extra=None, |
|
author=None, hw=None): |
|
meta_path, _ = sigmf_paths(path) |
|
g = {"core:datatype": fmt, "core:version": SIGMF_VERSION, "core:sample_rate": float(fs), |
|
"core:recorder": "chirpmodem", "core:extensions": [EXT]} |
|
if description: |
|
g["core:description"] = description |
|
if author: |
|
g["core:author"] = author |
|
if hw: |
|
g["core:hw"] = hw |
|
if extra: |
|
g.update({k if ":" in k else f"chirpmodem:{k}": v for k, v in extra.items()}) |
|
meta = {"global": g, |
|
"captures": [{"core:sample_start": 0, "core:frequency": float(fc), |
|
"core:datetime": utc_now()}], |
|
"annotations": sorted(annotations, key=lambda a: a["core:sample_start"])} |
|
with open(meta_path, "w") as f: |
|
json.dump(meta, f, indent=2) |
|
return meta_path |
|
|
|
|
|
def read_sigmf_meta(path): |
|
meta_path, data_path = sigmf_paths(path) |
|
with open(meta_path) as f: |
|
meta = json.load(f) |
|
g = meta["global"] |
|
caps = meta.get("captures") or [{}] |
|
return dict(meta=meta, data_path=data_path, fmt=g["core:datatype"], |
|
fs=g.get("core:sample_rate"), fc=caps[0].get("core:frequency"), |
|
ext={k.split(":", 1)[1]: v for k, v in g.items() |
|
if k.startswith("chirpmodem:")}) |
|
|
|
|
|
# ------------------------------------------------------------------ DSP glue |
|
def recording_samples(path): |
|
m = read_sigmf_meta(path) |
|
return os.path.getsize(m["data_path"]) // bytes_per_sample(m["fmt"]) |
|
|
|
|
|
def lowpass_taps(cutoff, transition, atten_db=80.0): |
|
"""Kaiser-windowed sinc. cutoff and transition relative to the input |
|
sample rate (cycles/sample, 0..0.5).""" |
|
beta = 0.1102 * (atten_db - 8.7) |
|
n = int(np.ceil((atten_db - 8) / (2.285 * 2 * np.pi * transition))) | 1 |
|
k = np.arange(n) - (n - 1) / 2 |
|
h = 2 * cutoff * np.sinc(2 * cutoff * k) * np.kaiser(n, beta) |
|
return h / h.sum() |
|
|
|
|
|
def _fft_valid(x, h): |
|
"""'valid' convolution via FFT: len(x) - len(h) + 1 outputs.""" |
|
n = len(x) + len(h) - 1 |
|
nfft = 1 << int(np.ceil(np.log2(n))) |
|
y = np.fft.ifft(np.fft.fft(x, nfft) * np.fft.fft(h, nfft))[:n] |
|
return y[len(h) - 1:len(x)] |
|
|
|
|
|
def _poly_decimate(buf, h, D, off): |
|
"""``_fft_valid(buf, h)[off::D]`` computed directly, one polyphase branch |
|
per phase, so only the kept outputs cost anything (taps MACs each).""" |
|
L = len(h) |
|
n_out = -(-(len(buf) - L + 1 - off) // D) |
|
if n_out <= 0: |
|
return np.zeros(0, complex) |
|
g = h[::-1] |
|
out = np.zeros(n_out, complex) |
|
for r in range(D): |
|
f = g[r::D] |
|
if len(f) == 0: |
|
continue |
|
seq = buf[off + r::D] |
|
out += np.correlate(seq, f, mode="valid")[:n_out] |
|
return out |
|
|
|
|
|
class RateConverter: |
|
"""Streaming SDR-rate -> modem-rate converter. |
|
|
|
Shifts the signal at ``f_shift`` Hz down to 0 Hz (offset tuning), low-pass |
|
filters and decimates by an integer D in FFT blocks, then |
|
``finish()`` applies the remaining fractional ratio with the windowed-sinc |
|
resampler. ``passband`` (one-sided, Hz) must hold the signal plus its |
|
worst-case frequency offset. |
|
""" |
|
|
|
def __init__(self, fs_in, fs_out, f_shift=0.0, passband=None): |
|
self.fs_in, self.fs_out, self.f_shift = float(fs_in), float(fs_out), float(f_shift) |
|
self.passband = passband or 0.3 * fs_out |
|
self.D = max(1, int(fs_in // (fs_out * 1.2))) |
|
self.fs_mid = self.fs_in / self.D |
|
self.n_in = 0 |
|
if self.D > 1: |
|
stop = self.fs_mid - self.passband # first alias that must be gone |
|
self.h = lowpass_taps((self.passband + stop) / 2 / self.fs_in, |
|
(stop - self.passband) / self.fs_in) |
|
self.tail = np.zeros(len(self.h) - 1, complex) |
|
self.mid = [] |
|
|
|
def process(self, x): |
|
x = np.asarray(x, complex) |
|
if self.f_shift: |
|
# block start phase x a cached table (exp per sample costs more than the filter) |
|
if getattr(self, "_rot_len", -1) < len(x): |
|
self._rot = np.exp(-2j * np.pi * self.f_shift * np.arange(len(x)) / self.fs_in) |
|
self._rot_len = len(x) |
|
ph = np.exp(-2j * np.pi * ((self.f_shift * self.n_in / self.fs_in) % 1.0)) |
|
x = x * (ph * self._rot[:len(x)]) |
|
if self.D == 1: |
|
out = x |
|
else: |
|
buf = np.concatenate([self.tail, x]) |
|
self.tail = buf[len(buf) - (len(self.h) - 1):] |
|
out = _poly_decimate(buf, self.h, self.D, (-self.n_in) % self.D) |
|
self.n_in += len(x) |
|
self.mid.append(out) |
|
return out |
|
|
|
def finish(self): |
|
mid = np.concatenate(self.mid) if self.mid else np.zeros(0, complex) |
|
self.mid = [] |
|
if abs(self.fs_mid - self.fs_out) < 1e-6: |
|
return mid |
|
step = self.fs_mid / self.fs_out |
|
pos = np.arange(int(len(mid) / step)) * step |
|
return resample_at(mid, pos, cutoff=min(1.0, self.fs_out / self.fs_mid)) |
|
|
|
|
|
class StreamConverter: |
|
"""RateConverter for unbounded streams: same integer FIR stage, then the |
|
fractional ratio applied block by block (windowed-sinc, carrying the |
|
kernel's history across blocks), so memory stays bounded. |
|
|
|
``process(x)`` returns the modem-rate samples that are complete so far; |
|
output j is the input evaluated at mid-rate position j * fs_mid / fs_out. |
|
``delay_s`` is the FIR group delay (output time = input time - delay). |
|
""" |
|
|
|
def __init__(self, fs_in, fs_out, f_shift=0.0, passband=None): |
|
self.rc = RateConverter(fs_in, fs_out, f_shift, passband) |
|
self.fs_out = float(fs_out) |
|
self.step = self.rc.fs_mid / self.fs_out |
|
self.frac = abs(self.step - 1.0) > 1e-9 |
|
self.cutoff = min(1.0, 1.0 / self.step) |
|
self.hw = int(np.ceil(6 / self.cutoff)) + 2 # resampler support, mid samples |
|
self.buf = np.zeros(0, complex) |
|
self.b0 = 0 # mid index of buf[0] |
|
self.j = 0 # next output index |
|
self.delay_s = ((len(self.rc.h) - 1) / 2 / fs_in) if self.rc.D > 1 else 0.0 |
|
|
|
def process(self, x): |
|
mid = self.rc.process(x) |
|
self.rc.mid = [] # don't accumulate |
|
if not self.frac: |
|
return mid |
|
self.buf = np.concatenate([self.buf, mid]) |
|
end = self.b0 + len(self.buf) # mid samples available |
|
j_hi = int(np.floor((end - self.hw) / self.step)) # last output with full support |
|
if j_hi < self.j: |
|
return np.zeros(0, complex) |
|
pos = np.arange(self.j, j_hi + 1) * self.step - self.b0 |
|
out = resample_at(self.buf, pos, cutoff=self.cutoff) |
|
self.j = j_hi + 1 |
|
keep = int(np.floor(self.j * self.step)) - self.hw - self.b0 |
|
if keep > 0: |
|
self.buf = self.buf[keep:] |
|
self.b0 += keep |
|
return out |
|
|
|
|
|
def upsample(x, fs_in, fs_out, f_shift=0.0): |
|
"""Modem-rate -> SDR-rate for transmit, then shift up to ``f_shift`` Hz.""" |
|
step = fs_in / fs_out |
|
pos = np.arange(int(len(x) / step)) * step |
|
y = resample_at(x, pos, cutoff=1.0) if fs_out != fs_in else np.asarray(x, complex) |
|
if f_shift: |
|
y = y * np.exp(2j * np.pi * f_shift * np.arange(len(y)) / fs_out) |
|
return y
|
|
|