diff --git a/lddecode/core.py b/lddecode/core.py index 588e71c36..4be7ba5fb 100644 --- a/lddecode/core.py +++ b/lddecode/core.py @@ -15,8 +15,7 @@ import numba from numba import njit -# Use standard numpy fft, since it's thread-safe -import numpy.fft as npfft +import scipy.fft as npfft # internal libraries @@ -51,22 +50,18 @@ def profile(fn): return fn -BLOCKSIZE = 32 * 1024 - -def calclinelen(SP, mult, mhz): - if type(mhz) == str: - mhz = SP[mhz] +# This is the size of each block of data which is processed in +# parallel. The beginning and end are cut off so that there's +# no distortion from the FFT and filtering. - return int(nb_round(SP["line_period"] * mhz * mult)) +BLOCKSIZE = 32 * 1024 +# These are constant, system-level parameters for PAL and NTSC -# states for first field of validpulses (second field is pulse #) -HSYNC, EQPL1, VSYNC, EQPL2 = range(4) - -# These are invariant parameters for PAL and NTSC SysParams_NTSC = { - "fsc_mhz": (315.0 / np.double(88.0)), - "pilot_mhz": (315.0 / 88.0), + "fsc_mhz": 315.0 / np.double(88.0), + # NTSC LD doesn't have a pilot signal, so just recycle FSC + "pilot_mhz": 315.0 / np.double(88.0), "frame_lines": 525, "field_lines": (263, 262), "ire0": 8100000, @@ -80,7 +75,6 @@ def calclinelen(SP, mult, mhz): # On AC3 disks, the right channel is replaced by a QPSK 2.88mhz channel "audio_rfreq_AC3": 2880000, "colorBurstUS": (5.3, 7.8), - "activeVideoUS": (9.45, 63.555 - 1.0), # Known-good area for computing black SNR - for NTSC pull from VSYNC # tuple: (line, beginning, length) "blacksnr_slice": (1, 10, 20), @@ -91,7 +85,7 @@ def calclinelen(SP, mult, mhz): "hsyncPulseUS": 4.7, "eqPulseUS": 2.3, "vsyncPulseUS": 27.1, - # What 0 IRE/0V should be in digitaloutput + # What 0 IRE/0V should be in 16-bit digital output "outputZero": 1024, "fieldPhases": 4, # Likely locations of solid white in VITS on LD's (line, start, length) @@ -103,6 +97,16 @@ def calclinelen(SP, mult, mhz): "LD_VITS_code_slices": [(16, 12, 48, 85), (17, 12, 48, 85), (10, 13, 39, 85)], } +# Calculate the exact line length for a given situation (such as +# 4FSC) +def calclinelen(SysParams, mult, mhz): + if type(mhz) == str: + mhz = SysParams[mhz] + + return int(nb_round(SysParams["line_period"] * mhz * mult)) + +# Compute dictionary entries for things tht use FSC, etc. + # In color NTSC, the line period was changed from 63.5 to 227.5 color cycles, # which works out to 63.555(with a bar on top) usec SysParams_NTSC["line_period"] = 1 / (SysParams_NTSC["fsc_mhz"] / np.double(227.5)) @@ -156,9 +160,7 @@ def calclinelen(SP, mult, mhz): SysParams_PAL["vsync_ire"] = -0.3 * (100 / 0.7) -# RFParams are tunable - -RFParams_NTSC = { +FilterParams_NTSC = { # The audio notch filters are important with DD v3.0+ boards "audio_notchwidth": 350000, "audio_notchorder": 2, @@ -184,12 +186,12 @@ def calclinelen(SP, mult, mhz): } # Settings for use with noisier disks -RFParams_NTSC_lowband = RFParams_NTSC.copy() -RFParams_NTSC_lowband['video_bpf_low'] = 3800000 -RFParams_NTSC_lowband['video_bpf_high'] = 12500000 -RFParams_NTSC_lowband['video_lpf_freq'] = 4200000 +FilterParams_NTSC_lowband = FilterParams_NTSC.copy() +FilterParams_NTSC_lowband['video_bpf_low'] = 3800000 +FilterParams_NTSC_lowband['video_bpf_high'] = 12500000 +FilterParams_NTSC_lowband['video_lpf_freq'] = 4200000 -RFParams_PAL = { +FilterParams_PAL = { # The audio notch filters are important with DD v3.0+ boards "audio_notchwidth": 200000, "audio_notchorder": 2, @@ -212,11 +214,11 @@ def calclinelen(SP, mult, mhz): } # Settings for use with noisier disks -RFParams_PAL_lowband = RFParams_PAL.copy() -RFParams_PAL_lowband['video_bpf_low'] = 3200000 -RFParams_PAL_lowband['video_bpf_high'] = 13000000 -RFParams_PAL_lowband['video_bpf_order'] = 13000000 -RFParams_PAL_lowband['video_lpf_freq'] = 4800000 +FilterParams_PAL_lowband = FilterParams_PAL.copy() +FilterParams_PAL_lowband['video_bpf_low'] = 3200000 +FilterParams_PAL_lowband['video_bpf_high'] = 13000000 +FilterParams_PAL_lowband['video_bpf_order'] = 13000000 +FilterParams_PAL_lowband['video_lpf_freq'] = 4800000 class RFDecode: """The core RF decoding code. @@ -251,19 +253,22 @@ class RFDecode: def __init__( self, - inputfreq=40, - system="NTSC", - blocklen=BLOCKSIZE, - decode_digital_audio=False, - decode_analog_audio=0, - has_analog_audio=True, - extra_options={}, + inputfreq = 40, + system = "NTSC", + blocklen = BLOCKSIZE, + decode_digital_audio = False, + decode_analog_audio = 0, + has_analog_audio = True, + extra_options = {}, + decoder_params_override = {}, ): """Initialize the RF decoder object. - inputfreq -- frequency of raw RF data (in Msps) - system -- Which system is in use (PAL or NTSC) - blocklen -- Block length for FFT processing + inputfreq -- frequency of raw RF data (in Msps) + WARNING: only tested at 40Msps w/other frequencies + scaled to 40 in utils.py. + system -- Which system is in use (PAL or NTSC) + blocklen -- Block length for FFT processing decode_digital_audio -- Whether to apply EFM filtering decode_analog_audio -- Whether or not to decode analog(ue) audio has_analog_audio -- Whether or not analog(ue) audio channels are on the disk @@ -276,12 +281,13 @@ def __init__( """ - self.blocklen = blocklen - self.blockcut = 1024 # ??? + self.blocklen = blocklen + self.blockcut = 1024 self.blockcut_end = 0 - self.system = system + + self.system = system - self.setupcount = 0 + self.setupcount = 0 self.NTSC_ColorNotchFilter = extra_options.get("NTSC_ColorNotchFilter", False) self.PAL_V4300D_NotchFilter = extra_options.get("PAL_V4300D_NotchFilter", False) @@ -299,20 +305,23 @@ def __init__( if system == "NTSC": self.SysParams = copy.deepcopy(SysParams_NTSC) if lowband: - self.DecoderParams = copy.deepcopy(RFParams_NTSC_lowband) + self.DecoderParams = copy.deepcopy(FilterParams_NTSC_lowband) else: - self.DecoderParams = copy.deepcopy(RFParams_NTSC) + self.DecoderParams = copy.deepcopy(FilterParams_NTSC) elif system == "PAL": self.SysParams = copy.deepcopy(SysParams_PAL) if lowband: - self.DecoderParams = copy.deepcopy(RFParams_PAL_lowband) + self.DecoderParams = copy.deepcopy(FilterParams_PAL_lowband) else: - self.DecoderParams = copy.deepcopy(RFParams_PAL) + self.DecoderParams = copy.deepcopy(FilterParams_PAL) # Make (intentionally) mutable copies of HZ<->IRE levels for irekey in ['ire0', 'hz_ire', 'vsync_ire']: self.DecoderParams[irekey] = self.SysParams[irekey] + for k in decoder_params_override.keys(): + self.DecoderParams[k] = decoder_params_override[k] + self.SysParams["analog_audio"] = has_analog_audio self.SysParams["AC3"] = extra_options.get("AC3", False) if self.SysParams["AC3"]: @@ -326,7 +335,7 @@ def __init__( # note that deemp[0] is the t1 (high freuqency) coefficient, and # deemp[1] is the t2 (low frequency) one. These are passed in as - # microseconds, but need to be converted to seconds. + # microseconds, but are converted to seconds here. deemp_low, deemp_high = extra_options.get("deemp_coeff", (0, 0)) if deemp_low > 0: @@ -348,7 +357,7 @@ def __init__( self.hsync_tolerance = 0.4 self.decode_digital_audio = decode_digital_audio - self.decode_analog_audio = decode_analog_audio + self.decode_analog_audio = decode_analog_audio self.computefilters() @@ -374,27 +383,22 @@ def computefilters(self): if self.SysParams['AC3']: apass = 288000 * .5 - # fpass = lambda apass: [(self.SysParams['audio_rfreq_AC3'] - apass) / self.freq_hz_half, - (self.SysParams['audio_rfreq_AC3'] + apass) / self.freq_hz_half] - - # Need to clean these up - # self.Filters['AC3_fir'] = [sps.firwin(257, fpass(apass), pass_zero=False), [1.0]] - # XXX Made into IIR for some reason, check commit history here - self.Filters['AC3_fir'] = sps.butter(3, fpass(apass), btype='bandpass') + (self.SysParams['audio_rfreq_AC3'] + apass) / self.freq_hz_half] # This analog audio bandpass filter is an approximation of # http://sim.okawa-denshi.jp/en/RLCtool.php with resistor 2200ohm, # inductor 180uH, and cap 27pF (taken from Pioneer service manuals) # self.Filters['AC3_iir'] = sps.butter(5, [1.48/20, 3.45/20], btype='bandpass') - # empirically determined - self.Filters['AC3_iir'] = sps.butter(3, [(2.88-.5)/20, (2.88+.5)/20], btype='bandpass') + # However, the above didn't work, and we wound up with two IIR filters + self.Filters['AC3_bp1'] = sps.butter(3, [(2.88-.5)/20, (2.88+.5)/20], btype='bandpass') + self.Filters['AC3_bp2'] = sps.butter(3, fpass(apass), btype='bandpass') - firfilt = filtfft(self.Filters['AC3_fir'], self.blocklen) - iirfilt = filtfft(self.Filters['AC3_iir'], self.blocklen) + filt1 = filtfft(self.Filters['AC3_bp1'], self.blocklen) + filt2 = filtfft(self.Filters['AC3_bp2'], self.blocklen) - self.Filters['AC3'] = iirfilt * firfilt + self.Filters['AC3'] = filt1 * filt2 self.computedelays() @@ -491,7 +495,7 @@ def computevideofilters(self): # Start building up the combined FFT filter using the BPF SF["RFVideo"] = filtfft(filt_rfvideo, self.blocklen) - # Notch filters for analog audio. DdD captures on NTSC need this. + # Notch filters for analog audio RF. DdD captures on NTSC need this. if SP["analog_audio"] and self.system == "NTSC": cut_left = sps.butter( DP["audio_notchorder"], @@ -542,7 +546,7 @@ def computevideofilters(self): # additional filters: 0.5mhz and color burst # Using an FIR filter here to get a known delay F0_5 = sps.firwin(65, [0.5 / self.freq_half], pass_zero=True) - SF["F05_offset"] = 32 + SF["F05_offset"] = 32 # Reduced because filtfft is half-strength on FIR F0_5_fft = filtfft((F0_5, [1.0]), self.blocklen) SF["FVideo05"] = SF["Fvideo_lpf"] * SF["Fdeemp"] * F0_5_fft @@ -927,6 +931,11 @@ def computedelays(self, mtf_level=0): return fakedecode, fakeoutput_emp +''' The DemodCache class keeps track of each block of data, from the raw + input to the demodulated output. This is threaded code and therefore + a bit of a mess, full of queues and locks and memory copies. +''' + class DemodCache: def __init__( self, @@ -943,38 +952,37 @@ def __init__( self.rf = rf self.rf_args = rf_args - self.currentMTF = 1 - self.MTF_tolerance = MTF_tolerance + self.currentMTF = 1 + self.MTF_tolerance = MTF_tolerance - self.blocksize = self.rf.blocklen - (self.rf.blockcut + self.rf.blockcut_end) + self.blocksize = self.rf.blocklen - (self.rf.blockcut + self.rf.blockcut_end) # Cache dictionary - key is block #, which holds data for that block - self.lrusize = cachesize + self.lrusize = cachesize # should be in self.rf, but may not be computed yet self.bytes_per_field = int(self.rf.freq_hz / (self.rf.SysParams["FPS"] * 2)) + 1 - self.prefetch = int((self.bytes_per_field * 2) / self.blocksize) + 4 + self.prefetch = int((self.bytes_per_field * 2) / self.blocksize) + 4 - self.lru = [] + self.lru = [] - self.lock = threading.Lock() - self.blocks = {} + self.lock = threading.Lock() - self.block_status = {} + self.blocks = {} - self.q_in = Queue() - self.q_out = Queue() - self.waiting = set() - self.q_out_event = threading.Event() + self.q_in = Queue() + self.q_out = Queue() + self.waiting = set() + self.q_out_event = threading.Event() - self.threadpipes = [] - self.threads = [] + self.threadpipes = [] + self.threads = [] - self.request = 0 - self.ended = False + self.request = 0 + self.ended = False - self.deqeue_thread = threading.Thread(target=self.dequeue, daemon=True) - num_worker_threads = max(num_worker_threads - 1, 1) + self.deqeue_thread = threading.Thread(target=self.dequeue, daemon=True) + num_worker_threads = max(num_worker_threads - 1, 1) for i in range(num_worker_threads): t = threading.Thread( @@ -1014,8 +1022,6 @@ def prune_cache(self): for k in self.lru[self.lrusize :]: if k in self.blocks: del self.blocks[k] - if k in self.block_status: - self.block_status[k] self.lru = self.lru[: self.lrusize] @@ -1024,26 +1030,28 @@ def flush_demod(self, first_block = 0, prefetch_only=False): blocks_toredo = [] with self.lock: - for k in self.block_status.keys(): + for k in self.blocks.keys(): if k < first_block: continue - if prefetch_only and self.block_status[k]['prefetch'] == False: + if prefetch_only and self.blocks[k]['prefetch'] == False: continue - if self.block_status[k]['prefetch'] == True: + if self.blocks[k]['prefetch'] == True: blocks_toredo.append(k) - self.block_status[k] = {'MTF': -1, 'request': -1, 'waiting': False, 'prefetch': False} + self.blocks[k]['MTF'] = -1 + self.blocks[k]['request'] = -1 + self.blocks[k]['waiting'] = False + self.blocks[k]['prefetch'] = False - for k in self.blocks.keys(): - if k < first_block or self.blocks[k] is None or 'demod' not in self.blocks[k]: + if 'demod' not in self.blocks[k]: continue if k not in blocks_toredo: blocks_toredo.append(k) - del self.blocks[k]["demod"] + del self.blocks[k]['demod'] return blocks_toredo @@ -1129,19 +1137,16 @@ def doread(self, blocknums, MTF, redo=False, prefetch=False): reached_end = True break - waiting = ( - self.block_status[b].get("waiting", False) - if b in self.block_status - else False - ) + waiting = False + if b in self.blocks: + waiting = self.blocks[b].get("waiting", False) # Until the block is actually ready, this comparison will hit an unknown key if ( not redo and not waiting and "request" in self.blocks[b] - and "request" in self.block_status[b] - and self.blocks[b]["request"] == self.block_status[b]["request"] + and "demod" in self.blocks[b] ): continue @@ -1155,12 +1160,10 @@ def doread(self, blocknums, MTF, redo=False, prefetch=False): self.waiting.add(b) for b in queuelist: - self.block_status[b] = { - "MTF": MTF, - "waiting": True, - "request": self.request, - "prefetch": prefetch, - } + self.blocks[b]['MTF'] = MTF + self.blocks[b]['request'] = self.request + self.blocks[b]['waiting'] = True + self.blocks[b]['prefetch'] = prefetch self.q_in.put(("DEMOD", b, self.blocks[b], MTF, self.request)) self.q_out_event.clear() @@ -1184,13 +1187,12 @@ def dequeue(self): self.q_in.put((blocknum, self.blocks[blocknum], self.currentMTF, self.request)) continue - if item['request'] == self.block_status[blocknum]['request']: + if item['request'] == self.blocks[blocknum]['request']: for k in item.keys(): self.blocks[blocknum][k] = item[k] if 'demod' in item.keys(): - if self.block_status[blocknum]['waiting']: - self.block_status[blocknum]['waiting'] = False + self.blocks[blocknum]['waiting'] = False if blocknum in self.waiting: self.waiting.remove(blocknum) @@ -1265,7 +1267,6 @@ def read(self, begin, length, MTF=0, getraw = False, forceredo=False): return rv def setparams(self, params): - # XXX: This should flush out the data, but right now this isn't used at all for p in self.threadpipes: p[0].send(("NEWPARAMS", params)) @@ -1432,6 +1433,9 @@ def downscale_audio(audio, lineinfo, rf, linecount, timeoffset=0, freq=44100, rv return output16, arange[-1] - frametime +# XXX: bring this enum-like thing into Field +# state order: HSYNC -> EQPUL1 -> VSYNC -> EQPUL2 -> HSYNC +HSYNC, EQPL1, VSYNC, EQPL2 = range(4) # The Field class contains common features used by NTSC and PAL class Field: @@ -1680,9 +1684,6 @@ def run_vblank_state_machine(self, pulses, LT): # ... and state length is set by the phase transition to set above (in H) state_length = None - # state order: HSYNC -> EQPUL1 -> VSYNC -> EQPUL2 -> HSYNC - HSYNC, EQPL1, VSYNC, EQPL2 = range(4) - for p in pulses: spulse = None @@ -1759,8 +1760,6 @@ def run_vblank_state_machine(self, pulses, LT): def refinepulses(self): self.LT = self.get_timings() - HSYNC, EQPL1, VSYNC, EQPL2 = range(4) - i = 0 valid_pulses = [] num_vblanks = 0 @@ -1848,8 +1847,6 @@ def processVBlank(self, validpulses, start, limit=None): loc_presync = validpulses[firstblank - 1][1].start - HSYNC, EQPL1, VSYNC, EQPL2 = range(4) - pt = np.array([v[0] for v in validpulses[firstblank:]]) pstart = np.array([v[1].start for v in validpulses[firstblank:]]) plen = np.array([v[1].len for v in validpulses[firstblank:]]) @@ -2418,11 +2415,10 @@ def fix_badlines(self, linelocs_in, linelocs_backup_in=None): for l in np.where(self.linebad)[0]: prevgood = l - 1 - nextgood = l + 1 - while prevgood >= 0 and self.linebad[prevgood]: prevgood -= 1 + nextgood = l + 1 while nextgood < len(linelocs) and self.linebad[nextgood]: nextgood += 1 @@ -2439,6 +2435,7 @@ def computewow(self, lineinfo): for l in range(0, len(wow) - 1): wow[l] = self.get_linelen(l) / self.inlinelen + # smooth out wow in the sync area for l in range(self.lineoffset, self.lineoffset + 10): wow[l] = np.median(wow[l : l + 4]) @@ -2483,8 +2480,8 @@ def downscale( # Finally convert to a time value audio_offset = -audsamp_offset * (self.rf.SysParams['line_period'] / 10000000) - else: + # Either analog audio is disabled, or we're using hsync-locked sampling audio_offset = 0 audio_thread = None @@ -2505,8 +2502,6 @@ def downscale( # self.lineoffset is an adjustment for 0-based lines *before* downscaling so add 1 here lineoffset = self.lineoffset + 1 - #print(lineinfo[linesout] - lineinfo[1]) - for l in range(lineoffset, linesout + lineoffset): if lineinfo[l + 1] > lineinfo[l]: scaled = scale( @@ -2709,8 +2704,6 @@ def dropout_detect_demod(self): iserr[:int(f.linelocs[f.lineoffset + 1])] = False iserr[int(f.linelocs[f.lineoffset + f.linecount + 1]):] = False - #print(iserr1.sum(), iserr2.sum(), iserr3.sum(), iserr_rf.sum(), iserr.sum()) - return iserr @profile @@ -3323,6 +3316,7 @@ def __init__( threads=4, inputfreq=40, extra_options={}, + DecoderParamsOverride={} ): global logger self.logger = _logger @@ -3411,6 +3405,7 @@ def __init__( 'has_analog_audio':self.has_analog_audio, 'extra_options':extra_options, 'blocklen': 32 * 1024, + 'decoder_params_override': DecoderParamsOverride } self.rf = RFDecode(**self.rf_opts) diff --git a/lddecode/main.py b/lddecode/main.py index 069607126..90f26070b 100644 --- a/lddecode/main.py +++ b/lddecode/main.py @@ -328,6 +328,19 @@ def main(args=None): version = get_git_info() logger.debug("ld-decode branch " + version[0] + " build " + version[1]) + DecoderParamsOverride = {} + if args.vbpf_low is not None: + DecoderParamsOverride["video_bpf_low"] = args.vbpf_low * 1000000 + + if args.vbpf_high is not None: + DecoderParamsOverride["video_bpf_high"] = args.vbpf_high * 1000000 + + if args.vlpf is not None: + DecoderParamsOverride["video_lpf_freq"] = args.vlpf * 1000000 + + if args.vlpf_order >= 1: + DecoderParamsOverride["video_lpf_order"] = args.vlpf_order + ldd = LDdecode( filename, outname, @@ -340,6 +353,7 @@ def main(args=None): doDOD=not args.nodod, threads=args.threads, extra_options=extra_options, + DecoderParamsOverride=DecoderParamsOverride, ) signal.signal(signal.SIGINT, original_sigint_handler) @@ -359,22 +373,6 @@ def main(args=None): print("ERROR: Seeking failed", file=sys.stderr) sys.exit(1) - DecoderParamsOverride = {} - if args.vbpf_high is not None: - DecoderParamsOverride["video_bpf_high"] = args.vbpf_high * 1000000 - - if args.vbpf_low is not None: - DecoderParamsOverride["video_bpf_low"] = args.vbpf_low * 1000000 - - if args.vlpf is not None: - DecoderParamsOverride["video_lpf_freq"] = args.vlpf * 1000000 - - if args.vlpf_order >= 1: - DecoderParamsOverride["video_lpf_order"] = args.vlpf_order - - if len(DecoderParamsOverride.keys()): - ldd.demodcache.setparams(DecoderParamsOverride) - if args.verboseVITS: ldd.verboseVITS = True