From 310330b588911c07f3717cc03abb203e876f31fe Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 16:57:27 -0400 Subject: [PATCH 01/10] wr_dat_file: do not modify contents of the d_signal argument. When writing to an output file, it may be necessary to transform the data in various ways, which is often most efficient to do by modifying the array in-place. However, for API cleanliness, the wr_dat_file function shouldn't modify its argument. Instead, the function itself should make a copy of the array if necessary. (In the future, we may ultimately be able to avoid a copy if the array is already in the correct format.) --- wfdb/io/_signal.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index d3d26439..b0206b10 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -954,8 +954,7 @@ def wr_dat_files(self, expanded=False, write_dir=""): write_dir=write_dir, ) else: - # Create a copy to prevent overwrite - dsig = self.d_signal.copy() + dsig = self.d_signal for fn in file_names: wr_dat_file( fn, @@ -2301,6 +2300,9 @@ def wr_dat_file( for framenum in range(spf): d_signal[:, expand_ch] = e_d_signal[ch][framenum::spf] expand_ch = expand_ch + 1 + else: + # Create a copy to prevent overwrite + d_signal = d_signal.copy() # This n_sig is used for making list items. # Does not necessarily represent number of signals (ie. for expanded=True) From 13b7633c1333020ae45274b66039ffd18791d8ab Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 17:06:19 -0400 Subject: [PATCH 02/10] wr_dat_file: fix documentation of arguments. The d_signal argument is always a 2D array, and is only used if expanded is false. The e_d_signal argument is always a list of 1D arrays, and is only used if expanded is true. --- wfdb/io/_signal.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index b0206b10..403f2664 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -2266,16 +2266,15 @@ def wr_dat_file( fmt : str WFDB fmt of the dat file. d_signal : ndarray - The digital conversion of the signal. Either a 2d numpy - array or a list of 1d numpy arrays. + The digital conversion of the signal, as a 2d numpy array. byte_offset : int The byte offset of the dat file. expanded : bool, optional Whether to transform the `e_d_signal` attribute (True) or the `d_signal` attribute (False). - d_signal : ndarray, optional - The expanded digital conversion of the signal. Either a 2d numpy - array or a list of 1d numpy arrays. + e_d_signal : ndarray, optional + The expanded digital conversion of the signal, as a list of 1d + numpy arrays. samps_per_frame : list, optional The samples/frame for each signal of the dat file. write_dir : str, optional From 132d097f10904e8e1ac7ae350877750624c38e31 Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Tue, 6 Sep 2022 14:38:18 -0400 Subject: [PATCH 03/10] wr_dat_file: consistently set n_sig and tsamps_per_frame. Previously the variable "n_sig" was used confusingly to mean different things at different points in the function. Instead, set these variables consistently: n_sig is the number of signals, samps_per_frame is the number of samples per frame of each signal, and tsamps_per_frame is the sum of samps_per_frame (which is also the number of columns of the reshaped d_signal array.) --- wfdb/io/_signal.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index 403f2664..a6111f35 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -2303,9 +2303,13 @@ def wr_dat_file( # Create a copy to prevent overwrite d_signal = d_signal.copy() - # This n_sig is used for making list items. - # Does not necessarily represent number of signals (ie. for expanded=True) - n_sig = d_signal.shape[1] + # Non-expanded format always has 1 sample per frame + n_sig = d_signal.shape[1] + samps_per_frame = [1] * n_sig + + # Total number of samples per frame (equal to number of signals if + # expanded=False, but may be greater for expanded=True) + tsamps_per_frame = d_signal.shape[1] if fmt == "80": # convert to 8 bit offset binary form @@ -2363,8 +2367,8 @@ def wr_dat_file( # convert to 16 bit two's complement d_signal[d_signal < 0] = d_signal[d_signal < 0] + 65536 # Split samples into separate bytes using binary masks - b1 = d_signal & [255] * n_sig - b2 = (d_signal & [65280] * n_sig) >> 8 + b1 = d_signal & [255] * tsamps_per_frame + b2 = (d_signal & [65280] * tsamps_per_frame) >> 8 # Interweave the bytes so that the same samples' bytes are consecutive b1 = b1.reshape((-1, 1)) b2 = b2.reshape((-1, 1)) @@ -2376,9 +2380,9 @@ def wr_dat_file( # convert to 24 bit two's complement d_signal[d_signal < 0] = d_signal[d_signal < 0] + 16777216 # Split samples into separate bytes using binary masks - b1 = d_signal & [255] * n_sig - b2 = (d_signal & [65280] * n_sig) >> 8 - b3 = (d_signal & [16711680] * n_sig) >> 16 + b1 = d_signal & [255] * tsamps_per_frame + b2 = (d_signal & [65280] * tsamps_per_frame) >> 8 + b3 = (d_signal & [16711680] * tsamps_per_frame) >> 16 # Interweave the bytes so that the same samples' bytes are consecutive b1 = b1.reshape((-1, 1)) b2 = b2.reshape((-1, 1)) @@ -2392,10 +2396,10 @@ def wr_dat_file( # convert to 32 bit two's complement d_signal[d_signal < 0] = d_signal[d_signal < 0] + 4294967296 # Split samples into separate bytes using binary masks - b1 = d_signal & [255] * n_sig - b2 = (d_signal & [65280] * n_sig) >> 8 - b3 = (d_signal & [16711680] * n_sig) >> 16 - b4 = (d_signal & [4278190080] * n_sig) >> 24 + b1 = d_signal & [255] * tsamps_per_frame + b2 = (d_signal & [65280] * tsamps_per_frame) >> 8 + b3 = (d_signal & [16711680] * tsamps_per_frame) >> 16 + b4 = (d_signal & [4278190080] * tsamps_per_frame) >> 24 # Interweave the bytes so that the same samples' bytes are consecutive b1 = b1.reshape((-1, 1)) b2 = b2.reshape((-1, 1)) From 211329c5b2118a76a622ef0b884fee44aeedee62 Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Tue, 6 Sep 2022 14:45:43 -0400 Subject: [PATCH 04/10] wr_dat_file: sanity-check samps_per_frame and e_d_signal. If expanded is true, then e_d_signal and samps_per_frame must have the same length (one entry in e_d_signal corresponds to one entry in samps_per_frame). The length of each channel must be a multiple of the number of samples per frame, and every channel must have the same number of frames. Add sanity checks to verify that the values provided by the caller make sense. --- wfdb/io/_signal.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index a6111f35..1caff0ca 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -2288,7 +2288,14 @@ def wr_dat_file( # Combine list of arrays into single array if expanded: n_sig = len(e_d_signal) - sig_len = int(len(e_d_signal[0]) / samps_per_frame[0]) + if len(samps_per_frame) != n_sig: + raise ValueError("mismatch between samps_per_frame and e_d_signal") + + sig_len = len(e_d_signal[0]) // samps_per_frame[0] + for sig, spf in zip(e_d_signal, samps_per_frame): + if len(sig) != sig_len * spf: + raise ValueError("mismatch in lengths of expanded signals") + # Effectively create MxN signal, with extra frame samples acting # like extra channels d_signal = np.zeros((sig_len, sum(samps_per_frame)), dtype="int64") From 7f9cd47cc3280f4e71b8d871b050c9547cfcdfb8 Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 17:49:22 -0400 Subject: [PATCH 05/10] wr_dat_file: support FLAC output formats. Writing FLAC formats, like reading FLAC formats, is done using the soundfile package, which is currently treated as an optional dependency (since on some systems it requires an external copy of libsndfile.) This code performs, in effect, the inverse of _rd_compressed_file. As with _rd_compressed_file, this has room to be optimized somewhat. --- wfdb/io/_signal.py | 51 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index 1caff0ca..5e1455db 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -2285,6 +2285,8 @@ def wr_dat_file( N/A """ + file_path = os.path.join(write_dir, file_name) + # Combine list of arrays into single array if expanded: n_sig = len(e_d_signal) @@ -2416,9 +2418,54 @@ def wr_dat_file( b_write = b_write.reshape((1, -1))[0] # Convert to un_signed 8 bit dtype to write b_write = b_write.astype("uint8") + + elif fmt in ("508", "516", "524"): + import soundfile + + if any(spf != samps_per_frame[0] for spf in samps_per_frame): + raise ValueError( + "All channels in a FLAC signal file must have the same " + "sampling rate and samples per frame" + ) + if n_sig > 8: + raise ValueError( + "A single FLAC signal file cannot contain more than 8 channels" + ) + + d_signal = d_signal.reshape(-1, n_sig, samps_per_frame[0]) + d_signal = d_signal.transpose(0, 2, 1) + d_signal = d_signal.reshape(-1, n_sig) + + if fmt == "508": + d_signal = d_signal.astype("int16") + np.left_shift(d_signal, 8, out=d_signal) + subtype = "PCM_S8" + elif fmt == "516": + d_signal = d_signal.astype("int16") + subtype = "PCM_16" + elif fmt == "524": + d_signal = d_signal.astype("int32") + np.left_shift(d_signal, 8, out=d_signal) + subtype = "PCM_24" + else: + raise ValueError(f"unknown format ({fmt})") + + sf = soundfile.SoundFile( + file_path, + mode="w", + samplerate=96000, + channels=n_sig, + subtype=subtype, + format="FLAC", + ) + with sf: + sf.write(d_signal) + return + else: raise ValueError( - "This library currently only supports writing the following formats: 80, 16, 24, 32" + "This library currently only supports writing the " + "following formats: 80, 16, 24, 32, 508, 516, 524" ) # Byte offset in the file @@ -2433,7 +2480,7 @@ def wr_dat_file( b_write = np.append(np.zeros(byte_offset, dtype="uint8"), b_write) # Write the bytes to the file - with open(os.path.join(write_dir, file_name), "wb") as f: + with open(file_path, "wb") as f: b_write.tofile(f) From de1461d4f581b480c47d605d177976e62542504f Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 17:46:19 -0400 Subject: [PATCH 06/10] Record.check_field: sig_name strings may contain whitespace. It is permitted and not unheard of for signal name strings to contain space characters. This should be allowed, provided that the string does not start or end with a space. All control characters should be disallowed. --- wfdb/io/record.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/wfdb/io/record.py b/wfdb/io/record.py index ba145407..399b69ff 100644 --- a/wfdb/io/record.py +++ b/wfdb/io/record.py @@ -520,9 +520,15 @@ def check_field(self, field, required_channels="all"): "block_size values must be non-negative integers" ) elif field == "sig_name": - if re.search(r"\s", item[ch]): + if item[ch][:1].isspace() or item[ch][-1:].isspace(): + raise ValueError( + "sig_name strings may not begin or end with " + "whitespace." + ) + if re.search(r"[\x00-\x1f\x7f-\x9f]", item[ch]): raise ValueError( - "sig_name strings may not contain whitespaces." + "sig_name strings may not contain " + "control characters." ) if len(set(item)) != len(item): raise ValueError("sig_name strings must be unique.") From d1d26ba5e5a3be87f8f2db41921612e09ec88bdf Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 17:54:18 -0400 Subject: [PATCH 07/10] Record.set_default: use multiple signal files if needed. It may be necessary to use multiple signal files in various cases: - signals are to be written in different formats (e.g. 8-bit signals stored in format 80 while 12-bit signals are stored in format 212) - multi-frequency signals are to be written in compressed format (FLAC supports only one frequency per signal file) - more than 8 signals are to be written in compressed format (FLAC supports a maximum of 8 channels) In each of these cases, if the caller didn't specify signal file names explicitly, generate signal file files by adding a numeric suffix after the record name. --- wfdb/io/_header.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/wfdb/io/_header.py b/wfdb/io/_header.py index 7322ee87..2bddd01a 100644 --- a/wfdb/io/_header.py +++ b/wfdb/io/_header.py @@ -361,6 +361,35 @@ def get_write_fields(self): return rec_write_fields, sig_write_fields + def _auto_signal_file_names(self): + fmt = self.fmt or [None] * self.n_sig + spf = self.samps_per_frame or [None] * self.n_sig + num_groups = 0 + group_number = [] + prev_fmt = prev_spf = None + channels_in_group = 0 + + for ch_fmt, ch_spf in zip(fmt, spf): + if ch_fmt != prev_fmt: + num_groups += 1 + channels_in_group = 0 + elif ch_fmt in ("508", "516", "524"): + if channels_in_group >= 8 or ch_spf != prev_spf: + num_groups += 1 + channels_in_group = 0 + group_number.append(num_groups) + prev_fmt = ch_fmt + prev_spf = ch_spf + + if num_groups < 2: + return [self.record_name + ".dat"] * self.n_sig + else: + digits = len(str(group_number[-1])) + return [ + self.record_name + "_" + str(g).rjust(digits, "0") + ".dat" + for g in group_number + ] + def set_default(self, field): """ Set the object's attribute to its default value if it is missing @@ -394,7 +423,7 @@ def set_default(self, field): # Specific dynamic case if field == "file_name" and self.file_name is None: - self.file_name = self.n_sig * [self.record_name + ".dat"] + self.file_name = self._auto_signal_file_names() return item = getattr(self, field) From f61d3a19f803438f8a3029b4633e010c61bc8a73 Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Tue, 6 Sep 2022 14:53:10 -0400 Subject: [PATCH 08/10] Record.wr_dat_files: handle multiple files in expanded format. When writing multiple signal files in expanded format, the samps_per_frame passed to wr_dat_file must be the list of samples-per-frame *for that particular dat file* (corresponding to the e_d_signal argument), not the samples-per-frame of the whole record. --- wfdb/io/_signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wfdb/io/_signal.py b/wfdb/io/_signal.py index 5e1455db..61abb568 100644 --- a/wfdb/io/_signal.py +++ b/wfdb/io/_signal.py @@ -950,7 +950,7 @@ def wr_dat_files(self, expanded=False, write_dir=""): dat_offsets[fn], True, [self.e_d_signal[ch] for ch in dat_channels[fn]], - self.samps_per_frame, + [self.samps_per_frame[ch] for ch in dat_channels[fn]], write_dir=write_dir, ) else: From 751ebef3192328c42a770dd14c0aad6966e5715a Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Fri, 2 Sep 2022 17:53:26 -0400 Subject: [PATCH 09/10] test_record: test writing FLAC output files. --- tests/test_record.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_record.py b/tests/test_record.py index 3d9d32a7..c18c19fb 100644 --- a/tests/test_record.py +++ b/tests/test_record.py @@ -219,7 +219,7 @@ def test_1f(self): "Mismatch in %s" % name, ) - def test_read_flac(self): + def test_read_write_flac(self): """ All FLAC formats, multiple signal files in one record. @@ -250,6 +250,11 @@ def test_read_flac(self): f"Mismatch in {name}", ) + # Test file writing + record.wrsamp() + record_write = wfdb.rdrecord("flacformats", physical=False) + assert record == record_write + def test_read_flac_longduration(self): """ Three signals multiplexed in a FLAC file, over 2**24 samples. @@ -628,6 +633,10 @@ def tearDownClass(cls): "100_3chan.hea", "a103l.hea", "a103l.mat", + "flacformats.d0", + "flacformats.d1", + "flacformats.d2", + "flacformats.hea", "s0010_re.dat", "s0010_re.hea", "s0010_re.xyz", From b0a3d8661a9f3868c438060380a63bd17fb5380d Mon Sep 17 00:00:00 2001 From: Benjamin Moody Date: Tue, 6 Sep 2022 15:15:21 -0400 Subject: [PATCH 10/10] test_record: test reading and writing a multi-frequency FLAC record. --- tests/test_record.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_record.py b/tests/test_record.py index c18c19fb..a6c57e79 100644 --- a/tests/test_record.py +++ b/tests/test_record.py @@ -255,6 +255,23 @@ def test_read_write_flac(self): record_write = wfdb.rdrecord("flacformats", physical=False) assert record == record_write + def test_read_write_flac_multifrequency(self): + """ + Format 516 with multiple signal files and variable samples per frame. + """ + # Check that we can read a record and write it out again + record = wfdb.rdrecord( + "sample-data/mixedsignals", + physical=False, + smooth_frames=False, + ) + record.wrsamp(expanded=True) + + # Check that result matches the original + record = wfdb.rdrecord("sample-data/mixedsignals", smooth_frames=False) + record_write = wfdb.rdrecord("mixedsignals", smooth_frames=False) + assert record == record_write + def test_read_flac_longduration(self): """ Three signals multiplexed in a FLAC file, over 2**24 samples. @@ -637,6 +654,10 @@ def tearDownClass(cls): "flacformats.d1", "flacformats.d2", "flacformats.hea", + "mixedsignals.hea", + "mixedsignals_e.dat", + "mixedsignals_p.dat", + "mixedsignals_r.dat", "s0010_re.dat", "s0010_re.hea", "s0010_re.xyz",