From e2c48c7f9c920d359f8e486d85b51908acd3992b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20K=C3=A1kona?= Date: Tue, 5 May 2026 20:01:48 +0200 Subject: [PATCH 1/6] Add enviromental metadata to calibration file. --- dosview/__init__.py | 20 ++- dosview/calibration_widget.py | 213 ++++++++++++++++++++++--- dosview/parsers.py | 28 ++++ tests/test_calibration_csv_metadata.py | 38 +++++ tests/test_parser.py | 12 ++ xDOS-versions | 2 +- 6 files changed, 289 insertions(+), 24 deletions(-) create mode 100644 tests/test_calibration_csv_metadata.py diff --git a/dosview/__init__.py b/dosview/__init__.py index ace1395..deed07e 100644 --- a/dosview/__init__.py +++ b/dosview/__init__.py @@ -1,5 +1,6 @@ import sys import argparse +import json from PyQt5 import QtNetwork from PyQt5.QtNetwork import QLocalSocket, QLocalServer @@ -28,7 +29,12 @@ from .version import __version__ from pyqtgraph import ImageView -from .calibration_widget import CalibrationTab +from .calibration_widget import ( + CALIBRATION_CSV_METADATA_KEY, + CalibrationTab, + summarize_device, + summarize_environment, +) from .parsers import ( BaseLogParser, Airdos04CLogParser, @@ -1224,8 +1230,20 @@ def export_spectrum_csv(self): path += ".csv" import csv hist = self.data[2] + metadata = self.data[3] if len(self.data) > 3 and isinstance(self.data[3], dict) else {} + telemetry = self.data[4] if len(self.data) > 4 and isinstance(self.data[4], dict) else {} + calibration_metadata = { + "version": 1, + "environment": summarize_environment(telemetry), + "device": summarize_device(metadata), + "source_metadata": metadata, + } with open(path, "w", newline="") as f: writer = csv.writer(f) + writer.writerow([ + CALIBRATION_CSV_METADATA_KEY, + json.dumps(calibration_metadata, ensure_ascii=True, allow_nan=True), + ]) writer.writerow(["channel", "counts"]) for ch, cnt in enumerate(hist): writer.writerow([ch, int(cnt)]) diff --git a/dosview/calibration_widget.py b/dosview/calibration_widget.py index 113a53e..73195c5 100644 --- a/dosview/calibration_widget.py +++ b/dosview/calibration_widget.py @@ -11,7 +11,9 @@ import csv import json +import math import os +import re from typing import Tuple import numpy as np @@ -25,6 +27,87 @@ QMessageBox ) +from .eeprom_schema import DeviceType, KNOWN_DEVICES_BY_NAME +from .parsers import parse_file + + +CALIBRATION_CSV_METADATA_KEY = "# dosview_calibration_metadata" + + +def instrument_metadata(model_name): + full_name = str(model_name or "").strip() + known = KNOWN_DEVICES_BY_NAME.get(full_name) + if known is not None: + return { + "model": known.full_name, + "device_type": known.device_type.name, + "device_type_value": int(known.device_type), + "device_version": known.device_version, + "hardware_revision": known.hardware_revision, + } + + match = re.match(r"([A-Za-z]+)(\d+)?([A-Za-z]?)$", full_name) + family = match.group(1).upper() if match else "" + try: + device_type = DeviceType[family] if family else DeviceType.UNKNOWN + except KeyError: + device_type = DeviceType.UNKNOWN + return { + "model": full_name or "UNKNOWN", + "device_type": device_type.name, + "device_type_value": int(device_type), + "device_version": int(match.group(2)) if match and match.group(2) else None, + "hardware_revision": match.group(3).upper() if match and match.group(3) else None, + } + + +def summarize_environment(telemetry): + def _mean_or_nan(*keys): + values = [] + for key in keys: + item = telemetry.get(key) if telemetry else None + if not item: + continue + _, series = item + arr = np.asarray(series, dtype=float) + arr = arr[np.isfinite(arr)] + if arr.size: + values.extend(arr.tolist()) + return float(np.mean(values)) if values else math.nan + + return { + "temperature_celsius": _mean_or_nan("temperature_0", "temperature_1", "temperature_2"), + "relative_humidity_percent": _mean_or_nan("humidity_0", "humidity_1"), + "pressure_hpa": _mean_or_nan("pressure_3"), + } + + +def summarize_device(metadata): + metadata = metadata if isinstance(metadata, dict) else {} + info = metadata.get("log_device_info", {}) + dos = info.get("DOS", {}) + adc = info.get("ADC", {}) + dig = info.get("DIG", {}) + analog_serial = adc.get("serial") or dos.get("hw-sn") or "" + return { + "instrument": instrument_metadata(dos.get("hw-model") or metadata.get("log_info", {}).get("detector_type")), + "analog_module": { + "type": adc.get("module-type", ""), + "serial_number": analog_serial, + "configuration": adc.get("configuration", ""), + }, + "digital_module": { + "type": dig.get("module-type", ""), + "serial_number": dig.get("serial", ""), + "configuration": dig.get("configuration", ""), + }, + "firmware": { + "version": dos.get("fw-version", ""), + "commit": dos.get("fw-commit", ""), + "build_info": dos.get("fw-build_info", ""), + }, + } + class EnergyAxisItem(pg.AxisItem): """Custom axis item for displaying energy values computed from channels.""" @@ -82,7 +165,7 @@ def initUI(self): logs_layout.addWidget(self.log_table) logs_buttons = QHBoxLayout() - add_log_button = QPushButton("Add CSV") + add_log_button = QPushButton("Add spectrum") add_log_button.clicked.connect(self.add_csv_logs) remove_log_button = QPushButton("Remove") remove_log_button.clicked.connect(self.remove_selected_logs) @@ -222,9 +305,9 @@ def initUI(self): def add_csv_logs(self): file_paths, _ = QFileDialog.getOpenFileNames( self, - "Select calibration CSV files", + "Select calibration spectrum/log files", "", - "CSV files (*.csv);;All files (*)" + "Spectrum/log files (*.csv *.TXT *.txt *.npz);;CSV files (*.csv);;Log files (*.TXT *.txt);;Saved data (*.npz);;All files (*)" ) if not file_paths: return @@ -232,11 +315,11 @@ def add_csv_logs(self): if file_path in self.log_data: continue try: - channels, counts = self.load_csv_counts(file_path) + spectrum_data = self.load_spectrum_data(file_path) except Exception as exc: - QMessageBox.warning(self, "CSV load error", f"Failed to load {file_path}: {exc}") + QMessageBox.warning(self, "Spectrum load error", f"Failed to load {file_path}: {exc}") continue - self.log_data[file_path] = {"channels": channels, "counts": counts} + self.log_data[file_path] = spectrum_data self.add_log_row(file_path) self.update_plot() @@ -285,12 +368,21 @@ def on_log_item_changed(self, item): self._suppress_log_item_changed = False self.update_plot() - def load_csv_counts(self, file_path): + def load_csv_spectrum_data(self, file_path): channels = [] counts = [] + csv_metadata = {} with open(file_path, "r", newline="") as handle: reader = csv.reader(handle) for row in reader: + if row and row[0].strip() == CALIBRATION_CSV_METADATA_KEY and len(row) >= 2: + try: + loaded = json.loads(row[1]) + except json.JSONDecodeError: + loaded = {} + if isinstance(loaded, dict): + csv_metadata = loaded + continue if len(row) < 2: continue try: @@ -302,7 +394,92 @@ def load_csv_counts(self, file_path): counts.append(count) if not channels: raise ValueError("No numeric channel/count data found") - return np.array(channels), np.array(counts) + return { + "channels": np.array(channels), + "counts": np.array(counts), + "metadata": csv_metadata.get("source_metadata", {}), + "telemetry": {}, + "source_format": "csv", + "environment": csv_metadata.get("environment"), + "device": csv_metadata.get("device"), + "csv_metadata": csv_metadata, + } + + def load_csv_counts(self, file_path): + data = self.load_csv_spectrum_data(file_path) + return data["channels"], data["counts"] + + def load_spectrum_data(self, file_path): + try: + parsed = parse_file(file_path) + except Exception: + return self.load_csv_spectrum_data(file_path) + + hist = np.asarray(parsed[2], dtype=float) + metadata = parsed[3] if len(parsed) > 3 and isinstance(parsed[3], dict) else {} + telemetry = parsed[4] if len(parsed) > 4 and isinstance(parsed[4], dict) else {} + return { + "channels": np.arange(hist.shape[0], dtype=float), + "counts": hist, + "metadata": metadata, + "telemetry": telemetry, + "source_format": "parsed_log", + "environment": summarize_environment(telemetry), + "device": summarize_device(metadata), + } + + def summarize_environment(self, telemetry): + return summarize_environment(telemetry) + + def summarize_device(self, metadata): + return summarize_device(metadata) + + def collect_log_metadata(self): + logs = [] + env_values = [] + devices = [] + for row in range(self.log_table.rowCount()): + log_item = self.log_table.item(row, 0) + label_item = self.log_table.item(row, 1) + if log_item is None: + continue + file_path = log_item.data(Qt.UserRole) + data = self.log_data.get(file_path, {}) + telemetry = data.get("telemetry", {}) + metadata = data.get("metadata", {}) + environment = data.get("environment") or self.summarize_environment(telemetry) + device = data.get("device") or self.summarize_device(metadata) + checked = log_item.checkState() == Qt.Checked + if checked: + env_values.append(environment) + devices.append(device) + logs.append({ + "path": file_path, + "label": label_item.text().strip() if label_item else "", + "checked": checked, + "source_format": data.get("source_format", "unknown"), + "environment": environment, + "device": device, + "source_metadata": metadata, + }) + return logs, self.merge_environment(env_values), self.merge_devices(devices) + + def merge_environment(self, environments): + def _mean_key(key): + values = [env.get(key) for env in environments] + values = [float(v) for v in values if v is not None and math.isfinite(float(v))] + return float(np.mean(values)) if values else math.nan + + return { + "temperature_celsius": _mean_key("temperature_celsius"), + "relative_humidity_percent": _mean_key("relative_humidity_percent"), + "pressure_hpa": _mean_key("pressure_hpa"), + } + + def merge_devices(self, devices): + if not devices: + return self.summarize_device({}) + return devices[0] def add_empty_calibration_point(self): row = self.channel_energy_table.rowCount() @@ -592,17 +769,7 @@ def import_selected_energies(self): self.update_line_label_positions() def collect_project_data(self): - logs = [] - for row in range(self.log_table.rowCount()): - log_item = self.log_table.item(row, 0) - label_item = self.log_table.item(row, 1) - if log_item is None: - continue - logs.append({ - "path": log_item.data(Qt.UserRole), - "label": label_item.text().strip() if label_item else "", - "checked": log_item.checkState() == Qt.Checked, - }) + logs, environment, device = self.collect_log_metadata() channel_energy = [] for row in range(self.channel_energy_table.rowCount()): @@ -626,8 +793,10 @@ def collect_project_data(self): }) return { - "version": 1, + "version": 2, "logs": logs, + "calibration_environment": environment, + "calibration_device": device, "channel_energy": channel_energy, "selected_energies": selected_energies, "constants": { @@ -662,11 +831,11 @@ def apply_project_data(self, data): if not file_path: continue try: - channels, counts = self.load_csv_counts(file_path) + spectrum_data = self.load_spectrum_data(file_path) except Exception: missing_logs.append(file_path) continue - self.log_data[file_path] = {"channels": channels, "counts": counts} + self.log_data[file_path] = spectrum_data self.add_log_row(file_path) row = self.log_table.rowCount() - 1 log_item = self.log_table.item(row, 0) diff --git a/dosview/parsers.py b/dosview/parsers.py index 7033f96..308bdb3 100644 --- a/dosview/parsers.py +++ b/dosview/parsers.py @@ -79,6 +79,20 @@ def parse(self): "hw-sn": parts[6].strip(), } metadata["log_runs_count"] += 1 + case "$DIG": + metadata["log_device_info"]["DIG"] = { + "type": parts[0], + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } + case "$ADC": + metadata["log_device_info"]["ADC"] = { + "type": parts[0], + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } case "$START": inside_run = True current_hist = np.zeros_like(hist) @@ -209,6 +223,20 @@ def parse(self): "hw-sn": parts[6].strip(), } metadata["log_runs_count"] += 1 + case "$DIG": + metadata["log_device_info"]["DIG"] = { + "type": parts[0], + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } + case "$ADC": + metadata["log_device_info"]["ADC"] = { + "type": parts[0], + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } case "$AIRDOS": metadata["log_device_info"]["AIRDOS"] = { "type": parts[0], diff --git a/tests/test_calibration_csv_metadata.py b/tests/test_calibration_csv_metadata.py new file mode 100644 index 0000000..0c72872 --- /dev/null +++ b/tests/test_calibration_csv_metadata.py @@ -0,0 +1,38 @@ +import csv +import json + +from dosview.calibration_widget import CALIBRATION_CSV_METADATA_KEY, CalibrationTab + + +def test_calibration_csv_metadata_is_loaded(tmp_path): + payload = { + "version": 1, + "environment": { + "temperature_celsius": 20.5, + "relative_humidity_percent": 45.0, + "pressure_hpa": 971.2, + }, + "device": { + "analog_module": { + "type": "USTSIPIN03C", + "serial_number": "09104108741008504c3ba080a0800056", + "configuration": "ffff", + } + }, + } + path = tmp_path / "spectrum.csv" + with path.open("w", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([CALIBRATION_CSV_METADATA_KEY, json.dumps(payload)]) + writer.writerow(["channel", "counts"]) + writer.writerow([0, 10]) + writer.writerow([1, 20]) + + tab = CalibrationTab.__new__(CalibrationTab) + data = tab.load_spectrum_data(str(path)) + + assert data["source_format"] == "csv" + assert data["environment"]["pressure_hpa"] == 971.2 + assert data["device"]["analog_module"]["serial_number"] == "09104108741008504c3ba080a0800056" + assert data["channels"].tolist() == [0.0, 1.0] + assert data["counts"].tolist() == [10.0, 20.0] diff --git a/tests/test_parser.py b/tests/test_parser.py index aa87fb1..c61cad7 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -71,3 +71,15 @@ def test_parse_fixture_returns_consistent_shapes(log_path): if metadata["log_info"].get("detector_type") == "AIRDOS04C": assert metadata["log_info"].get("histogram_channels") == hist.shape[0] + + +def test_airdos_v2_parser_keeps_module_identification(): + log_path = DATA_DIR / "DATALOG_AIRDOS04_2.0.TXT" + parsed = parse_file(log_path) + metadata = parsed[3] + device_info = metadata["log_device_info"] + + assert device_info["ADC"]["module-type"] == "USTSIPIN03A" + assert device_info["ADC"]["serial"] == "0950710874100851f80aa0c0a08000b6" + assert device_info["DIG"]["module-type"] == "BATDATUNIT01B" + assert device_info["DIG"]["serial"] == "1470c00806c200949c49a000a00d009c" diff --git a/xDOS-versions b/xDOS-versions index 7e29e51..2f34cf8 160000 --- a/xDOS-versions +++ b/xDOS-versions @@ -1 +1 @@ -Subproject commit 7e29e51496a3af85adac7643e56a7da8d7f28d7b +Subproject commit 2f34cf8cec9ba3870016c174cf4c056221dbf86f From 3b7f290be646dea85251f9b8a80452916784eda0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20K=C3=A1kona?= Date: Tue, 5 May 2026 20:06:56 +0200 Subject: [PATCH 2/6] Change spectrum CSV source path to relative path. --- dosview/calibration_widget.py | 39 ++++++++++++++++++-------- tests/test_calibration_csv_metadata.py | 20 ++++++++++++- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/dosview/calibration_widget.py b/dosview/calibration_widget.py index 73195c5..82c5b1e 100644 --- a/dosview/calibration_widget.py +++ b/dosview/calibration_widget.py @@ -109,6 +109,21 @@ def summarize_device(metadata): } +def make_project_relative_path(file_path, project_path): + if not file_path or not project_path: + return file_path + project_dir = os.path.dirname(os.path.abspath(project_path)) + abs_file_path = os.path.abspath(file_path) + return os.path.relpath(abs_file_path, project_dir) + + +def resolve_project_path(file_path, project_path): + if not file_path or not project_path or os.path.isabs(file_path): + return file_path + project_dir = os.path.dirname(os.path.abspath(project_path)) + return os.path.abspath(os.path.join(project_dir, file_path)) + + class EnergyAxisItem(pg.AxisItem): """Custom axis item for displaying energy values computed from channels.""" @@ -434,7 +449,7 @@ def summarize_environment(self, telemetry): def summarize_device(self, metadata): return summarize_device(metadata) - def collect_log_metadata(self): + def collect_log_metadata(self, project_path=None): logs = [] env_values = [] devices = [] @@ -454,7 +469,7 @@ def collect_log_metadata(self): env_values.append(environment) devices.append(device) logs.append({ - "path": file_path, + "path": make_project_relative_path(file_path, project_path), "label": label_item.text().strip() if label_item else "", "checked": checked, "source_format": data.get("source_format", "unknown"), @@ -768,8 +783,8 @@ def import_selected_energies(self): self.sync_channel_energy_lines() self.update_line_label_positions() - def collect_project_data(self): - logs, environment, device = self.collect_log_metadata() + def collect_project_data(self, project_path=None): + logs, environment, device = self.collect_log_metadata(project_path=project_path) channel_energy = [] for row in range(self.channel_energy_table.rowCount()): @@ -793,7 +808,8 @@ def collect_project_data(self): }) return { - "version": 2, + "version": 3, + "path_base": "project_directory", "logs": logs, "calibration_environment": environment, "calibration_device": device, @@ -806,7 +822,7 @@ def collect_project_data(self): "log_scale": self.log_scale_checkbox.isChecked(), } - def apply_project_data(self, data): + def apply_project_data(self, data, project_path=None): if not isinstance(data, dict): QMessageBox.warning(self, "Project load error", "Project file has invalid format.") return @@ -827,13 +843,14 @@ def apply_project_data(self, data): missing_logs = [] for entry in data.get("logs", []): - file_path = entry.get("path") - if not file_path: + stored_path = entry.get("path") + if not stored_path: continue + file_path = resolve_project_path(stored_path, project_path) try: spectrum_data = self.load_spectrum_data(file_path) except Exception: - missing_logs.append(file_path) + missing_logs.append(stored_path) continue self.log_data[file_path] = spectrum_data self.add_log_row(file_path) @@ -896,7 +913,7 @@ def save_project(self): return if not file_path.endswith(".dosview_calib"): file_path = f"{file_path}.dosview_calib" - payload = self.collect_project_data() + payload = self.collect_project_data(project_path=file_path) try: with open(file_path, "w", encoding="utf-8") as handle: json.dump(payload, handle, indent=2, ensure_ascii=True) @@ -919,7 +936,7 @@ def load_project(self, path=None): except Exception as exc: QMessageBox.warning(self, "Load error", f"Failed to load project: {exc}") return - self.apply_project_data(payload) + self.apply_project_data(payload, project_path=path) def update_plot(self): plot_item = self.plot_widget.plotItem diff --git a/tests/test_calibration_csv_metadata.py b/tests/test_calibration_csv_metadata.py index 0c72872..0be80f5 100644 --- a/tests/test_calibration_csv_metadata.py +++ b/tests/test_calibration_csv_metadata.py @@ -1,7 +1,12 @@ import csv import json -from dosview.calibration_widget import CALIBRATION_CSV_METADATA_KEY, CalibrationTab +from dosview.calibration_widget import ( + CALIBRATION_CSV_METADATA_KEY, + CalibrationTab, + make_project_relative_path, + resolve_project_path, +) def test_calibration_csv_metadata_is_loaded(tmp_path): @@ -36,3 +41,16 @@ def test_calibration_csv_metadata_is_loaded(tmp_path): assert data["device"]["analog_module"]["serial_number"] == "09104108741008504c3ba080a0800056" assert data["channels"].tolist() == [0.0, 1.0] assert data["counts"].tolist() == [10.0, 20.0] + + +def test_project_paths_are_relative_to_calib_file(tmp_path): + project_path = tmp_path / "calibration" / "project.dosview_calib" + csv_path = tmp_path / "calibration" / "spectra" / "source.csv" + project_path.parent.mkdir() + csv_path.parent.mkdir() + csv_path.write_text("channel,counts\n0,1\n", encoding="utf-8") + + stored_path = make_project_relative_path(str(csv_path), str(project_path)) + + assert stored_path == "spectra/source.csv" + assert resolve_project_path(stored_path, str(project_path)) == str(csv_path) From 1c0741dedf1d087a76daf509765aeb5183792f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20K=C3=A1kona?= Date: Tue, 5 May 2026 20:21:37 +0200 Subject: [PATCH 3/6] Add saving of the plot range into calib file. --- dosview/calibration_widget.py | 50 ++++++++++++++++++++++++-- tests/test_calibration_csv_metadata.py | 10 ++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/dosview/calibration_widget.py b/dosview/calibration_widget.py index 82c5b1e..be2c20b 100644 --- a/dosview/calibration_widget.py +++ b/dosview/calibration_widget.py @@ -124,6 +124,27 @@ def resolve_project_path(file_path, project_path): return os.path.abspath(os.path.join(project_dir, file_path)) +def sanitize_plot_range(plot_range): + if not isinstance(plot_range, dict): + return None + try: + x_min = float(plot_range["x"][0]) + x_max = float(plot_range["x"][1]) + y_min = float(plot_range["y"][0]) + y_max = float(plot_range["y"][1]) + except (KeyError, TypeError, ValueError, IndexError): + return None + values = (x_min, x_max, y_min, y_max) + if not all(math.isfinite(value) for value in values): + return None + if x_min == x_max or y_min == y_max: + return None + return { + "x": [min(x_min, x_max), max(x_min, x_max)], + "y": [min(y_min, y_max), max(y_min, y_max)], + } + + class EnergyAxisItem(pg.AxisItem): """Custom axis item for displaying energy values computed from channels.""" @@ -155,6 +176,8 @@ def __init__(self): self.energy_config_key = "calibration/selected_energies" self.plot_legend = None self.legend_formula_item = None + self._pending_plot_range = None + self._plot_range_initialized = False self._suppress_log_item_changed = False self._suppress_energy_item_changed = False self._suppress_channel_energy_item_changed = False @@ -320,9 +343,9 @@ def initUI(self): def add_csv_logs(self): file_paths, _ = QFileDialog.getOpenFileNames( self, - "Select calibration spectrum/log files", + "Select spectrum CSV files", "", - "Spectrum/log files (*.csv *.TXT *.txt *.npz);;CSV files (*.csv);;Log files (*.TXT *.txt);;Saved data (*.npz);;All files (*)" + "Spectrum CSV files (*.csv);;All files (*)" ) if not file_paths: return @@ -785,6 +808,11 @@ def import_selected_energies(self): def collect_project_data(self, project_path=None): logs, environment, device = self.collect_log_metadata(project_path=project_path) + view_range = self.plot_widget.plotItem.vb.viewRange() + plot_range = sanitize_plot_range({ + "x": view_range[0], + "y": view_range[1], + }) channel_energy = [] for row in range(self.channel_energy_table.rowCount()): @@ -819,6 +847,7 @@ def collect_project_data(self, project_path=None): "slope": self.slope_spin.value(), "offset": self.offset_spin.value(), }, + "plot_range": plot_range, "log_scale": self.log_scale_checkbox.isChecked(), } @@ -840,6 +869,7 @@ def apply_project_data(self, data, project_path=None): self.log_data = {} self.channel_energy_lines = [] self.energy_lines = [] + self._plot_range_initialized = False missing_logs = [] for entry in data.get("logs", []): @@ -890,6 +920,7 @@ def apply_project_data(self, data, project_path=None): self.slope_spin.setValue(float(constants.get("slope", self.slope_spin.value()))) self.offset_spin.setValue(float(constants.get("offset", self.offset_spin.value()))) self.log_scale_checkbox.setChecked(bool(data.get("log_scale", False))) + self._pending_plot_range = sanitize_plot_range(data.get("plot_range")) self.save_energy_config() self.sync_channel_energy_lines() @@ -939,6 +970,16 @@ def load_project(self, path=None): self.apply_project_data(payload, project_path=path) def update_plot(self): + current_view_range = self.plot_widget.plotItem.vb.viewRange() + if self._pending_plot_range is not None: + restore_range = self._pending_plot_range + elif self._plot_range_initialized: + restore_range = sanitize_plot_range({ + "x": current_view_range[0], + "y": current_view_range[1], + }) + else: + restore_range = None plot_item = self.plot_widget.plotItem plot_item.clear() self.plot_widget.showGrid(x=True, y=True, alpha=0.25) @@ -993,6 +1034,11 @@ def update_plot(self): self.sync_channel_energy_lines() self.update_energy_lines() self.update_line_label_positions() + if restore_range is not None: + self.plot_widget.setXRange(restore_range["x"][0], restore_range["x"][1], padding=0) + self.plot_widget.setYRange(restore_range["y"][0], restore_range["y"][1], padding=0) + self._plot_range_initialized = bool(active_rows) + self._pending_plot_range = None def update_energy_lines(self): for line in self.energy_lines: diff --git a/tests/test_calibration_csv_metadata.py b/tests/test_calibration_csv_metadata.py index 0be80f5..03db182 100644 --- a/tests/test_calibration_csv_metadata.py +++ b/tests/test_calibration_csv_metadata.py @@ -6,6 +6,7 @@ CalibrationTab, make_project_relative_path, resolve_project_path, + sanitize_plot_range, ) @@ -54,3 +55,12 @@ def test_project_paths_are_relative_to_calib_file(tmp_path): assert stored_path == "spectra/source.csv" assert resolve_project_path(stored_path, str(project_path)) == str(csv_path) + + +def test_plot_range_is_sanitized_for_project_file(): + assert sanitize_plot_range({"x": [200, 100], "y": [0.8, 0.1]}) == { + "x": [100.0, 200.0], + "y": [0.1, 0.8], + } + assert sanitize_plot_range({"x": [1, 1], "y": [0, 1]}) is None + assert sanitize_plot_range({"x": [1, "bad"], "y": [0, 1]}) is None From d24158be6ce5778f7b74a647d945c2d043ded656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20K=C3=A1kona?= Date: Tue, 5 May 2026 20:28:17 +0200 Subject: [PATCH 4/6] Fix failing pytest. --- tests/test_calibration_csv_metadata.py | 29 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/test_calibration_csv_metadata.py b/tests/test_calibration_csv_metadata.py index 03db182..176a8ca 100644 --- a/tests/test_calibration_csv_metadata.py +++ b/tests/test_calibration_csv_metadata.py @@ -1,13 +1,30 @@ import csv +import importlib.util import json +import sys +import types +from pathlib import Path -from dosview.calibration_widget import ( - CALIBRATION_CSV_METADATA_KEY, - CalibrationTab, - make_project_relative_path, - resolve_project_path, - sanitize_plot_range, +ROOT = Path(__file__).resolve().parent.parent +PACKAGE = types.ModuleType("dosview") +PACKAGE.__path__ = [str(ROOT / "dosview")] +sys.modules.setdefault("dosview", PACKAGE) + +CALIBRATION_WIDGET_PATH = ROOT / "dosview" / "calibration_widget.py" +spec = importlib.util.spec_from_file_location( + "dosview.calibration_widget", + CALIBRATION_WIDGET_PATH, ) +calibration_widget = importlib.util.module_from_spec(spec) +assert spec.loader is not None +sys.modules["dosview.calibration_widget"] = calibration_widget +spec.loader.exec_module(calibration_widget) + +CALIBRATION_CSV_METADATA_KEY = calibration_widget.CALIBRATION_CSV_METADATA_KEY +CalibrationTab = calibration_widget.CalibrationTab +make_project_relative_path = calibration_widget.make_project_relative_path +resolve_project_path = calibration_widget.resolve_project_path +sanitize_plot_range = calibration_widget.sanitize_plot_range def test_calibration_csv_metadata_is_loaded(tmp_path): From ecc8f793499a909337e18c147687b0f2b21cb6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20K=C3=A1kona?= Date: Tue, 5 May 2026 20:35:08 +0200 Subject: [PATCH 5/6] Make submodule available for test without key. --- .github/workflows/PyInstaller.yml | 2 ++ .github/workflows/python-publish.yml | 2 ++ .github/workflows/tests.yml | 2 ++ .gitmodules | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/PyInstaller.yml b/.github/workflows/PyInstaller.yml index f866032..c3faf2f 100644 --- a/.github/workflows/PyInstaller.yml +++ b/.github/workflows/PyInstaller.yml @@ -16,6 +16,8 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 + with: + submodules: recursive - uses: actions/setup-python@v4 with: python-version: "3.12" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 2bdc22d..3ce215b 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -16,6 +16,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + submodules: recursive - name: Set up Python uses: actions/setup-python@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f84785..2931188 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + submodules: recursive - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.gitmodules b/.gitmodules index e4c29f5..663c70e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "xDOS-versions"] path = xDOS-versions - url = git@github.com:UniversalScientificTechnologies/xDOS-versions.git + url = https://github.com/UniversalScientificTechnologies/xDOS-versions.git From 9d49422c194c79e4b3cfda9f7cecf1fb1e7bbfd6 Mon Sep 17 00:00:00 2001 From: Jakub Kakona Date: Tue, 5 May 2026 22:38:56 +0200 Subject: [PATCH 6/6] Fix parsing of metadata during live recording. --- dosview/__init__.py | 82 +++++++++++++++++++++++++++++++++++++++------ dosview/parsers.py | 23 ++++++++----- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/dosview/__init__.py b/dosview/__init__.py index deed07e..086112b 100644 --- a/dosview/__init__.py +++ b/dosview/__init__.py @@ -465,11 +465,30 @@ def run(self): spectral_records = [] metadata = {"log_runs_count": 0, "log_device_info": {}} fmt = None # 'old' or 'v2' + env_records = [] # v2-specific inter-record state current_hist = None current_counts = 0 + def _build_telemetry(env_recs): + if not env_recs: + return {} + ea = np.array(env_recs) + tel = { + "temperature_0": (ea[:, 0], ea[:, 1]), + "humidity_0": (ea[:, 0], ea[:, 2]), + } + if ea.shape[1] > 3: + tel["temperature_1"] = (ea[:, 0], ea[:, 3]) + if ea.shape[1] > 4: + tel["humidity_1"] = (ea[:, 0], ea[:, 4]) + if ea.shape[1] > 5: + tel["temperature_2"] = (ea[:, 0], ea[:, 5]) + if ea.shape[1] > 6: + tel["pressure_3"] = (ea[:, 0], ea[:, 6]) + return tel + try: self._ser = serial.Serial(self._port, self._baud, timeout=1) self.connected.emit(True) @@ -484,6 +503,30 @@ def run(self): parts = line.split(",") msg = parts[0] + # --- Device header records (format-independent) --- + if msg == "$DOS" and len(parts) > 6: + metadata["log_device_info"]["DOS"] = { + "hw-model": parts[1], + "fw-version": parts[2], + "eeprom": parts[3] if len(parts) > 3 else "", + "fw-commit": parts[4] if len(parts) > 4 else "", + "fw-build_info": parts[5] if len(parts) > 5 else "", + "hw-sn": parts[6].strip() if len(parts) > 6 else "", + } + metadata["log_runs_count"] += 1 + elif msg == "$ADC" and len(parts) >= 2: + metadata["log_device_info"]["ADC"] = { + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } + elif msg == "$DIG" and len(parts) >= 2: + metadata["log_device_info"]["DIG"] = { + "module-type": parts[1] if len(parts) > 1 else "", + "serial": parts[2].strip() if len(parts) > 2 else "", + "configuration": parts[3].strip() if len(parts) > 3 else "", + } + # --- Format detection --- if fmt is None: if msg in ("$HIST", "$AIRDOS"): @@ -501,6 +544,19 @@ def run(self): "detector": parts[2] if len(parts) > 2 else "", "hw-sn": parts[3].strip() if len(parts) > 3 else "", } + elif msg == "$ENV" and len(parts) >= 5: + try: + env_records.append(( + float(parts[2]), + float(parts[3]), + float(parts[4]), + float(parts[5]) if len(parts) > 5 else float("nan"), + float(parts[6]) if len(parts) > 6 else float("nan"), + float(parts[7]) if len(parts) > 7 else float("nan"), + float(parts[8]) if len(parts) > 8 else float("nan"), + )) + except ValueError: + pass elif msg == "$HIST": try: t = float(parts[2]) @@ -514,22 +570,26 @@ def run(self): metadata["log_runs_count"] = len(time_axis) sm = np.array(spectral_records) self.dataUpdated.emit( - [np.array(time_axis), np.array(sums), hist.copy(), metadata, {}, sm] + [np.array(time_axis), np.array(sums), hist.copy(), metadata, _build_telemetry(env_records), sm] ) except (ValueError, IndexError): pass # --- V2 format --- elif fmt == "v2": - if msg == "$DOS" and len(parts) > 6: - metadata["log_device_info"]["DOS"] = { - "hw-model": parts[1], - "fw-version": parts[2], - "eeprom": parts[3] if len(parts) > 3 else "", - "fw-commit": parts[4] if len(parts) > 4 else "", - "hw-sn": parts[6].strip() if len(parts) > 6 else "", - } - metadata["log_runs_count"] += 1 + if msg == "$ENV" and len(parts) >= 5: + try: + env_records.append(( + float(parts[2]), + float(parts[3]), + float(parts[4]), + float(parts[5]) if len(parts) > 5 else float("nan"), + float(parts[6]) if len(parts) > 6 else float("nan"), + float(parts[7]) if len(parts) > 7 else float("nan"), + float(parts[8]) if len(parts) > 8 else float("nan"), + )) + except ValueError: + pass elif msg == "$START": current_hist = np.zeros_like(hist) current_counts = 0 @@ -560,7 +620,7 @@ def run(self): sums.append(int(current_hist.sum())) sm = np.array(spectral_records) self.dataUpdated.emit( - [np.array(time_axis), np.array(sums), hist.copy(), metadata, {}, sm] + [np.array(time_axis), np.array(sums), hist.copy(), metadata, _build_telemetry(env_records), sm] ) except (ValueError, IndexError): pass diff --git a/dosview/parsers.py b/dosview/parsers.py index 308bdb3..6b1883d 100644 --- a/dosview/parsers.py +++ b/dosview/parsers.py @@ -34,13 +34,18 @@ class AirdosV2LogParser(BaseLogParser): @staticmethod def detect(file_path: str | Path) -> bool: + has_dos = False with open(file_path, "r") as f: for line in f: if line.startswith("$DOS"): + has_dos = True parts = line.strip().split(",") # parts[2] = fw-version; MAJOR.MINOR encodes the data format version if len(parts) > 2 and parts[2].startswith("2."): return True + if line.startswith("$START") and has_dos: + # $START/$STOP record format used from fw 1.3 onward + return True return False def parse(self): @@ -119,17 +124,17 @@ def parse(self): inside_run = False current_hist = None case "$ENV": - # $ENV,count,tm.tm_s100,T1,H1,T2,H2,T_MS5611,P_MS5611 - if len(parts) >= 9: + # $ENV,count,tm.tm_s100,T1,H1[,T2,H2,T_MS5611,P_MS5611] + if len(parts) >= 5: try: env_records.append(( - float(parts[2]), # time - float(parts[3]), # T1 (SHT31 primary) - float(parts[4]), # H1 - float(parts[5]), # T2 (SHT31 secondary) - float(parts[6]), # H2 - float(parts[7]), # T_MS5611 - float(parts[8]), # P_MS5611 + float(parts[2]), # time + float(parts[3]), # T1 (SHT31 primary) + float(parts[4]), # H1 + float(parts[5]) if len(parts) > 5 else float("nan"), # T2 + float(parts[6]) if len(parts) > 6 else float("nan"), # H2 + float(parts[7]) if len(parts) > 7 else float("nan"), # T_MS5611 + float(parts[8]) if len(parts) > 8 else float("nan"), # P_MS5611 )) except ValueError: pass