Skip to content

Calibration workflow improvement #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kaklik merged 6 commits into from
May 20, 2026
Merged

Calibration workflow improvement #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e2c48c7
Add enviromental metadata to calibration file.
kaklik May 5, 2026
3b7f290
Change spectrum CSV source path to relative path.
kaklik May 5, 2026
1c0741d
Add saving of the plot range into calib file.
kaklik May 5, 2026
d24158b
Fix failing pytest.
kaklik May 5, 2026
ecc8f79
Make submodule available for test without key.
kaklik May 5, 2026
9d49422
Fix parsing of metadata during live recording.
kaklik May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/PyInstaller.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ jobs:
os: [ubuntu-latest, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/setup-python@v4
with:
python-version: "3.12"
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/python-publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ jobs:

steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v3
with:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[submodule "xDOS-versions"]
path = xDOS-versions
url = git@github.com:UniversalScientificTechnologies/xDOS-versions.git
url = https://github.com/UniversalScientificTechnologies/xDOS-versions.git
102 changes: 90 additions & 12 deletions dosview/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import sys
import argparse
import json

from PyQt5 import QtNetwork
from PyQt5.QtNetwork import QLocalSocket, QLocalServer
Expand Down Expand Up @@ -28,7 +29,12 @@
from .version import __version__
from pyqtgraph import ImageView

from .calibration_widget import CalibrationTab
from .calibration_widget import (
CALIBRATION_CSV_METADATA_KEY,
CalibrationTab,
summarize_device,
summarize_environment,
)
from .parsers import (
BaseLogParser,
Airdos04CLogParser,
Expand Down Expand Up @@ -459,11 +465,30 @@ def run(self):
spectral_records = []
metadata = {"log_runs_count": 0, "log_device_info": {}}
fmt = None # 'old' or 'v2'
env_records = []

# v2-specific inter-record state
current_hist = None
current_counts = 0

def _build_telemetry(env_recs):
if not env_recs:
return {}
ea = np.array(env_recs)
tel = {
"temperature_0": (ea[:, 0], ea[:, 1]),
"humidity_0": (ea[:, 0], ea[:, 2]),
}
if ea.shape[1] > 3:
tel["temperature_1"] = (ea[:, 0], ea[:, 3])
if ea.shape[1] > 4:
tel["humidity_1"] = (ea[:, 0], ea[:, 4])
if ea.shape[1] > 5:
tel["temperature_2"] = (ea[:, 0], ea[:, 5])
if ea.shape[1] > 6:
tel["pressure_3"] = (ea[:, 0], ea[:, 6])
return tel

try:
self._ser = serial.Serial(self._port, self._baud, timeout=1)
self.connected.emit(True)
Expand All @@ -478,6 +503,30 @@ def run(self):
parts = line.split(",")
msg = parts[0]

# --- Device header records (format-independent) ---
if msg == "$DOS" and len(parts) > 6:
metadata["log_device_info"]["DOS"] = {
"hw-model": parts[1],
"fw-version": parts[2],
"eeprom": parts[3] if len(parts) > 3 else "",
"fw-commit": parts[4] if len(parts) > 4 else "",
"fw-build_info": parts[5] if len(parts) > 5 else "",
"hw-sn": parts[6].strip() if len(parts) > 6 else "",
}
metadata["log_runs_count"] += 1
elif msg == "$ADC" and len(parts) >= 2:
metadata["log_device_info"]["ADC"] = {
"module-type": parts[1] if len(parts) > 1 else "",
"serial": parts[2].strip() if len(parts) > 2 else "",
"configuration": parts[3].strip() if len(parts) > 3 else "",
}
elif msg == "$DIG" and len(parts) >= 2:
metadata["log_device_info"]["DIG"] = {
"module-type": parts[1] if len(parts) > 1 else "",
"serial": parts[2].strip() if len(parts) > 2 else "",
"configuration": parts[3].strip() if len(parts) > 3 else "",
}

# --- Format detection ---
if fmt is None:
if msg in ("$HIST", "$AIRDOS"):
Expand All @@ -495,6 +544,19 @@ def run(self):
"detector": parts[2] if len(parts) > 2 else "",
"hw-sn": parts[3].strip() if len(parts) > 3 else "",
}
elif msg == "$ENV" and len(parts) >= 5:
try:
env_records.append((
float(parts[2]),
float(parts[3]),
float(parts[4]),
float(parts[5]) if len(parts) > 5 else float("nan"),
float(parts[6]) if len(parts) > 6 else float("nan"),
float(parts[7]) if len(parts) > 7 else float("nan"),
float(parts[8]) if len(parts) > 8 else float("nan"),
))
except ValueError:
pass
elif msg == "$HIST":
try:
t = float(parts[2])
Expand All @@ -508,22 +570,26 @@ def run(self):
metadata["log_runs_count"] = len(time_axis)
sm = np.array(spectral_records)
self.dataUpdated.emit(
[np.array(time_axis), np.array(sums), hist.copy(), metadata, {}, sm]
[np.array(time_axis), np.array(sums), hist.copy(), metadata, _build_telemetry(env_records), sm]
)
except (ValueError, IndexError):
pass

# --- V2 format ---
elif fmt == "v2":
if msg == "$DOS" and len(parts) > 6:
metadata["log_device_info"]["DOS"] = {
"hw-model": parts[1],
"fw-version": parts[2],
"eeprom": parts[3] if len(parts) > 3 else "",
"fw-commit": parts[4] if len(parts) > 4 else "",
"hw-sn": parts[6].strip() if len(parts) > 6 else "",
}
metadata["log_runs_count"] += 1
if msg == "$ENV" and len(parts) >= 5:
try:
env_records.append((
float(parts[2]),
float(parts[3]),
float(parts[4]),
float(parts[5]) if len(parts) > 5 else float("nan"),
float(parts[6]) if len(parts) > 6 else float("nan"),
float(parts[7]) if len(parts) > 7 else float("nan"),
float(parts[8]) if len(parts) > 8 else float("nan"),
))
except ValueError:
pass
elif msg == "$START":
current_hist = np.zeros_like(hist)
current_counts = 0
Expand Down Expand Up @@ -554,7 +620,7 @@ def run(self):
sums.append(int(current_hist.sum()))
sm = np.array(spectral_records)
self.dataUpdated.emit(
[np.array(time_axis), np.array(sums), hist.copy(), metadata, {}, sm]
[np.array(time_axis), np.array(sums), hist.copy(), metadata, _build_telemetry(env_records), sm]
)
except (ValueError, IndexError):
pass
Expand Down Expand Up @@ -1224,8 +1290,20 @@ def export_spectrum_csv(self):
path += ".csv"
import csv
hist = self.data[2]
metadata = self.data[3] if len(self.data) > 3 and isinstance(self.data[3], dict) else {}
telemetry = self.data[4] if len(self.data) > 4 and isinstance(self.data[4], dict) else {}
calibration_metadata = {
"version": 1,
"environment": summarize_environment(telemetry),
"device": summarize_device(metadata),
"source_metadata": metadata,
}
with open(path, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow([
CALIBRATION_CSV_METADATA_KEY,
json.dumps(calibration_metadata, ensure_ascii=True, allow_nan=True),
])
writer.writerow(["channel", "counts"])
for ch, cnt in enumerate(hist):
writer.writerow([ch, int(cnt)])
Expand Down
Loading
Loading