From bd4e2f59a933b23007c47d986728a8b0e2cbe560 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Mon, 16 Feb 2026 11:36:26 -0500 Subject: [PATCH 01/23] First attempt at adding optical photon tracking to ddceler plugin --- src/ddceler/CelerOpticalOffload.cc | 129 +++++++++++++++++++++++++++++ src/ddceler/CelerOpticalOffload.hh | 42 ++++++++++ src/ddceler/CelerPhysics.cc | 20 +++++ src/ddceler/CelerPhysics.hh | 2 + 4 files changed, 193 insertions(+) create mode 100644 src/ddceler/CelerOpticalOffload.cc create mode 100644 src/ddceler/CelerOpticalOffload.hh diff --git a/src/ddceler/CelerOpticalOffload.cc b/src/ddceler/CelerOpticalOffload.cc new file mode 100644 index 0000000000..2db4f24aa1 --- /dev/null +++ b/src/ddceler/CelerOpticalOffload.cc @@ -0,0 +1,129 @@ +//------------------------------- -*- C++ -*- -------------------------------// +// Copyright Celeritas contributors: see top-level COPYRIGHT file for details +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +//---------------------------------------------------------------------------// +//! \file ddceler/CelerOpticalOffload.cc +//---------------------------------------------------------------------------// +#include "CelerOpticalOffload.hh" + +#include +#include +#include +#include +#include +#include + +#include "corecel/io/Logger.hh" +#include "corecel/math/UnitUtils.hh" +#include "geocel/g4/Convert.hh" +#include "accel/LocalOpticalGenOffload.hh" +#include "accel/detail/IntegrationSingleton.hh" + +namespace celeritas +{ +namespace dd +{ +//---------------------------------------------------------------------------// +/*! + * Standard constructor + */ +CelerOpticalOffload::CelerOpticalOffload(dd4hep::sim::Geant4Context* ctxt, + std::string const& name) + : dd4hep::sim::Geant4SteppingAction(ctxt, name) +{ + CELER_LOG(info) << "Registered CelerOpticalOffload stepping action"; +} + +//---------------------------------------------------------------------------// +/*! + * Stepping action to offload optical distributions to Celeritas. + */ +void CelerOpticalOffload::operator()(G4Step const* step, G4SteppingManager*) +{ + CELER_EXPECT(step); + + constexpr double clhep_time{1 / units::nanosecond}; + constexpr double clhep_length{1 / units::centimeter}; + + auto& local = detail::IntegrationSingleton::instance().local_offload(); + if (!local) + { + // Offloading is disabled + return; + } + + if (step->GetStepLength() == 0) + { + // Skip "no-process"-defined steps + return; + } + + auto* pm = step->GetTrack()->GetDefinition()->GetProcessManager(); + CELER_ASSERT(pm); + + // Determine how many Cherenkov and scintillation photons to generate + size_type num_cherenkov{0}; + size_type num_scintillation{0}; + + if (auto const* p = dynamic_cast(pm->GetProcess("Cerenk" + "ov"))) + { + num_cherenkov = p->GetNumPhotons(); + } + if (auto const* p = dynamic_cast( + pm->GetProcess("Scintillation"))) + { + num_scintillation = p->GetNumPhotons(); + } + + if (num_cherenkov == 0 && num_scintillation == 0) + { + return; + } + + auto* pre_step = step->GetPreStepPoint(); + auto* post_step = step->GetPostStepPoint(); + CELER_ASSERT(pre_step && post_step); + + // Create distribution and push to Celeritas + // TODO: Get optical material ID from geometry + optical::GeneratorDistributionData data; + data.time = convert_from_geant(post_step->GetGlobalTime(), clhep_time); + data.step_length = convert_from_geant(step->GetStepLength(), clhep_length); + data.charge = units::ElementaryCharge{ + static_cast(post_step->GetCharge())}; + data.material = OptMatId(0); // TODO: map from G4Material to OptMatId + data.points[StepPoint::pre] + = {units::LightSpeed(pre_step->GetBeta()), + convert_from_geant(pre_step->GetPosition(), clhep_length)}; + data.points[StepPoint::post] + = {units::LightSpeed(post_step->GetBeta()), + convert_from_geant(post_step->GetPosition(), clhep_length)}; + + auto& gen_offload = dynamic_cast(local); + + if (num_cherenkov > 0) + { + data.type = GeneratorType::cherenkov; + data.num_photons = num_cherenkov; + CELER_ASSERT(data); + gen_offload.Push(data); + } + if (num_scintillation > 0) + { + data.type = GeneratorType::scintillation; + data.num_photons = num_scintillation; + CELER_ASSERT(data); + gen_offload.Push(data); + } + + CELER_LOG_LOCAL(debug) << "Offloading " << num_cherenkov + << " Cherenkov and " << num_scintillation + << " scintillation photons"; +} + +//---------------------------------------------------------------------------// +} // namespace dd +} // namespace celeritas + +DECLARE_GEANT4ACTION_NS(celeritas::dd, CelerOpticalOffload) diff --git a/src/ddceler/CelerOpticalOffload.hh b/src/ddceler/CelerOpticalOffload.hh new file mode 100644 index 0000000000..41495c5171 --- /dev/null +++ b/src/ddceler/CelerOpticalOffload.hh @@ -0,0 +1,42 @@ +//------------------------------- -*- C++ -*- -------------------------------// +// Copyright Celeritas contributors: see top-level COPYRIGHT file for details +// SPDX-License-Identifier: (Apache-2.0 OR MIT) +//---------------------------------------------------------------------------// +//! \file ddceler/CelerOpticalOffload.hh +//---------------------------------------------------------------------------// +#pragma once + +#include + +namespace celeritas +{ +namespace dd +{ +//---------------------------------------------------------------------------// +/*! + * DDG4 stepping action for optical distribution offloading to Celeritas. + * + * This action intercepts Cherenkov and scintillation photon generation in + * Geant4 and offloads the distribution data to Celeritas for GPU tracking. + * + * The optical physics in Geant4 must be configured to *not* stack photons + * (stack_photons = false) so that photon counts are calculated but photons + * are not created in Geant4. + */ +class CelerOpticalOffload final : public dd4hep::sim::Geant4SteppingAction +{ + public: + // Standard constructor + CelerOpticalOffload(dd4hep::sim::Geant4Context* ctxt, + std::string const& name); + + // Delete copy/move + DDG4_DEFINE_ACTION_CONSTRUCTORS(CelerOpticalOffload); + + // Stepping action callback + virtual void operator()(G4Step const* step, G4SteppingManager* mgr) final; +}; + +//---------------------------------------------------------------------------// +} // namespace dd +} // namespace celeritas diff --git a/src/ddceler/CelerPhysics.cc b/src/ddceler/CelerPhysics.cc index 16536c18ff..653aa39f26 100644 --- a/src/ddceler/CelerPhysics.cc +++ b/src/ddceler/CelerPhysics.cc @@ -63,6 +63,8 @@ CelerPhysics::CelerPhysics(Geant4Context* ctxt, std::string const& name) declareProperty("MaxNumTracks", max_num_tracks_); declareProperty("InitCapacity", init_capacity_); declareProperty("IgnoreProcesses", ignore_processes_); + declareProperty("OpticalTracks", optical_tracks_); + declareProperty("OpticalGenerators", optical_generators_); } //---------------------------------------------------------------------------// @@ -181,6 +183,24 @@ SetupOptions CelerPhysics::make_options() opts.make_along_step = UniformAlongStepFactory(make_field_input); opts.sd.ignore_zero_deposition = false; + // Configure optical photon offloading if requested + if (optical_tracks_ > 0) + { + OpticalSetupOptions opt; + opt.capacity.tracks = optical_tracks_; + opt.capacity.generators = (optical_generators_ > 0) + ? optical_generators_ + : optical_tracks_ * 8; + opt.capacity.primaries = opt.capacity.generators; + opt.generator = inp::OpticalEmGenerator{}; + opt.limits = inp::OpticalTrackingLimits{}; + opts.optical = opt; + + CELER_LOG(info) << "Optical photon offloading enabled: tracks=" + << opt.capacity.tracks + << ", generators=" << opt.capacity.generators; + } + // Save diagnostic file to a unique name opts.output_file = "ddceler.out.json"; opts.geometry_output_file = "ddceler.out.gdml"; diff --git a/src/ddceler/CelerPhysics.hh b/src/ddceler/CelerPhysics.hh index f99ccc14d5..2935ca2359 100644 --- a/src/ddceler/CelerPhysics.hh +++ b/src/ddceler/CelerPhysics.hh @@ -37,6 +37,8 @@ class CelerPhysics final : public dd4hep::sim::Geant4PhysicsList int max_num_tracks_{0}; int init_capacity_{0}; std::vector ignore_processes_; + int optical_tracks_{0}; + int optical_generators_{0}; // Make options for Celeritas tracking manager SetupOptions make_options(); From cbcddb94626e4257f265a32ff2155157df990fa0 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Mon, 16 Feb 2026 13:39:29 -0500 Subject: [PATCH 02/23] Enable optical photons inDD4hep steering file --- example/ddceler/input/steeringFile.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/example/ddceler/input/steeringFile.py b/example/ddceler/input/steeringFile.py index 4b497c7cac..ea9349a89c 100644 --- a/example/ddceler/input/steeringFile.py +++ b/example/ddceler/input/steeringFile.py @@ -42,6 +42,32 @@ def setup_physics(kernel): from DDG4 import Geant4, PhysicsList phys = Geant4(kernel).setupPhysics("QGSP_BERT") + + # Optical photon physics + ph = PhysicsList(kernel, "Geant4OpticalPhotonPhysics/OpticalGammaPhys") + ph.VerboseLevel = 1 + ph.addParticleConstructor("G4OpticalPhoton") + ph.enableUI() + phys.adopt(ph) + + # Scintillation physics + ph = PhysicsList(kernel, "Geant4ScintillationPhysics/ScintillatorPhys") + ph.ScintillationYieldFactor = 1.0 + ph.ScintillationExcitationRatio = 1.0 + ph.TrackSecondariesFirst = False + ph.VerboseLevel = 1 + ph.enableUI() + phys.adopt(ph) + + # Cerenkov physics + ph = PhysicsList(kernel, "Geant4CerenkovPhysics/CerenkovPhys") + ph.MaxNumPhotonsPerStep = 10 + ph.MaxBetaChangePerStep = 10.0 + ph.TrackSecondariesFirst = True + ph.VerboseLevel = 1 + ph.enableUI() + phys.adopt(ph) + celer_phys = PhysicsList(kernel, str("CelerPhysics")) # MaxNumTracks: max number of tracks in flight # InitCapacity: initial capacity for state data allocation From 37e320e875cd947b94b11850a1d20ee0a73cc6d9 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Wed, 18 Feb 2026 22:14:50 -0500 Subject: [PATCH 03/23] Add modified definition of Polystyrene with placeholder optical properties in ddceler example xml --- example/ddceler/input/Preshower.xml | 48 ++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/example/ddceler/input/Preshower.xml b/example/ddceler/input/Preshower.xml index 3ce753067e..ac494b6a0b 100644 --- a/example/ddceler/input/Preshower.xml +++ b/example/ddceler/input/Preshower.xml @@ -18,11 +18,51 @@ + + + + + + + + + - + + - - + + + + + + + + + @@ -65,7 +105,7 @@ - + From 7bb0f45a256bcdb56dba69159871502e1659cdb0 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Wed, 18 Feb 2026 22:17:44 -0500 Subject: [PATCH 04/23] Add value for Optical Track parameters in steering file --- example/ddceler/input/steeringFile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/ddceler/input/steeringFile.py b/example/ddceler/input/steeringFile.py index ea9349a89c..f9108c0a3b 100644 --- a/example/ddceler/input/steeringFile.py +++ b/example/ddceler/input/steeringFile.py @@ -77,6 +77,9 @@ def setup_physics(kernel): celer_phys.InitCapacity = 245760 # Celeritas does not support single scattering celer_phys.IgnoreProcesses = ["CoulombScat"] + # Optical tracking: number of optical photon track slots + # OpticalGenerators defaults to OpticalTracks * 8 if unset + celer_phys.OpticalTracks = 2048 phys.adopt(celer_phys) phys.dump() return None From ff9927a5a1371d23a0ad10c63476ad1f0fec4199 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 19 Feb 2026 08:32:49 -0500 Subject: [PATCH 05/23] Revert optical property changes from Preshower example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move optical photon demonstration to a dedicated separate example instead. Restore Preshower.xml and steeringFile.py to their original state before optical physics was added. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- example/ddceler/input/Preshower.xml | 48 +++------------------------ example/ddceler/input/steeringFile.py | 29 ---------------- 2 files changed, 4 insertions(+), 73 deletions(-) diff --git a/example/ddceler/input/Preshower.xml b/example/ddceler/input/Preshower.xml index ac494b6a0b..3ce753067e 100644 --- a/example/ddceler/input/Preshower.xml +++ b/example/ddceler/input/Preshower.xml @@ -18,51 +18,11 @@ - - - - - - - - - - - + - - - - - - - - - + + @@ -105,7 +65,7 @@ - + diff --git a/example/ddceler/input/steeringFile.py b/example/ddceler/input/steeringFile.py index f9108c0a3b..4b497c7cac 100644 --- a/example/ddceler/input/steeringFile.py +++ b/example/ddceler/input/steeringFile.py @@ -42,32 +42,6 @@ def setup_physics(kernel): from DDG4 import Geant4, PhysicsList phys = Geant4(kernel).setupPhysics("QGSP_BERT") - - # Optical photon physics - ph = PhysicsList(kernel, "Geant4OpticalPhotonPhysics/OpticalGammaPhys") - ph.VerboseLevel = 1 - ph.addParticleConstructor("G4OpticalPhoton") - ph.enableUI() - phys.adopt(ph) - - # Scintillation physics - ph = PhysicsList(kernel, "Geant4ScintillationPhysics/ScintillatorPhys") - ph.ScintillationYieldFactor = 1.0 - ph.ScintillationExcitationRatio = 1.0 - ph.TrackSecondariesFirst = False - ph.VerboseLevel = 1 - ph.enableUI() - phys.adopt(ph) - - # Cerenkov physics - ph = PhysicsList(kernel, "Geant4CerenkovPhysics/CerenkovPhys") - ph.MaxNumPhotonsPerStep = 10 - ph.MaxBetaChangePerStep = 10.0 - ph.TrackSecondariesFirst = True - ph.VerboseLevel = 1 - ph.enableUI() - phys.adopt(ph) - celer_phys = PhysicsList(kernel, str("CelerPhysics")) # MaxNumTracks: max number of tracks in flight # InitCapacity: initial capacity for state data allocation @@ -77,9 +51,6 @@ def setup_physics(kernel): celer_phys.InitCapacity = 245760 # Celeritas does not support single scattering celer_phys.IgnoreProcesses = ["CoulombScat"] - # Optical tracking: number of optical photon track slots - # OpticalGenerators defaults to OpticalTracks * 8 if unset - celer_phys.OpticalTracks = 2048 phys.adopt(celer_phys) phys.dump() return None From 8bf1df7c32b1d8a247089d991dbfd1cf20ee8f8f Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 19 Feb 2026 09:41:53 -0500 Subject: [PATCH 06/23] Add simple water scintillator box example to test optical photon offload --- example/ddceler/optical/input/ScintWater.xml | 136 ++++++++++++++++++ example/ddceler/optical/input/steeringFile.py | 89 ++++++++++++ example/ddceler/optical/run-scintwater.sh | 71 +++++++++ 3 files changed, 296 insertions(+) create mode 100644 example/ddceler/optical/input/ScintWater.xml create mode 100644 example/ddceler/optical/input/steeringFile.py create mode 100755 example/ddceler/optical/run-scintwater.sh diff --git a/example/ddceler/optical/input/ScintWater.xml b/example/ddceler/optical/input/ScintWater.xml new file mode 100644 index 0000000000..826ddf1282 --- /dev/null +++ b/example/ddceler/optical/input/ScintWater.xml @@ -0,0 +1,136 @@ + + + + + + + + Minimal geometry for optical photon generation and propagation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + system:8,x:24:-12,y:-12 + + + + + + + + + + + + diff --git a/example/ddceler/optical/input/steeringFile.py b/example/ddceler/optical/input/steeringFile.py new file mode 100644 index 0000000000..2565eefb50 --- /dev/null +++ b/example/ddceler/optical/input/steeringFile.py @@ -0,0 +1,89 @@ +# Copyright Celeritas contributors: see top-level COPYRIGHT file for details +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""DDG4 steering file for the optical photon demonstration. + +A 1 GeV e- enters a 30 cm scintillating water cube, initiating an EM +shower (X0 ~ 36 cm in water). All shower secondaries are above the +Cherenkov threshold (~0.26 MeV kinetic energy in water), producing a +dense cone of optical photons. Scintillation photons are generated +along the shower axis proportional to the local energy deposit. +All optical photons are offloaded to Celeritas for GPU tracking. +""" + +from DDSim.DD4hepSimulation import DD4hepSimulation + +runner = DD4hepSimulation() + +runner.action.run = "CelerRun" +runner.action.tracker = "Geant4TrackerAction" +runner.action.trackerSDTypes = ["tracker"] +runner.action.calo = "Geant4CalorimeterAction" +runner.action.calorimeterSDTypes = ["calorimeter"] + +runner.outputConfig.forceDD4HEP = True +runner.numberOfEvents = 100 + +# Field tracking configuration +runner.field.delta_chord = 0.025 # mm +runner.field.delta_intersection = 1e-2 # mm +runner.field.delta_one_step = 0.001 # mm +runner.field.eps_min = 5e-5 # mm +runner.field.eps_max = 0.001 # mm +runner.field.min_chord_step = 1e-6 # mm + +# 1 GeV e-: EM shower mostly contained within the 30 cm water box +runner.enableGun = True +runner.gun.particle = "e-" +runner.gun.energy = "1*GeV" +runner.gun.distribution = "uniform" +runner.gun.etaMin = 5.0 +runner.gun.etaMax = 5.0 + + +def setup_physics(kernel): + """Configure optical physics list with Celeritas integration.""" + from DDG4 import Geant4, PhysicsList + + phys = Geant4(kernel).setupPhysics("QGSP_BERT") + + # Optical photon processes: boundary, absorption, Rayleigh + ph = PhysicsList(kernel, "Geant4OpticalPhotonPhysics/OpticalGammaPhys") + ph.VerboseLevel = 1 + ph.addParticleConstructor("G4OpticalPhoton") + ph.enableUI() + phys.adopt(ph) + + # Cherenkov radiation from shower tracks. + # MaxNumPhotonsPerStep is kept low to avoid exhausting the optical + # generator buffer given the high shower track multiplicity. + ph = PhysicsList(kernel, "Geant4CerenkovPhysics/CerenkovPhys") + ph.MaxNumPhotonsPerStep = 50 + ph.MaxBetaChangePerStep = 10.0 + ph.TrackSecondariesFirst = False + ph.VerboseLevel = 1 + ph.enableUI() + phys.adopt(ph) + + # Scintillation along the shower axis (yield = 100/MeV in geometry XML) + ph = PhysicsList(kernel, "Geant4ScintillationPhysics/ScintillatorPhys") + ph.ScintillationYieldFactor = 1.0 + ph.ScintillationExcitationRatio = 1.0 + ph.TrackSecondariesFirst = False + ph.VerboseLevel = 1 + ph.enableUI() + phys.adopt(ph) + + # Celeritas offload with optical tracking + celer_phys = PhysicsList(kernel, str("CelerPhysics")) + celer_phys.MaxNumTracks = 2048 + celer_phys.InitCapacity = 245760 + celer_phys.IgnoreProcesses = ["CoulombScat"] + # OpticalGenerators defaults to OpticalTracks * 8 = 16384 + celer_phys.OpticalTracks = 2048 + phys.adopt(celer_phys) + phys.dump() + return None + + +runner.physics.setupUserPhysics(setup_physics) +runner.part.userParticleHandler = "" diff --git a/example/ddceler/optical/run-scintwater.sh b/example/ddceler/optical/run-scintwater.sh new file mode 100755 index 0000000000..638b1a4f0b --- /dev/null +++ b/example/ddceler/optical/run-scintwater.sh @@ -0,0 +1,71 @@ +#!/bin/sh -e +#-------------------------------- -*- sh -*- ---------------------------------# +# Copyright Celeritas contributors: see top-level COPYRIGHT file for details +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +#-----------------------------------------------------------------------------# +# Run the optical photon demonstration with or without Celeritas offload. +# +# Usage: +# ./run.sh [celeritas|geant4] [additional ddsim options] +# +# Examples: +# ./run.sh celeritas +# ./run.sh celeritas --numberOfEvents 10 +# ./run.sh geant4 +#-----------------------------------------------------------------------------# + +log() { printf "%s\n" "$1" >&2; } + +resolve_symlinks() { + _path="$1" + while [ -L "$_path" ]; do + _path=$(readlink -f "$_path") + done + printf "%s\n" "$_path" +} + +MODE=$1 +if [ "$MODE" != "celeritas" ] && [ "$MODE" != "geant4" ]; then + log "Usage: $0 [celeritas|geant4] [additional ddsim options]" + exit 1 +fi +shift + +[ "$MODE" = "geant4" ] && export CELER_DISABLE=1 + +EXAMPLE_DIR=$(cd "$(dirname $0)" && pwd) + +if [ -z "${Celeritas_ROOT}" ]; then + Celeritas_ROOT=$(cd "$EXAMPLE_DIR"/../../.. && pwd)/install + log "warning: Celeritas_ROOT is undefined: using ${Celeritas_ROOT}" +fi + +DDSIM=$(command -v ddsim 2>/dev/null || printf "") +[ -z "$DDSIM" ] && { log "error: ddsim not found"; exit 1; } +DDSIM=$(resolve_symlinks "$DDSIM") + +[ -z "$DD4hepINSTALL" ] && { log "error: DD4hepINSTALL not set"; exit 1; } + +CELER_LIB_DIR=$(ls -1 -d "$Celeritas_ROOT"/lib 2>/dev/null | head -1) +[ -z "$CELER_LIB_DIR" ] && { log "error: celeritas not found in $Celeritas_ROOT"; exit 1; } + +if [ "$(uname -s)" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=${CELER_LIB_DIR}:${DYLD_LIBRARY_PATH} +else + export LD_LIBRARY_PATH=${CELER_LIB_DIR}:${LD_LIBRARY_PATH} +fi + +PYTHON=$(command -v python3 2>/dev/null || command -v python 2>/dev/null || printf "") +[ -z "$PYTHON" ] && { log "error: python not found"; exit 1; } +PYTHON=$(resolve_symlinks "$PYTHON") + +log "Running optical demo with ${MODE} physics" +mkdir -p "${EXAMPLE_DIR}/output/${MODE}" +cd "${EXAMPLE_DIR}/output/${MODE}" + +set -x +exec "$PYTHON" "$DDSIM" \ + --compactFile="${EXAMPLE_DIR}/input/ScintWater.xml" \ + --steering="${EXAMPLE_DIR}/input/steeringFile.py" \ + --outputFile="optical_demo.root" \ + "$@" From cab02402dba714419101d7ef2fcae5190edc9e93 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Mon, 2 Mar 2026 10:55:59 -0500 Subject: [PATCH 07/23] Remove redundant ddceler classes for optical offload --- src/ddceler/CelerOpticalOffload.cc | 129 ----------------------------- src/ddceler/CelerOpticalOffload.hh | 42 ---------- 2 files changed, 171 deletions(-) delete mode 100644 src/ddceler/CelerOpticalOffload.cc delete mode 100644 src/ddceler/CelerOpticalOffload.hh diff --git a/src/ddceler/CelerOpticalOffload.cc b/src/ddceler/CelerOpticalOffload.cc deleted file mode 100644 index 2db4f24aa1..0000000000 --- a/src/ddceler/CelerOpticalOffload.cc +++ /dev/null @@ -1,129 +0,0 @@ -//------------------------------- -*- C++ -*- -------------------------------// -// Copyright Celeritas contributors: see top-level COPYRIGHT file for details -// SPDX-License-Identifier: (Apache-2.0 OR MIT) -//---------------------------------------------------------------------------// -//! \file ddceler/CelerOpticalOffload.cc -//---------------------------------------------------------------------------// -#include "CelerOpticalOffload.hh" - -#include -#include -#include -#include -#include -#include - -#include "corecel/io/Logger.hh" -#include "corecel/math/UnitUtils.hh" -#include "geocel/g4/Convert.hh" -#include "accel/LocalOpticalGenOffload.hh" -#include "accel/detail/IntegrationSingleton.hh" - -namespace celeritas -{ -namespace dd -{ -//---------------------------------------------------------------------------// -/*! - * Standard constructor - */ -CelerOpticalOffload::CelerOpticalOffload(dd4hep::sim::Geant4Context* ctxt, - std::string const& name) - : dd4hep::sim::Geant4SteppingAction(ctxt, name) -{ - CELER_LOG(info) << "Registered CelerOpticalOffload stepping action"; -} - -//---------------------------------------------------------------------------// -/*! - * Stepping action to offload optical distributions to Celeritas. - */ -void CelerOpticalOffload::operator()(G4Step const* step, G4SteppingManager*) -{ - CELER_EXPECT(step); - - constexpr double clhep_time{1 / units::nanosecond}; - constexpr double clhep_length{1 / units::centimeter}; - - auto& local = detail::IntegrationSingleton::instance().local_offload(); - if (!local) - { - // Offloading is disabled - return; - } - - if (step->GetStepLength() == 0) - { - // Skip "no-process"-defined steps - return; - } - - auto* pm = step->GetTrack()->GetDefinition()->GetProcessManager(); - CELER_ASSERT(pm); - - // Determine how many Cherenkov and scintillation photons to generate - size_type num_cherenkov{0}; - size_type num_scintillation{0}; - - if (auto const* p = dynamic_cast(pm->GetProcess("Cerenk" - "ov"))) - { - num_cherenkov = p->GetNumPhotons(); - } - if (auto const* p = dynamic_cast( - pm->GetProcess("Scintillation"))) - { - num_scintillation = p->GetNumPhotons(); - } - - if (num_cherenkov == 0 && num_scintillation == 0) - { - return; - } - - auto* pre_step = step->GetPreStepPoint(); - auto* post_step = step->GetPostStepPoint(); - CELER_ASSERT(pre_step && post_step); - - // Create distribution and push to Celeritas - // TODO: Get optical material ID from geometry - optical::GeneratorDistributionData data; - data.time = convert_from_geant(post_step->GetGlobalTime(), clhep_time); - data.step_length = convert_from_geant(step->GetStepLength(), clhep_length); - data.charge = units::ElementaryCharge{ - static_cast(post_step->GetCharge())}; - data.material = OptMatId(0); // TODO: map from G4Material to OptMatId - data.points[StepPoint::pre] - = {units::LightSpeed(pre_step->GetBeta()), - convert_from_geant(pre_step->GetPosition(), clhep_length)}; - data.points[StepPoint::post] - = {units::LightSpeed(post_step->GetBeta()), - convert_from_geant(post_step->GetPosition(), clhep_length)}; - - auto& gen_offload = dynamic_cast(local); - - if (num_cherenkov > 0) - { - data.type = GeneratorType::cherenkov; - data.num_photons = num_cherenkov; - CELER_ASSERT(data); - gen_offload.Push(data); - } - if (num_scintillation > 0) - { - data.type = GeneratorType::scintillation; - data.num_photons = num_scintillation; - CELER_ASSERT(data); - gen_offload.Push(data); - } - - CELER_LOG_LOCAL(debug) << "Offloading " << num_cherenkov - << " Cherenkov and " << num_scintillation - << " scintillation photons"; -} - -//---------------------------------------------------------------------------// -} // namespace dd -} // namespace celeritas - -DECLARE_GEANT4ACTION_NS(celeritas::dd, CelerOpticalOffload) diff --git a/src/ddceler/CelerOpticalOffload.hh b/src/ddceler/CelerOpticalOffload.hh deleted file mode 100644 index 41495c5171..0000000000 --- a/src/ddceler/CelerOpticalOffload.hh +++ /dev/null @@ -1,42 +0,0 @@ -//------------------------------- -*- C++ -*- -------------------------------// -// Copyright Celeritas contributors: see top-level COPYRIGHT file for details -// SPDX-License-Identifier: (Apache-2.0 OR MIT) -//---------------------------------------------------------------------------// -//! \file ddceler/CelerOpticalOffload.hh -//---------------------------------------------------------------------------// -#pragma once - -#include - -namespace celeritas -{ -namespace dd -{ -//---------------------------------------------------------------------------// -/*! - * DDG4 stepping action for optical distribution offloading to Celeritas. - * - * This action intercepts Cherenkov and scintillation photon generation in - * Geant4 and offloads the distribution data to Celeritas for GPU tracking. - * - * The optical physics in Geant4 must be configured to *not* stack photons - * (stack_photons = false) so that photon counts are calculated but photons - * are not created in Geant4. - */ -class CelerOpticalOffload final : public dd4hep::sim::Geant4SteppingAction -{ - public: - // Standard constructor - CelerOpticalOffload(dd4hep::sim::Geant4Context* ctxt, - std::string const& name); - - // Delete copy/move - DDG4_DEFINE_ACTION_CONSTRUCTORS(CelerOpticalOffload); - - // Stepping action callback - virtual void operator()(G4Step const* step, G4SteppingManager* mgr) final; -}; - -//---------------------------------------------------------------------------// -} // namespace dd -} // namespace celeritas From 36419af042603a0cf9bae6825e306d337d81b8d8 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 12 Mar 2026 13:32:29 -0400 Subject: [PATCH 08/23] SurfaceSteppingAction (runs at post) kills photons before the old post-ordered DetectorAction could score them. Moving to user_post and scoring killed tracks closes that gap. The direction field and volume hierarchy storage are added here since they are part of the same executor rewrite. --- .../optical/action/DetectorAction.hh | 12 ++-- .../optical/action/detail/DetectorExecutor.hh | 68 ++++++++++++------- 2 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/celeritas/optical/action/DetectorAction.hh b/src/celeritas/optical/action/DetectorAction.hh index fa6a8df9b9..7db2ee8986 100644 --- a/src/celeritas/optical/action/DetectorAction.hh +++ b/src/celeritas/optical/action/DetectorAction.hh @@ -50,12 +50,12 @@ class DetectorAction final : public OpticalStepActionInterface, void step(CoreParams const&, CoreStateDevice&) const final; //! Dependency ordering of the action - StepActionOrder order() const final { return StepActionOrder::post; } + StepActionOrder order() const final { return StepActionOrder::user_post; } private: //// TYPES //// - using VecHit = std::vector; + using HitsOutput = DetectorHitsOutput; //// DATA //// @@ -63,11 +63,11 @@ class DetectorAction final : public OpticalStepActionInterface, //// HELPER FUNCTIONS //// - // Copy hits from device - VecHit load_hits_sync(CoreStateDevice const&) const; + // Copy hits from device and return as output struct + HitsOutput load_hits_sync(CoreStateDevice const&) const; - // Send hits to the callback - void callback_hits(VecHit const&) const; + // Build output and send to the callback + void callback_hits(HitsOutput&&) const; }; //---------------------------------------------------------------------------// diff --git a/src/celeritas/optical/action/detail/DetectorExecutor.hh b/src/celeritas/optical/action/detail/DetectorExecutor.hh index dca29a6ad6..e4aad5d771 100644 --- a/src/celeritas/optical/action/detail/DetectorExecutor.hh +++ b/src/celeritas/optical/action/detail/DetectorExecutor.hh @@ -6,6 +6,7 @@ //---------------------------------------------------------------------------// #pragma once +#include "corecel/cont/Range.hh" #include "celeritas/optical/CoreTrackView.hh" #include "celeritas/optical/DetectorData.hh" @@ -20,10 +21,13 @@ namespace detail * Populate detector state buffer at the end of a step. * * All tracks have hits copied into the state buffer. If the track is not alive - * or is not in a detector region, an invalid hit is set in the corresponding - * buffer track slot. + * or killed, or is not in a detector region, an invalid hit is set in the + * corresponding buffer track slot. * - * When a track generates a valid hit, it is killed (absorbed by the detector). + * This action runs at \c user_post, after surface interactions (which absorb + * photons and set \c TrackStatus::killed ). Both \c alive and \c killed tracks + * are scored, analogous to how the EM \c StepGatherExecutor handles killed + * tracks. Inactive and errored tracks are skipped. */ struct DetectorExecutor { @@ -45,37 +49,51 @@ DetectorExecutor::operator()(CoreTrackView const& track) const auto& hit = detector_state_.detector_hits[track.track_slot_id()]; auto sim = track.sim(); - if (sim.status() == TrackStatus::alive) + auto const status = sim.status(); + if (status == TrackStatus::inactive || status == TrackStatus::errored) { - auto const detectors = track.detectors(); - - auto geometry = track.geometry(); + // Skip empty slots and errored tracks + hit.detector = {}; + return; + } - auto const volume_id = geometry.volume_id(); - auto const detector_id = detectors.detector_id(volume_id); + auto const detectors = track.detectors(); + auto geometry = track.geometry(); + auto const volume_id = geometry.volume_id(); + auto const detector_id = detectors.detector_id(volume_id); - if (detector_id) - { - // Score a valid hit - hit = DetectorHit{detector_id, - track.sim().primary_id(), - track.particle().energy(), - sim.time(), - geometry.pos(), - geometry.volume_instance_id()}; + if (detector_id) + { + // Score a valid hit for alive or killed tracks in a detector volume + hit = DetectorHit{detector_id, + track.particle().energy(), + sim.time(), + geometry.pos(), + geometry.dir(), + geometry.volume_instance_id()}; - // Kill the track - sim.status(TrackStatus::killed); - } - else + // Store full volume hierarchy if buffer is allocated + auto const num_levels = detector_state_.num_volume_levels; + if (num_levels > 0) { - // Mark that the track is not in a detector - hit.detector = {}; + auto const tid = track.track_slot_id(); + auto all_ids + = detector_state_ + .volume_instance_ids[AllItems{}]; + auto dst = all_ids.subspan(tid.unchecked_get() * num_levels, + num_levels); + size_type depth = geometry.volume_level().unchecked_get() + 1; + CELER_ASSERT(depth <= dst.size()); + geometry.volume_instance_id(dst.first(depth)); + for (auto level : range(depth, num_levels)) + { + dst[level] = {}; + } } } else { - // Ensure killed, inactive, and errored tracks don't contribute to hits + // Track is not in a detector volume hit.detector = {}; } } From 4744661a2e2cc6106e5f9b853c70268549696af4 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 12 Mar 2026 13:47:58 -0400 Subject: [PATCH 09/23] remove PrimaryId primary from the struct, since the executor never passes it and it's not needed by OpticalHitProcessor --- src/celeritas/optical/DetectorData.hh | 58 +++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/celeritas/optical/DetectorData.hh b/src/celeritas/optical/DetectorData.hh index e5bd6ace19..9aa6dd2424 100644 --- a/src/celeritas/optical/DetectorData.hh +++ b/src/celeritas/optical/DetectorData.hh @@ -6,6 +6,8 @@ //---------------------------------------------------------------------------// #pragma once +#include + #include "corecel/data/Collection.hh" #include "celeritas/Quantities.hh" #include "celeritas/optical/Types.hh" @@ -23,10 +25,10 @@ struct DetectorHit using Energy = units::MevEnergy; DetectorId detector{}; - PrimaryId primary{}; Energy energy; real_type time{}; Real3 position{}; + Real3 direction{}; VolumeInstanceId volume_instance; //! An actual hit has a valid detector @@ -42,6 +44,11 @@ struct DetectorHit * * Detector hits is large enough to store a hit for every track at the end of a * step. Stored hits may be invalid if the track is not in a detector region. + * + * When \c num_volume_levels is nonzero, \c volume_instance_ids stores the full + * volume hierarchy for each track slot as a flat buffer of size + * \c num_track_slots * num_volume_levels, indexed + * \c [track_slot * num_volume_levels + level]. */ template struct DetectorStateData @@ -50,10 +57,18 @@ struct DetectorStateData //! \name Type aliases template using StateItems = StateCollection; + template + using Items = Collection; //!@} StateItems detector_hits; + //! Flat volume hierarchy buffer: size = num_track_slots * + //! num_volume_levels + Items volume_instance_ids; + //! Number of volume levels per track slot (0 if hierarchy not stored) + size_type num_volume_levels{0}; + //! Whether data is assigned and valid explicit CELER_FUNCTION operator bool() const { @@ -69,6 +84,8 @@ struct DetectorStateData { CELER_EXPECT(other); detector_hits = other.detector_hits; + volume_instance_ids = other.volume_instance_ids; + num_volume_levels = other.num_volume_levels; return *this; } }; @@ -78,19 +95,54 @@ struct DetectorStateData //---------------------------------------------------------------------------// /*! * Resize the state in host code. + * + * When \c num_levels is nonzero, allocates space for the full volume hierarchy + * buffer (size * num_levels entries). */ template -inline void -resize(DetectorStateData* state, size_type size) +inline void resize(DetectorStateData* state, + size_type size, + size_type num_levels = 0) { CELER_EXPECT(state); CELER_EXPECT(size > 0); resize(&state->detector_hits, size); + if (num_levels > 0) + { + resize(&state->volume_instance_ids, size * num_levels); + state->num_volume_levels = num_levels; + } + CELER_ENSURE(*state); } //---------------------------------------------------------------------------// } // namespace optical + +//---------------------------------------------------------------------------// +/*! + * Host-side output of optical detector hits, including full volume hierarchy. + * + * This is populated by \c optical::DetectorAction and passed to the + * \c HitCallbackFunc. The \c volume_instance_ids flat buffer is indexed as + * \c [hit_idx * num_volume_levels + level], world at level 0, leaf at + * \c num_volume_levels - 1. The buffer is empty when \c num_volume_levels + * is zero (i.e. Geant4 SD integration is disabled). + */ +struct DetectorHitsOutput +{ + std::vector hits; + //! Flat volume hierarchy for each hit (size = hits.size() * + //! num_volume_levels) + std::vector volume_instance_ids; + //! Number of volume levels per hit + size_type num_volume_levels{0}; + + //! True when there are hits to process + explicit operator bool() const { return !hits.empty(); } +}; + +//---------------------------------------------------------------------------// } // namespace celeritas From 1a9564669ca8d0e3ad472be1ec16497c6516ed83 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Thu, 12 Mar 2026 14:07:55 -0400 Subject: [PATCH 10/23] Extend DetectorHit output to include volume hierarchy --- src/celeritas/inp/Scoring.hh | 43 ++++--- .../optical/action/DetectorAction.cc | 109 ++++++++++++++---- .../optical/action/DetectorAction.cu | 3 +- 3 files changed, 107 insertions(+), 48 deletions(-) diff --git a/src/celeritas/inp/Scoring.hh b/src/celeritas/inp/Scoring.hh index ec9342843c..04260dd06f 100644 --- a/src/celeritas/inp/Scoring.hh +++ b/src/celeritas/inp/Scoring.hh @@ -20,10 +20,7 @@ class G4LogicalVolume; namespace celeritas { -namespace optical -{ -struct DetectorHit; -} +struct DetectorHitsOutput; namespace inp { @@ -138,22 +135,6 @@ struct SimpleCalo std::vector