diff --git a/docs/adjust_CTD.md b/docs/adjust_CTD.md new file mode 100644 index 00000000..61d6b7ce --- /dev/null +++ b/docs/adjust_CTD.md @@ -0,0 +1,198 @@ +# PyGlider: Adjust CTD Variables + +PyGlider applies a post-processing protocol to conductivity, temperature, and salinity variables in NetCDF timeseries files, and then generates depth–time NetCDF grids using `xarray`. The resulting NetCDF files are largely CF-compliant. + +The basic workflow converts a NetCDF timeseries into an adjusted timeseries and corresponding depth–time grids. This follows the `binary_to_timeseries` (for Slocum gliders) and `raw_to_timeseries` (for Alseamar gliders) protocols, which convert raw glider data into NetCDF format. outname refers to the NetCDF timeseries produced by the initial conversion step. + +The adjusted file is generated through three steps: flagging CTD data, adjusting CTD variables, and gridding the time series. This workflow uses known thermal lag constants and the lag between temperature and conductivity signals for each sensor. + +An example of how to determine these constants is available at: +https://cproof.uvic.ca/gliderdata/deployments/reports/ + +--- + +## Workflow Overview + +The CTD adjustment pipeline consists of three main steps: + +1. **Flag CTD data** + Identify and flag unphysical conductivity, temperature, and salinity values. + +2. **Apply CTD corrections** + - Correct temperature–conductivity lag (`dTdC`) + - Apply thermal lag correction (`alpha`, `tau`) + +3. **Generate gridded products** + Create depth–time NetCDF grids from the adjusted time series. + +--- + +## User Options + +The following parameters can be customized: + +- **alpha, tau** + Thermal lag correction constants. + - Can be provided as function arguments + - Or stored in the deployment YAML file + - If neither is provided, thermal lag correction is not applied + +- **dTdC** + Time lag (seconds) between temperature and conductivity sensors + +- **interpolate_filter** + Optional function to interpolate over small gaps before applying thermal lag correction + +- **max_gap (in gridding)** + Maximum vertical gap size (in meters) to interpolate + +--- + +## Example Processing Script + +```python +import logging +from pathlib import Path +import xarray as xr +import pyglider.ncprocess as ncprocess +import pyglider.utils as utils + +logging.basicConfig(level=logging.INFO) +_log = logging.getLogger(__name__) + +cwd = Path.cwd() +deploy_name = cwd.name +glider_name = cwd.parent.name + +openfile = f'./L0-timeseries/{deploy_name}_delayed.nc' +deploymentyaml = './deploymentRealtime.yml' +gridpath = './L0-gridfiles/' +ts_path = './L0-timeseries/' + +ts = xr.open_dataset(openfile) +deployment = utils._get_deployment(deploymentyaml) +ts = utils.flag_CTD_data(ts) +ts2 = utils.adjust_CTD(ts, deployment, interpolate_filter=None) + +outfile = f'{ts_path}/{deploy_name}_adjusted.nc' +_log.info('Saving adjusted timeseries to netcdf') +outname = ts2.to_netcdf(outfile) + +outname2 = ncprocess.make_gridfiles( + outname, + gridpath, + deploymentyaml, + fnamesuffix='_adjusted', + maskfunction=None, + max_gap=50 +) +``` + +The following sections describe each processing step in detail. + +--- + +## flag_CTD_data + +This step identifies and flags CTD values that are clearly unphysical (QC4), typically caused by air bubbles in the conductivity cell. + +Conductivity data are grouped into profile bins (`d_profile`) and depth bins (`dz`). Within each depth bin: + +1. Values more than 5 standard deviations from the mean are temporarily excluded +2. A cleaned mean and standard deviation are recomputed +3. Values exceeding `clean_stdev` from the cleaned mean are flagged as QC4 + +If `accuracy` is provided, small deviations below this threshold are not flagged. + +General behavior: +- QC variables are added if missing +- Values not flagged as QC4 are set to QC1 +- `salinity_QC` is set to QC4 wherever `conductivity_QC` is QC4 + +--- + +## adjust_CTD + +This step applies two corrections: + +1. **CT lag correction (`dTdC`)** +2. **Thermal lag correction (`alpha`, `tau`)** + +Correction constants can be: +- Passed as function arguments +- Loaded from the deployment YAML file + +If both are provided and differ, function arguments take precedence and a warning is issued. + +### Example YAML configuration + +```yaml +glider_devices: + ctd: + Thermal_lag_constants_[alpha,tau]: [0.2, 2] + dTdC: 0 +``` + +--- + +### CT lag correction + +If `dTdC` is not `None`, temperature is shifted back in time to align with conductivity. + +If `dTdC` is `None` or `0`, no lag correction is applied. + +--- + +### Thermal lag correction + +If `alpha` and `tau` are provided, `apply_thermal_lag` is used to: +- Estimate conductivity cell temperature +- Recalculate `salinity_adjusted` + +--- + +## Optional: interpolate_filter + +An optional preprocessing step can be applied to reduce noise and prevent spikes from propagating during filtering. This function should take and return an xarray Dataset (e.g., linear interpolation over short gaps). + +--- + +--- + +## Output Variables + +The adjusted dataset includes: + +- `temperature_adjusted` +- `salinity_adjusted` +- `potential_density_adjusted` +- `potential_temperature_adjusted` + +Quality control flags are propagated to adjusted variables. + +Metadata are updated to document: +- Applied corrections +- Processing date +- Data provenance + +--- + +## Notes + +- Thermal lag correction is only applied if `alpha` and `tau` are defined +- CT lag correction is only applied if `dTdC` is non-zero +- QC4 values may be excluded from processing if a masking function is applied during gridding +- Small data gaps can be interpolated prior to filtering to improve stability + +--- + +## Summary + +This workflow provides a reproducible method for: +- Flagging bad CTD data +- Correcting sensor response lags +- Producing adjusted physical variables +- Generating gridded NetCDF products + +It is designed to be flexible, allowing users to customize correction parameters and preprocessing steps depending on sensor characteristics and mission requirements. + diff --git a/docs/index.md b/docs/index.md index 3b4fcaa6..2f03c801 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,7 @@ Install getting-started-seaexplorer getting-started-slocum pyglider/pyglider - +adjust_CTD ``` ## Acknowledgements diff --git a/pyglider/ncprocess.py b/pyglider/ncprocess.py index b366d417..35ede770 100644 --- a/pyglider/ncprocess.py +++ b/pyglider/ncprocess.py @@ -9,7 +9,13 @@ import netCDF4 import numpy as np import scipy.stats as stats +from scipy import signal import xarray as xr +import gsw +import yaml +from datetime import date + + import pyglider.utils as utils @@ -187,14 +193,16 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml, force=False): def make_gridfiles( - inname, - outdir, - deploymentyaml, - *, - fnamesuffix='', - depth_bins=None, - dz=1, - starttime='1970-01-01', + inname, + outdir, + deploymentyaml, + *, + fnamesuffix='', + depth_bins=None, + dz=1, + starttime='1970-01-01', + maskfunction=None, + max_gap=100, ): """ Turn a timeseries netCDF file into a vertically gridded netCDF. @@ -213,19 +221,25 @@ def make_gridfiles( depth_bins : array, default = None User-defined depth bins, for instance ``np.arange(0, 1000.1, 1)``. - If not None, these are the depth bins into which the data will be + If not None, these are the depth bins into which the data will be gridded. If None, ``dz`` is used to generate bins between 0 and 1100m dz : float, default = 1 - Vertical grid spacing in meters. Ignored if ``depth_bins`` is not None + Vertical grid spacing in meters. + + maskfunction : callable or None, optional + Function applied to the dataset before gridding, + usually to choose what data will be set to NaN based on quality flags. - Returns + max_gap : int, default = 100 + Maximum number of consecutive NaN values to fill when interpolating. This is used to + prevent interpolation across large gaps in the data. ------- outname : str - Name of gridded netCDF file. The gridded netCDF file has dimensions of + Name of gridded netCDF file. The gridded netCDF file has coordinates of 'depth' and 'profile', so each variable is gridded in depth bins and by - profile number. Each profile has a time, latitude, and longitude. - The depth values are the bin centers + profile number. Each profile has a time, latitude, and longitude. + If deploymentyaml is a list, attributes """ try: os.mkdir(outdir) @@ -236,6 +250,11 @@ def make_gridfiles( profile_meta = deployment['profile_variables'] + ds = xr.open_dataset(inname, decode_times=True) + + if maskfunction is not None: + ds = maskfunction(ds) + ds = xr.open_dataset(inname, decode_times=True) ds = ds.where(ds.time > np.datetime64(starttime), drop=True) _log.info(f'Working on: {inname}') @@ -256,7 +275,7 @@ def make_gridfiles( else: # sanity check user-provided bins if ( - depth_bins.ndim != 1 + depth_bins.ndim != 1 or not np.all(np.isfinite(depth_bins)) or not np.issubdtype(depth_bins.dtype, np.number) ): @@ -265,7 +284,7 @@ def make_gridfiles( raise ValueError('There must be at least two depth bins edges') if not np.all(np.diff(depth_bins) > 0): raise ValueError('Depth bin edges must be strictly increasing and non-overlapping') - + # calculate bin centers depths = 0.5*(depth_bins[:-1] + depth_bins[1:]) _log.debug(f'depth bins and centers {depth_bins} {{depths}}') @@ -279,7 +298,7 @@ def make_gridfiles( 'long_name': 'Depth', 'standard_name': 'depth', 'positive': 'down', - 'source': ds.depth.attrs["source"], + 'source': ds.depth.attrs["source"], 'coverage_content_type': 'coordinate', 'comment': 'center of depth bins', } @@ -287,17 +306,24 @@ def make_gridfiles( # Bin by profile index, for the mean time, lat, and lon values for each profile ds['time_1970'] = ds.temperature.copy() ds['time_1970'].values = ds.time.values.astype(np.float64) + # print(ds.time_1970.values) + # print(ds.profile_index.values) for td in ('time_1970', 'longitude', 'latitude'): + good = np.where(~np.isnan(ds[td]) & (ds['profile_index'] % 1 == 0))[0] - dat, xedges, binnumber = stats.binned_statistic( - ds['profile_index'].values[good], - ds[td].values[good], - statistic='mean', - bins=[profile_bins], - ) - if td == 'time_1970': - td = 'time' - dat = dat.astype('timedelta64[ns]') + np.datetime64('1970-01-01T00:00:00') + if len(good) > 1: + dat, xedges, binnumber = stats.binned_statistic( + ds['profile_index'].values[good], + ds[td].values[good], + statistic='mean', + bins=[profile_bins], + ) + if td == 'time_1970': + td = 'time' + dat = dat.astype('timedelta64[ns]') + np.datetime64('1970-01-01T00:00:00') + else: + dat = np.full(len(profiles), np.nan) + _log.info(f'Only {len(good)} good values for {td}, filling with NaN') _log.info(f'{td} {len(dat)}') dsout[td] = (('time'), dat, ds[td].attrs) @@ -316,9 +342,10 @@ def make_gridfiles( _log.info(f'{td} {len(dat)}') dsout[td] = ((xdimname), dat, profile_meta[td]) - ds = ds.drop('time_1970') + ds = ds.drop_vars('time_1970') _log.info(f'Done times!') + for k in ds.keys(): if k in ['time', 'profile', 'longitude', 'latitude', 'depth'] or 'time' in k: continue @@ -326,32 +353,38 @@ def make_gridfiles( good = np.where(~np.isnan(ds[k]) & (ds['profile_index'] % 1 == 0))[0] if len(good) <= 0: continue - if 'average_method' in ds[k].attrs: - average_method = ds[k].attrs['average_method'] - ds[k].attrs['processing'] = ( - f'Using average method {average_method} for ' - f'variable {k} following deployment yaml.' - ) - if average_method == 'geometric mean': - average_method = stats.gmean - ds[k].attrs['processing'] += ( - ' Using geometric mean implementation ' 'scipy.stats.gmean' - ) + if 'QC_protocol' in ds[k].attrs.values(): + # QC variables are treated as discrete flags rather than continuous data. + # If a variable has a QC_protocol attribute, it is gridded using the + # maximum flag in each bin (e.g. any QC3 in a bin makes the gridded bin QC3). + method = np.nanmax else: - average_method = 'mean' + # variables are treated as continuous data. + # If a variable has a average_method attribute, it is gridded using the + # mean in each bin + if 'average_method' in ds[k].attrs.values(): + method = ds[k].attrs['average_method'] + if method == 'geometric mean': + method = stats.gmean + else: + method = 'mean' + dat, xedges, yedges, binnumber = stats.binned_statistic_2d( ds['profile_index'].values[good], ds['depth'].values[good], values=ds[k].values[good], - statistic=average_method, + statistic=method, bins=[profile_bins, depth_bins], ) _log.debug(f'dat{np.shape(dat)}') dsout[k] = (('depth', xdimname), dat.T, ds[k].attrs) - # fill gaps in data: - dsout[k].values = utils.gappy_fill_vertical(dsout[k].values) + dsout[k] = dsout[k].interpolate_na( + dim="depth", + method="linear", + max_gap=max_gap, + ) # fix u and v, because they should really not be gridded... if ('water_velocity_eastward' in dsout.keys()) and ('u' in profile_meta.keys()): @@ -413,19 +446,19 @@ def make_gridfiles( dsout.to_netcdf( outname, encoding={ - 'time': time_encoding, - 'profile_time_start': time_encoding, - 'profile_time_end': time_encoding, + 'time': time_encoding, + 'profile_time_start': time_encoding, + 'profile_time_end': time_encoding, }, ) _log.info('Done gridding') return outname - + _log.info('Done gridding') # aliases extract_L0timeseries_profiles = extract_timeseries_profiles make_L0_gridfiles = make_gridfiles -__all__ = ['extract_timeseries_profiles', 'make_gridfiles'] +__all__ = ['extract_timeseries_profiles', 'make_gridfiles'] \ No newline at end of file diff --git a/pyglider/process_adjusted.py b/pyglider/process_adjusted.py new file mode 100644 index 00000000..3e191edc --- /dev/null +++ b/pyglider/process_adjusted.py @@ -0,0 +1,56 @@ +import logging +from pathlib import Path +import xarray as xr + +import pyglider.ncprocess as ncprocess +import pyglider.utils as utils + +logging.basicConfig(level=logging.INFO) +_log = logging.getLogger(__name__) + + +def run_process_adjusted( + base_dir, + deploy_name=None, + deployfile=None, + adjustedyaml=None, +): + base_dir = Path(base_dir) + + deploy_name = deploy_name or base_dir.name + + # --- Paths --- + ts_path = base_dir / 'L0-timeseries' + gridpath = base_dir / 'L0-gridfiles' + + openfile = ts_path / f'{deploy_name}.nc' + deployfile = Path(deployfile) if deployfile else base_dir / 'deploymentRealtime.yml' + adjustedyaml = Path(adjustedyaml) if adjustedyaml else base_dir / 'adjusted.yml' + + # --- Load --- + ts = xr.open_dataset(openfile) + deployment = utils._get_deployment([deployfile, adjustedyaml]) + + # --- Processing --- + ts = utils.flag_CTD_data(ts) + ts = utils.adjust_CTD(ts, deployment) + + # --- Save --- + outfile = ts_path / f'{deploy_name}_adjusted.nc' + _log.info('Saving adjusted timeseries to netcdf') + ts.to_netcdf(outfile) + + # --- Grid --- + ncprocess.make_gridfiles( + str(outfile), + str(gridpath), + [str(deployfile), str(adjustedyaml)], + fnamesuffix='_adjusted', + maskfunction=utils.maskQC4, + ) + + return outfile + + +if __name__ == "__main__": + run_process_adjusted(Path.cwd()) \ No newline at end of file diff --git a/pyglider/utils.py b/pyglider/utils.py index dc439d1c..dc482b0c 100644 --- a/pyglider/utils.py +++ b/pyglider/utils.py @@ -11,6 +11,8 @@ import xarray as xr import yaml from scipy.signal import argrelextrema +from scipy import signal +from datetime import datetime, date from pyglider._version import __version__ @@ -347,10 +349,10 @@ def get_derived_eos_raw(ds): [ ('long_name', 'water salinity'), ('standard_name', 'sea_water_practical_salinity'), - ('units', '1'), - ('comment', 'raw, uncorrected practical salinity; Units are also known as PSU'), + ('units', '1e-3'), + ('comment', 'raw, uncorrected salinity'), ('sources', 'conductivity temperature pressure'), - ('method', 'pyglider.utils.get_derived_eos_raw; gsw.conversions.SP_from_C'), + ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), ('valid_max', 40.0), @@ -372,9 +374,9 @@ def get_derived_eos_raw(ds): ('long_name', 'water potential density'), ('standard_name', 'sea_water_potential_density'), ('units', 'kg m-3'), - ('comment', 'raw, uncorrected potential density'), + ('comment', 'raw, uncorrected salinity'), ('sources', 'salinity temperature pressure'), - ('method', 'pyglider.utils.get_derived_eos_raw; gsw.density.sigma0 using gsw.SA_from_SP and gsw.CT_from_t'), + ('method', 'get_derived_eos_raw'), ('observation_type', 'calulated'), ('instrument', 'instrument_ctd'), ('accuracy', 0.01), @@ -394,11 +396,11 @@ def get_derived_eos_raw(ds): ('long_name', 'Density'), ('standard_name', 'sea_water_density'), ('units', 'kg m-3'), - ('comment', 'raw, uncorrected density'), + ('comment', 'raw, uncorrected salinity'), ('observation_type', 'calulated'), ('sources', 'salinity temperature pressure'), ('instrument', 'instrument_ctd'), - ('method', 'pyglider.utils.get_derived_eos_raw; gsw.density.rho using gsw.SA_from_SP and gsw.CT_from_t'), + ('method', 'get_derived_eos_raw'), ('valid_min', 990.0), ('valid_max', 1040.0), ('accuracy', 0.01), @@ -417,10 +419,10 @@ def get_derived_eos_raw(ds): ('long_name', 'water potential temperature'), ('standard_name', 'sea_water_potential_temperature'), ('units', 'Celsius'), - ('comment', 'raw, uncorrected potential temperature'), + ('comment', 'raw, uncorrected salinity'), ('sources', 'salinity temperature pressure'), ('observation_type', 'calulated'), - ('method', 'pyglider.utils.get_derived_eos_raw; gsw.conversions.pt0_from_t using gsw.SA_from_SP'), + ('method', 'get_derived_eos_raw'), ('instrument', 'instrument_ctd'), ('accuracy', 0.002), ('precision', 0.001), @@ -657,26 +659,6 @@ def _passthrough(val): return val -def gappy_fill_vertical(data): - """ - Fill vertical gaps from the first to last bin with data in them. - Applied column-wise. - - data = gappy_fill_vertical(data) - """ - m, n = np.shape(data) - for j in range(n): - ind = np.where(~np.isnan(data[:, j]))[0] - if ( - len(ind) > 0 - and len(ind) < (ind[-1] - ind[0]) - and len(ind) > (ind[-1] - ind[0]) * 0.05 - ): - int = np.arange(ind[0], ind[-1]) - data[:, j][ind[0] : ind[-1]] = np.interp(int, ind, data[ind, j]) - return data - - def find_gaps(sample_time, timebase, maxgap): """ Return an index into *timebase* where True are times in gaps of *sample_time* larger @@ -845,6 +827,612 @@ def _get_glider_name_slocum(current_directory): return glider, mission, slocum_glider +def flag_conductivity_in_depth_space( + ts0, + d_profile=50, + dz=5, + clean_stdev=3, + accuracy=None, +): + """ + Flag conductivity as QC1 (good) or QC4 (bad) using profile bins and depth bins. + + Conductivity data are grouped into profile bins of width `d_profile` and depth bins + of width `dz`. Within each depth bin, points farther than 5 standard deviations from + the mean are temporarily excluded. A second mean and standard deviation are then + computed from the remaining points, and values farther than `clean_stdev` standard + deviations from that cleaned mean are flagged as QC4. If `accuracy` is provided, + deviations smaller than `accuracy` are not flagged. + + Parameters + ---------- + ts0 : xarray.Dataset + Timeseries dataset containing conductivity, depth, and profile_index. + + d_profile : float, optional + Width of the profile bins. + + dz : float, optional + Width of the depth bins in meters. + + clean_stdev : float, optional + Number of standard deviations used in the second-pass flagging step. + + accuracy : float or None, optional + Sensor accuracy threshold. Deviations smaller than this are not flagged. + + Returns + ------- + qc : np.ndarray + Array of QC flags with the same shape as ts0.conductivity. + Good data are flagged as 1, bad data as 4. + """ + ts = ts0.copy(deep=True).load() + ts = ts.where(np.isfinite(ts.conductivity), drop=False) + _log.info( + 'Flagging conductivity in profile-depth space with ' + 'd_profile=%s, dz=%s, clean_stdev=%s, accuracy=%s', + d_profile, + dz, + clean_stdev, + accuracy, + ) + prof_vals = ts.profile_index.values + depth_vals = ts.depth.values + cond_vals = ts.conductivity.values + + prof_bins = np.arange( + np.nanmin(prof_vals), + np.nanmax(prof_vals) + d_profile, + d_profile, + ) + zbins = np.arange(np.nanmin(depth_vals), np.nanmax(depth_vals) + dz, dz) + + qc = np.full(cond_vals.shape, 4, dtype=int) + finite_cond = np.isfinite(cond_vals) + + acc_thresh = 0 if accuracy is None else accuracy + + for n in range(len(prof_bins) - 1): + ind_profbin = ( + (prof_vals >= prof_bins[n]) & + (prof_vals < prof_bins[n + 1]) & + finite_cond + ) + + if not np.any(ind_profbin): + continue + + cond = cond_vals[ind_profbin] + depth = depth_vals[ind_profbin] + + ind_bad_z = np.zeros(len(cond), dtype=bool) + + for m in range(len(zbins) - 1): + ind_zbin = (depth >= zbins[m]) & (depth < zbins[m + 1]) + + if not np.any(ind_zbin): + continue + + var_z = cond[ind_zbin] + var_mean = np.nanmean(var_z) + var_std = np.nanstd(var_z) + + if not np.isfinite(var_std) or var_std == 0: + ind_bad = np.zeros_like(var_z, dtype=bool) + else: + ind_flag = ( + (np.abs(var_z - var_mean) > 5 * var_std) & + (np.abs(var_z - var_mean) > acc_thresh) + ) + + if np.all(ind_flag): + ind_bad = ind_flag + else: + clean_mean = np.nanmean(var_z[~ind_flag]) + clean_std = np.nanstd(var_z[~ind_flag]) + + if not np.isfinite(clean_std) or clean_std == 0: + ind_bad = np.zeros_like(var_z, dtype=bool) + else: + ind_bad = ( + (np.abs(var_z - clean_mean) > clean_stdev * clean_std) & + (np.abs(var_z - clean_mean) > acc_thresh) + ) + + ind_bad_z[ind_zbin] = ind_bad + + qc_subset = np.where(ind_bad_z, 4, 1) + qc[ind_profbin] = qc_subset + + return qc + + +def interpolate_over_salinity_NANs(ds): + """ + Function applied to the dataset before finding the internal temperature. + Function interpolates temperature over bad data and small data gaps + to prevent errors from affecting the neighbouring cells. + + Parameters + ---------- + ds: DataArray + Timeseries of mission data + + Returns + ---------- + interp: DataArray + Timeseries of interpolated temperature + + """ + _log.info( + 'Interpolating temperature over salinity NaNs and small data gaps ' + 'before applying thermal lag correction' + ) + interp = ds["temperature"].where(ds["temperature_QC"] != 4) + qc4 = (ds["temperature_QC"] == 4) + qc4_buf = qc4.rolling(time=5, center=True, min_periods=1).max().astype(bool) + interp = interp.where(~qc4_buf) + + interp = interp.interpolate_na( + dim="time", + method="linear", + max_gap=np.timedelta64(60, "s")) + + return interp + +def apply_thermal_lag( + ds, + fn, + alpha, + tau, + interpolate_filter=None, +): + """ + Function from Garau et al. (2011): estimates temperature inside the + conductivity cell then recalculates salinity + + Parameters + ---------- + ds: DataArray + Timeseries of mission data + + fn: float + Sampling frequency of the sensor + + alpha : float + Thermal lag strength constant for the sensor. + + tau: float + Thermal lag time constant for the sensor. + + interpolate_filter: callable or None, optional + Function applied to the dataset before finding the internal temperature. + Function interpolates over bad data and small data gaps + to prevent errors from affecting the neighbouring cells. + + Returns + ---------- + sal: DataArray + Timeseries of salinity_adjusted calculated using the internal temperature of + the conductivity cell. + """ + if interpolate_filter is not None: + temp = interpolate_filter(ds) + _log.info('Interpolating over bad data and small data gaps before' + 'applying thermal lag correction') + + else: + temp = ds.temperature + + _log.info( + 'Applying thermal lag correction with alpha = %s, tau = %s, ' + 'and sampling frequency = %s', + alpha, + tau, + fn, + ) + a = 4 * fn * alpha * tau / (1 + 4*fn*tau) + b = 1 - 2 * a / alpha + aa = [1, b] + bb = [a, -a] + tempcorr = temp.values.copy() + tempcell = temp.values.copy() + good = ~np.isnan(tempcell) + tempcorr[good] = signal.lfilter(bb, aa, temp.values[good]) + tempcell = tempcell - tempcorr + sal = gsw.SP_from_C(ds.conductivity * 10, tempcell, ds.pressure) + + return sal + + +def flag_CTD_data( + ts0, + clean_stdev=3, + accuracy=None, +): + """ + Wrapper function to flag CTD data. + + Uses `flag_conductivity_in_depth_space` to flag conductivity as QC1 (good) + or QC4 (bad) in profile-depth space. Conductivity and salinity are then + flagged as QC4 wherever conductivity is flagged as QC4. + + Creates `conductivity_QC`, `salinity_QC`, and `temperature_QC` if they do + not already exist. + + Parameters + ---------- + ts0 : xarray.Dataset + Timeseries of mission data. + + clean_stdev : float, optional + Number of standard deviations from the cleaned mean for data to be + flagged as QC4. + + Returns + ------- + ts : xarray.Dataset + Timeseries of mission data with `conductivity_QC`, `salinity_QC`, + and `temperature_QC`. + """ + _log.info('Screening CTD data') + + ts = ts0.copy() + + ts["conductivity"] = ts["conductivity"].where(ts["conductivity"] >= 0.1) + + cond_qc = flag_conductivity_in_depth_space( + ts, + d_profile=50, + dz=5, + clean_stdev=clean_stdev, + accuracy=accuracy + ) + + if "conductivity_QC" not in ts.data_vars: + _log.info('Adding conductivity_QC variable to dataset') + + ts["conductivity_QC"] = xr.DataArray( + np.ones(ts["conductivity"].shape, dtype=int), + dims=ts["conductivity"].dims, + coords=ts["conductivity"].coords, + ) + + if "salinity_QC" not in ts.data_vars: + _log.info('Adding salinity_QC variable to dataset') + ts["salinity_QC"] = xr.DataArray( + np.ones(ts["salinity"].shape, dtype=int), + dims=ts["salinity"].dims, + coords=ts["salinity"].coords, + ) + + if "temperature_QC" not in ts.data_vars: + _log.info('Adding temperature_QC variable to dataset') + ts["temperature_QC"] = xr.DataArray( + np.ones(ts["temperature"].shape, dtype=int), + dims=ts["temperature"].dims, + coords=ts["temperature"].coords, + ) + + ts["conductivity_QC"] = xr.where(cond_qc == 4, 4, ts["conductivity_QC"]) + ts["salinity_QC"] = xr.where(ts["conductivity_QC"] == 4, 4, ts["salinity_QC"]) + + return ts + + +def adjust_CTD( + ts, + deploymentyaml, + alpha=None, + tau=None, + dTdC=None, + interpolate_filter=None, +): + """ + Pulls correction constants from `deploymentyaml`. If `alpha`, `tau`, or `dTdC` + differ from the values in the YAML file, the values provided as function arguments + are used and a warning is issued. + + Applies conductivity–temperature lag correction and thermal lag correction when + the corresponding constants are not `None` or 0. This produces the variables + `temperature_adjusted` and `salinity_adjusted`. The variables + `potential_density_adjusted` and `potential_temperature_adjusted` are derived + from the adjusted temperature and salinity. + + Parameters + ---------- + ts : xarray.Dataset + Time series of mission data. + + deploymentyaml : str or list + Path to a YAML file containing deployment information for the glider. + + If a list is provided, YAML files are read in order, and top-level keys + in later files overwrite those in earlier files. + + alpha : float, optional + Thermal lag correction parameter alpha. Default is None. + + tau : float, optional + Thermal lag correction parameter tau. Default is None. + + dTdC : float, optional + Time lag (seconds) between temperature and conductivity sensors. Default is None. + + interpolate_filter: callable or None, optional + Function applied to the dataset before finding the internal temperature. + Function interpolates over bad data and small data gaps + to prevent errors from affecting the neighbouring cells. Default is None. + Returns + ------- + ts : xarray.Dataset + Time series dataset with the additional variables: + `temperature_adjusted`, `salinity_adjusted`, + `potential_density_adjusted`, and `potential_temperature_adjusted`. + Metadata are updated to reflect applied corrections. + """ + logger = logging.getLogger(__name__) + _log.info('Adjusting CTD data') + + atr = deploymentyaml.get("glider_devices", {}).get("ctd", {}) + thermal = atr.get("Thermal_lag_constants_[alpha,tau]") + _log.info('CTD thermal lag constants from YAML: %s', thermal) + yaml_vals = { + "alpha": thermal[0] if thermal and len(thermal) > 0 else None, + "tau": thermal[1] if thermal and len(thermal) > 1 else None, + "dTdC": atr.get("dTdC"), + } + + kw_vals = { + "alpha": alpha, + "tau": tau, + "dTdC": dTdC, + } + + out = {} + for key in yaml_vals: + y = yaml_vals[key] + k = kw_vals[key] + + if k is not None: + if y is not None and y != k and logger is not None: + logger.warning( + "%s differs between YAML (%r) and kwargs (%r); using kwargs.", + key, y, k + ) + out[key] = k + else: + out[key] = y + + alpha = out.get("alpha", {}) + tau = out.get("tau", {}) + dTdC = out.get("dTdC", {}) + + if all(out.get(k) is None for k in ["alpha", "tau", "dTdC"]): + raise ValueError( + "Missing required CTD constants after checking kwargs and YAML:'" + "alpha, tau, dTdC" + ) + + temp_adj = ts.temperature.copy() + temp_adj.attrs = ts.temperature.attrs.copy() + temp_adj.attrs["comment"] = "temperature [degC]" + + if dTdC not in (None, 0): + _log.info('Interpolating temperature data back by %s seconds', dTdC) + dt = np.timedelta64(dTdC, "s") + temp_adj = temp_adj.interp(time=ts.time + dt) + + temp_adj.attrs["history"] = "temperature [degC] adjusted by CT lag" + temp_adj.attrs["time_lag"] = f"{dTdC} second CT lag corrected" + ts.attrs["dTdC"] = f"{dTdC} second CT lag corrected" + else: + temp_adj.attrs["comment"] = "equivalent to raw temperature" + ts.attrs["dTdC"] = "No CT lag applied" + + ts["temperature_adjusted"] = temp_adj + + if tau is not None: + dt = np.diff(ts.time.values).astype("timedelta64[s]").astype(int) + vals, counts = np.unique(dt, return_counts=True) + srate = vals[np.argmax(counts)] + + fs = 1 / float(srate) + fn = 0.5 * fs + + s = apply_thermal_lag( + ts, + fn, + alpha=alpha, + tau=tau, + interpolate_filter=interpolate_filter, + ) + sal_adj = xr.where(ts.salinity_QC == 1, s, ts.salinity) + sal_adj.attrs = ts.salinity.attrs.copy() + sal_adj.attrs["history"] = ( + f"adjusted salinity [psu] using thermal lag correction " + f"(alpha={alpha}, tau={tau})" + ) + + if dTdC not in (None, 0): + sal_adj.attrs["sources"] = ( + f"conductivity pressure temperature_adjusted " + f"(corrected for {dTdC} second CT lag)" + ) + + ts["salinity_adjusted"] = sal_adj + ts.attrs['correction_constants_alpha'] = alpha + ts.attrs['correction_constants_tau'] = tau + + else: + _log.info( + 'No thermal lag correction applied; calculating salinity_adjusted ' + 'using temperature_adjusted and raw conductivity' + ) + sal_adj = xr.DataArray( + gsw.conversions.SP_from_C( + 10 * ts["conductivity"], + ts["temperature_adjusted"], + + ts.pressure, + ).values, + dims=ts.salinity.dims, + coords=ts.salinity.coords, + ) + ts.attrs['correction_constants_alpha'] = "None" + ts.attrs['correction_constants_tau'] = "None" + + sal_adj.attrs = ts.salinity.attrs.copy() + + if dTdC is not None: + sal_adj.attrs["time_lag"] = "found using temperature_adjusted" + + ts["salinity_adjusted"] = sal_adj + + ts["salinity_adjusted"].attrs = sal_adj.attrs + + ts.attrs["quality_flags"] = ( + "1 = good data; 3 = bad data, potentially correctable; " + "4 = bad data; 8 = estimated data" + ) + + ts["conductivity"].attrs["comment"] = "raw conductivity" + ts["conductivity_QC"] = ts.conductivity_QC + + ts["temperature"].attrs["comment"] = "raw temperature [degC]" + ts["temperature_QC"] = ts.temperature_QC + + ts["temperature_adjusted_QC"] = ts["temperature_QC"] + + ts["salinity"].attrs["comment"] = "raw salinity [psu]" + ts["salinity_adjusted_QC"] = ts["salinity_QC"] + + ts["density"].attrs["comment"] = "raw density" + ts["density_QC"] = ts["salinity_QC"] + + ts["potential_density"].attrs["history"] = ( + "calculated using raw salinity and temperature" + ) + ts["potential_temperature"].attrs["history"] = ( + "calculated using raw salinity and temperature" + ) + + long = ts.longitude.fillna(ts.longitude.mean(skipna=True)) + lat = ts.latitude.fillna(ts.latitude.mean(skipna=True)) + + sa_adj = gsw.SA_from_SP(ts["salinity_adjusted"], ts["pressure"], long, lat) + ct_adj = gsw.CT_from_t(sa_adj, ts["temperature_adjusted"], ts["pressure"]) + + _log.info( + 'Calculating potential density and potential temperature using ' + 'adjusted salinity and temperature' + ) + ts["potential_density_adjusted"] = ( + ("time"), 1000 + gsw.density.sigma0(sa_adj, ct_adj).values + ) + + ts["potential_density"].attrs = ts.potential_density.attrs.copy() + + ts["potential_density_adjusted"].attrs["history"] = ( + "calculated using salinity_adjusted and temperature_adjusted" + ) + ts["potential_density_adjusted"].attrs["sources"] = ( + "salinity_adjusted temperature_adjusted pressure" + ) + + ts["potential_density_adjusted_QC"] = ts["salinity_adjusted_QC"] + ts["potential_density_adjusted_QC"].attrs = ts.salinity_adjusted_QC.attrs.copy() + + ts["potential_temperature_adjusted"] = ( + ("time"), + gsw.conversions.pt0_from_t( + ts.salinity_adjusted, + ts.temperature_adjusted, + ts.pressure, + ).values, + ) + + ts["potential_temperature_adjusted"].attrs = ( + ts.potential_temperature.attrs.copy() + ) + ts["potential_temperature_adjusted"].attrs["history"] = ( + "calculated using salinity_adjusted and temperature_adjusted" + ) + ts["potential_temperature_adjusted"].attrs["sources"] = ( + "salinity_adjusted temperature_adjusted pressure" + ) + + ts["potential_temperature_adjusted_QC"] = ts["salinity_adjusted_QC"] + ts["potential_temperature_adjusted_QC"].attrs = ( + ts.salinity_adjusted_QC.attrs.copy() + ) + + processing_date = date.today().strftime("%Y%m%d") + + vars_ = [ + "salinity_adjusted", + "temperature_adjusted", + "potential_density_adjusted", + "potential_temperature_adjusted", + ] + + for var in vars_: + ts[var].attrs["processing_date"] = processing_date + + QC_COMMENT = ( + "1 = good data; 3 = bad data, potentially correctable; " + "4 = bad data; 8 = estimated data" + ) + ncvars = deploymentyaml.get("netcdf_variables", {}) + + for k in ts.data_vars: + if k in ts and k in ncvars: + ts[k].attrs.update(ncvars[k]) + if k.endswith("_QC"): + ts[k].attrs["comment"] = QC_COMMENT + + return ts + + +def maskQC4(ds): + """ + Optional: + Masks QC4 samples in data variables (set to NaN) so gridding ignores + them. Only QC1 (good) data are gridded. + + Parameters + ---------- + ds: DataArray + Timeseries of a data + + Returns + ---------- + ds: DataArray + Timeseries of a data with QC4 data masked + + + """ + _log = logging.getLogger(__name__) + + ds = ds.copy() + _log.info('Masking QC4 data in dataset') + for k in list(ds.data_vars): + # skip QC variables themselves + if k.endswith("_QC"): + continue + + qc_name = f"{k}_QC" + if qc_name not in ds: + continue + + # mask data where QC == 4, preserving dims/coords + ds[k] = ds[k].where(ds[qc_name] != 4) + ds[qc_name] = ds[qc_name].where(ds[qc_name] != 4) + + return ds + + __all__ = [ 'get_distance_over_ground', 'get_glider_depth', @@ -852,6 +1440,11 @@ def _get_glider_name_slocum(current_directory): 'get_derived_eos_raw', 'fill_metadata', 'nmea2deg', - 'gappy_fill_vertical', 'oxygen_concentration_correction', -] + 'flag_conductivity_in_depth_space', + 'interpolate_over_salinity_NANs', + 'apply_thermal_lag', + 'flag_CTD_data', + 'adjust_CTD', + 'maskQC4' +] \ No newline at end of file diff --git a/tests/example-data/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid.nc b/tests/example-data/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid.nc new file mode 100644 index 00000000..0a444ca6 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190721T2352.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190721T2352.nc new file mode 100644 index 00000000..e3ba58e0 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190721T2352.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0125.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0125.nc new file mode 100644 index 00000000..d836b201 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0125.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0245.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0245.nc new file mode 100644 index 00000000..90e401bb Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0245.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0422.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0422.nc new file mode 100644 index 00000000..dcc546c2 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0422.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0545.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0545.nc new file mode 100644 index 00000000..5642a334 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0545.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0722.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0722.nc new file mode 100644 index 00000000..1bb82a9f Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0722.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0843.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0843.nc new file mode 100644 index 00000000..1eeeb291 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T0843.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1018.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1018.nc new file mode 100644 index 00000000..08ede0c2 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1018.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1134.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1134.nc new file mode 100644 index 00000000..8b6719a4 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1134.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1310.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1310.nc new file mode 100644 index 00000000..24c01cc9 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1310.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1445.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1445.nc new file mode 100644 index 00000000..61aa6118 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1445.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1550.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1550.nc new file mode 100644 index 00000000..b41feaae Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1550.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1652.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1652.nc new file mode 100644 index 00000000..d05e2321 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1652.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1830.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1830.nc new file mode 100644 index 00000000..aa80e8cf Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1830.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1946.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1946.nc new file mode 100644 index 00000000..3793e124 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T1946.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2120.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2120.nc new file mode 100644 index 00000000..761f637a Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2120.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2235.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2235.nc new file mode 100644 index 00000000..ae3ea9bd Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190722T2235.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0010.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0010.nc new file mode 100644 index 00000000..6b143373 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0010.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0125.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0125.nc new file mode 100644 index 00000000..3e3a890a Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0125.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0300.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0300.nc new file mode 100644 index 00000000..dd745c21 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0300.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0408.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0408.nc new file mode 100644 index 00000000..e0b82984 Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0408.nc differ diff --git a/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0546.nc b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0546.nc new file mode 100644 index 00000000..44569e3c Binary files /dev/null and b/tests/example-data/example-seaexplorer/L0-profiles/dfo-eva035035-20190723T0546.nc differ diff --git a/tests/example-data/example-seaexplorer/adjusted.yml b/tests/example-data/example-seaexplorer/adjusted.yml new file mode 100644 index 00000000..e16d72a2 --- /dev/null +++ b/tests/example-data/example-seaexplorer/adjusted.yml @@ -0,0 +1,10 @@ +ls tests/expected/example-seaexplorer/L0-timeseries/metadata: + quality_flags: '1 = good data; 3 = bad data, potentially correctable; 4 = bad data; 8 = estimated data' + +glider_devices: + ctd: + Thermal_lag_constants_[alpha,tau] : [0.34,4.6] + dTdC : 0 + comment: Constants were found using dfo-bb046-20220707. A detailed report of how the constants are defined can be found on https://cproof.uvic.ca. There is no lag between temperature and conductivity. + + \ No newline at end of file diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc new file mode 100644 index 00000000..465a4de2 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0636.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc new file mode 100644 index 00000000..315bbf3e Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0646.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc new file mode 100644 index 00000000..2bb1c772 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0657.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc new file mode 100644 index 00000000..d87054a0 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0708.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc new file mode 100644 index 00000000..c5b96013 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0718.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc new file mode 100644 index 00000000..77dcd483 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0729.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc new file mode 100644 index 00000000..a22f5728 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0739.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc new file mode 100644 index 00000000..cf02a37d Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0749.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc new file mode 100644 index 00000000..06e06647 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0800.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc new file mode 100644 index 00000000..f8b3243e Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0831.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc new file mode 100644 index 00000000..32f7da0f Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0842.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc new file mode 100644 index 00000000..427d884c Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0853.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc new file mode 100644 index 00000000..e087455a Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0902.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc new file mode 100644 index 00000000..730d5518 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0913.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc new file mode 100644 index 00000000..4d0e1135 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0923.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc new file mode 100644 index 00000000..532407f1 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0933.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc new file mode 100644 index 00000000..59709281 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0944.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc new file mode 100644 index 00000000..38b1de89 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T0954.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc new file mode 100644 index 00000000..f7ee12b1 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1004.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc new file mode 100644 index 00000000..d113d47d Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1033.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc new file mode 100644 index 00000000..35b86fc5 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1044.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc new file mode 100644 index 00000000..93aaaa39 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1055.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc new file mode 100644 index 00000000..bced3d6f Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1105.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc new file mode 100644 index 00000000..317ba794 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1115.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc new file mode 100644 index 00000000..cea0cc17 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1125.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc new file mode 100644 index 00000000..7073015d Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1209.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc new file mode 100644 index 00000000..719462d2 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1220.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc new file mode 100644 index 00000000..cb81015a Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1231.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc new file mode 100644 index 00000000..55e27008 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1241.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc new file mode 100644 index 00000000..66412ce9 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1252.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc new file mode 100644 index 00000000..99856be1 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1302.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc new file mode 100644 index 00000000..a1ad5aae Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1313.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc new file mode 100644 index 00000000..e1b7e84c Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1323.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc new file mode 100644 index 00000000..d6035fcc Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1335.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc new file mode 100644 index 00000000..da609c98 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1345.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc new file mode 100644 index 00000000..715fa5bf Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1415.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc new file mode 100644 index 00000000..d5939d0d Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1425.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc new file mode 100644 index 00000000..334ed54c Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1435.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc new file mode 100644 index 00000000..f44fa357 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1445.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc new file mode 100644 index 00000000..627f3087 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1455.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc new file mode 100644 index 00000000..9b247cbc Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1506.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc new file mode 100644 index 00000000..2806fe35 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1516.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc new file mode 100644 index 00000000..a0576007 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1526.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc new file mode 100644 index 00000000..be7f8efa Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1536.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc new file mode 100644 index 00000000..3d7f00b1 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1546.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc new file mode 100644 index 00000000..4545b01e Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1607.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc new file mode 100644 index 00000000..8a23eb90 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1618.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc new file mode 100644 index 00000000..b6dc1b62 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1627.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc new file mode 100644 index 00000000..01560fba Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1637.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc new file mode 100644 index 00000000..a2649862 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1646.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc new file mode 100644 index 00000000..f734ecc3 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1656.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc new file mode 100644 index 00000000..fff3f791 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1705.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc new file mode 100644 index 00000000..92ea8010 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1714.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc new file mode 100644 index 00000000..3bb5e0e5 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1723.nc differ diff --git a/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc new file mode 100644 index 00000000..52c026b9 Binary files /dev/null and b/tests/example-data/example-slocum/L0-profiles/dfo-rosie713-20190620T1732.nc differ diff --git a/tests/example-data/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc b/tests/example-data/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc new file mode 100644 index 00000000..467031fc Binary files /dev/null and b/tests/example-data/example-slocum/L0-timeseries/dfo-rosie713-20190615.nc differ diff --git a/tests/example-data/example-slocum/adjusted.yml b/tests/example-data/example-slocum/adjusted.yml new file mode 100644 index 00000000..c5f3b90b --- /dev/null +++ b/tests/example-data/example-slocum/adjusted.yml @@ -0,0 +1,14 @@ +metadata: + quality_flags: '1 = good data; 3 = bad data, potentially correctable; 4 = bad data; 8 = estimated data' + +glider_devices: + ctd: + Thermal_lag_constants_[alpha,tau]: [&alpha 0.2, &tau 2] + dTdC: &dTdC 0 + comment: 'Constants for ctd_9507 are found using dfo-rosie713-20230810. A detailed report of how the constants are defined can be found on https://cproof.uvic.ca. There is no lag between temperature and conductivity.' + + oxygen: + calibration_constants[c0, c1, c2, c3, c4, c5, c6]: [2.768177E-03, 1.173389E-04, 2.355152E-06, 2.292242E+02, -4.113573E-01, -5.832774E+01, 4.551447E+00] + Comment: 'Calibration constants from manufacturer' + + \ No newline at end of file diff --git a/tests/expected/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid_adjusted.nc b/tests/expected/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid_adjusted.nc new file mode 100644 index 00000000..55e3c43e Binary files /dev/null and b/tests/expected/example-seaexplorer/L0-gridfiles/dfo-eva035-20190718_grid_adjusted.nc differ diff --git a/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718_adjusted.nc b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718_adjusted.nc new file mode 100644 index 00000000..bd70b1ed Binary files /dev/null and b/tests/expected/example-seaexplorer/L0-timeseries/dfo-eva035-20190718_adjusted.nc differ diff --git a/tests/expected/example-slocum/L0-gridfiles/dfo-rosie713-20190615_grid_adjusted.nc b/tests/expected/example-slocum/L0-gridfiles/dfo-rosie713-20190615_grid_adjusted.nc new file mode 100644 index 00000000..d5cd2216 Binary files /dev/null and b/tests/expected/example-slocum/L0-gridfiles/dfo-rosie713-20190615_grid_adjusted.nc differ diff --git a/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615_adjusted.nc b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615_adjusted.nc new file mode 100644 index 00000000..b1769066 Binary files /dev/null and b/tests/expected/example-slocum/L0-timeseries/dfo-rosie713-20190615_adjusted.nc differ diff --git a/tests/test_process_adjusted_seaexplorer.py b/tests/test_process_adjusted_seaexplorer.py new file mode 100644 index 00000000..64208b5d --- /dev/null +++ b/tests/test_process_adjusted_seaexplorer.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +from pyglider.process_adjusted import run_process_adjusted + +library_dir = Path(__file__).parent.parent.absolute() +example_dir = library_dir / 'tests/expected/example-seaexplorer' +expected_dir = library_dir / 'tests/expected/example-seaexplorer' +yaml_dir = library_dir / 'tests/example-data/example-seaexplorer' + +outname = run_process_adjusted( + example_dir, + deploy_name = 'dfo-eva035-20190718', + deployfile=yaml_dir / 'deploymentRealtime.yml', + adjustedyaml=yaml_dir / 'adjusted.yml', +) +output = xr.open_dataset(outname) + +test_data = xr.open_dataset( + expected_dir / 'L0-timeseries' / 'dfo-eva035-20190718_adjusted.nc' +) + +variables = list(output.variables) + + +def test_variables_process_adjusted(): + test_variables = list(test_data.variables) + variables.sort() + test_variables.sort() + assert variables == test_variables + + +@pytest.mark.parametrize('var', variables) +def test_process_adjusted_timeseries(var): + assert output[var].attrs == test_data[var].attrs + if var not in ['time']: + np.testing.assert_allclose( + output[var].values, + test_data[var].values, + rtol=1e-5, + equal_nan=True, + ) + else: + dt0 = output[var].values - np.datetime64('2000-01-01') + dt1 = test_data[var].values - np.datetime64('2000-01-01') + assert np.allclose( + np.array(dt0, dtype='float64'), + np.array(dt1, dtype='float64'), + ) + + +def test_process_adjusted_metadata(): + output.attrs.pop('date_created', None) + output.attrs.pop('date_issued', None) + test_data.attrs.pop('date_created', None) + test_data.attrs.pop('date_issued', None) + assert output.attrs == test_data.attrs diff --git a/tests/test_process_adjusted_slocum.py b/tests/test_process_adjusted_slocum.py new file mode 100644 index 00000000..6104dd5d --- /dev/null +++ b/tests/test_process_adjusted_slocum.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +from pyglider.process_adjusted import run_process_adjusted + +library_dir = Path(__file__).parent.parent.absolute() +example_dir = library_dir / 'tests/expected/example-slocum' +expected_dir = library_dir / 'tests/expected/example-slocum' +yaml_dir = library_dir / 'tests/example-data/example-slocum' + +outname = run_process_adjusted( + example_dir, + deploy_name='dfo-rosie713-20190615', + deployfile=yaml_dir / 'deploymentRealtime.yml', + adjustedyaml=yaml_dir / 'adjusted.yml', +) +output = xr.open_dataset(outname) + +test_data = xr.open_dataset( + expected_dir / 'L0-timeseries' / 'dfo-rosie713-20190615_adjusted.nc' +) + +variables = list(output.variables) + + +def test_variables_process_adjusted(): + test_variables = list(test_data.variables) + variables.sort() + test_variables.sort() + assert variables == test_variables + + +@pytest.mark.parametrize('var', variables) +def test_process_adjusted_timeseries(var): + assert output[var].attrs == test_data[var].attrs + if var not in ['time']: + np.testing.assert_allclose( + output[var].values, + test_data[var].values, + rtol=1e-5, + equal_nan=True, + ) + else: + dt0 = output[var].values - np.datetime64('2000-01-01') + dt1 = test_data[var].values - np.datetime64('2000-01-01') + assert np.allclose( + np.array(dt0, dtype='float64'), + np.array(dt1, dtype='float64'), + ) + + +def test_process_adjusted_metadata(): + output.attrs.pop('date_created', None) + output.attrs.pop('date_issued', None) + test_data.attrs.pop('date_created', None) + test_data.attrs.pop('date_issued', None) + assert output.attrs == test_data.attrs