QC testing, bug fixing and improvements in superfloat_oxygen

.gitignore

@@ -13,3 +13,6 @@ makefile
 *.grib
 /.pytest_cache/
 .hypothesis/*
+bitsea-copilot-coder-bundle.zip
+docs
+*pyc

src/bitsea/Float/CORIOLIS_checks.py

@@ -30,16 +30,26 @@ def check_lenght_timeseries(df):
     days = duration.days
     return(days>365)
 
-# check over nans
+# check over nans 
 def nans_check(df, namecol):
-    """ input  : dataframe & nameof column selected for the check 
+    """
         output : Boolean
         method : fill nans inside the dataset to remove the starting part of time series: create list_index
                  nan are counted in the core of time series df_ix[list_index]
-        """
+    """
     df[namecol]   = pd.to_numeric(df[namecol])
-    list_index    = list((df.interpolate()).dropna().index)
-    return(df.iloc[list_index].VAR.isna().sum()  < df.shape[0]/2)
+    list_index    = list((df.VAR.interpolate()).dropna().index)
+    sliced_time_df = df.iloc[list_index]  
+    if sliced_time_df.empty:
+        return False
+    if len(sliced_time_df) <5:
+        return False
+    sliced_time_df["time"] = pd.to_datetime(sliced_time_df["time"])
+    duration_days = (sliced_time_df["time"].iloc[-1] - sliced_time_df["time"].iloc[0]).days
+
+    if duration_days > 365 : return True
+    else: return False
+
 
 def lenght_timeseries(df, namecol):
     df1=df.copy()

src/bitsea/Float/Float_opt/superfloat_cdom.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of cdom.
@@ -47,8 +55,7 @@ def dump_cdom_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_chla.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of chla.
@@ -47,8 +55,7 @@ def dump_chla_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_irr380.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of irr_380.
@@ -47,8 +55,7 @@ def dump_irr_380_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_irr412.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of irr_412.
@@ -47,8 +55,7 @@ def dump_irr_412_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_irr490.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of irr_490.
@@ -47,8 +55,7 @@ def dump_irr_490_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_nitrate.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of chla.
@@ -51,8 +59,7 @@ def dump_nitrate_file(outfile, p_pos, p, Pres, Value, Qc, metadata,mode='w'):
         os.system(command)
     ncOUT = NC.netcdf_file(outfile + ".tmp",mode)
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_oxygen.py

@@ -39,6 +39,14 @@ def argument():
 import numpy as np
 import gsw
 
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def remove_bad_sensors(Profilelist,var):
     '''
 
@@ -70,8 +78,7 @@ def convert_oxygen(p,doxypres,doxyprofile):
     from micromol/Kg to  mmol/m3
     '''
     if doxypres.size == 0: return doxyprofile
-    Pres, temp, Qc = p.read("TEMP",read_adjusted=False)
-    Pres, sali, Qc = p.read("PSAL",read_adjusted=False)
+    PresT, temp, QcT, Pres, sali, QcS = read_temp_psal(p)
     SA = gsw.SA_from_SP(sali, Pres, p.lon, p.lat)
     density = gsw.rho(SA, gsw.CT_from_t(SA, temp, Pres), Pres)
     density_on_zdoxy = np.interp(doxypres,Pres,density)
@@ -85,8 +92,7 @@ def dump_oxygen_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_par.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of par.
@@ -47,8 +55,7 @@ def dump_par_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/Float_opt/superfloat_ph.py

@@ -1,4 +1,12 @@
 import argparse
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def argument():
     parser = argparse.ArgumentParser(description = '''
     Creates superfloat files of ph.
@@ -47,8 +55,7 @@ def dump_ph_file(outfile, p, Pres, Value, Qc, metatata, mode='w'):
     ncOUT = NC.netcdf_file(outfile + ".tmp" ,mode)
 
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL', read_adjusted=False)        
+        PresT, Temp, QcT, Pres, Sali, QcS = read_temp_psal(p)        
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/MedBGCins.py

@@ -7,7 +7,7 @@
 from bitsea.commons.grid import RegularGrid
 from bitsea.mhelpers.linear_shift import linear_shift
 
-default_clim_n3n_nc = "/gss/gss_work/DRES_OGS_BiGe/Observations/CLIMATOLOGY/Nutrients/MedBGC-ins/Climatology_N3n_600_800m.nc"
+default_clim_n3n_nc = "/g100_work/OGS_test2528/Observations/CLIMATOLOGY/Nutrients/MedBGC-ins/Climatology_N3n_600_800m.nc"
 
 clim_file = Path(os.getenv("CLIM_MED_BGC_INS_FILE", default_clim_n3n_nc))
 if not clim_file.exists():

src/bitsea/Float/Superfloat_oxy/superfloat_generator.py

@@ -4,6 +4,13 @@
 import os
 
 
+def read_temp_adjusted(profile_obj):
+    PresT, Temp, QcT = profile_obj.read('TEMP', read_adjusted=True)
+    if PresT is None or Temp is None or len(PresT) == 0 or len(Temp) == 0:
+        PresT, Temp, QcT = profile_obj.read('TEMP', read_adjusted=False)
+    return PresT, Temp, QcT
+
+
 def general_quenching(profile_obj, Pres, Profile, Qc):
     '''
     Manages the quenching for Coriolis profiles
@@ -14,7 +21,7 @@ def general_quenching(profile_obj, Pres, Profile, Qc):
     In case of cutted profile (Z_top > 10.) the quench value is replied up to the sea surface
     on a set of extrapolation points, every 5 meters.
     '''
-    PresT, Temp, _ = profile_obj.read('TEMP', read_adjusted=False)
+    PresT, Temp, _ = read_temp_adjusted(profile_obj)
     mld = calculated_depths.mld(Temp, PresT, zref=0, deltaTemp=0.1)
     if mld < 80:
         if Pres[0] <= mld+5:
@@ -57,7 +64,7 @@ def quenching(profile_obj, PresChl, Chl, chl_lov_zero):
     '''
     Quenched_Chl = Chl.copy()
     if profile_obj.time.month in range(1,13):
-        PresT, Temp, _ = profile_obj.read('TEMP', read_adjusted=False)
+        PresT, Temp, _ = read_temp_adjusted(profile_obj)
         mld = calculated_depths.mld(Temp, PresT, zref=0, deltaTemp=0.1)
         if mld > 200:mld=20
 

src/bitsea/Float/Superfloat_oxy/superfloat_oxy_function.py

@@ -8,6 +8,14 @@
 import numpy as np
 import datetime
 
+def read_temp_psal(p):
+    PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+    Pres, Sali, QcS = p.read('PSAL', read_adjusted=True)
+    if (Pres is None or PresT is None or Temp is None or Sali is None or len(Pres) < 5 or len(PresT) < 5):
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        Pres, Sali, QcS = p.read('PSAL', read_adjusted=False)
+    return PresT, Temp, QcT, Pres, Sali, QcS
+
 def remove_bad_sensors(Profilelist,var):
     '''
 
@@ -44,8 +52,7 @@ def dump_oxygen_file(outfile, p, Pres, Value, Qc, metadata, mode='a'):
     if mode=='w': # if not existing file, we'll put header, TEMP, PSAL
         setattr(ncOUT, 'origin'     , 'coriolis')
         setattr(ncOUT, 'file_origin', metadata.filename)
-        PresT, Temp, QcT = p.read('TEMP')  #, read_adjusted=False)
-        PresT, Sali, QcS = p.read('PSAL')  # , read_adjusted=False)
+        PresT, Temp, QcT, PresS, Sali, QcS = read_temp_psal(p)
         ncOUT.createDimension("DATETIME",14)
         ncOUT.createDimension("NPROF", 1)
         ncOUT.createDimension('nTEMP', len(PresT))

src/bitsea/Float/TREND_ANALYSIS.py

@@ -48,7 +48,7 @@ def compute_trend(df):
 
 
 
-def trend_conditions(WMO,  days, Bool, DEPTH, min_d, max_d , TI_3, tmp):
+def trend_conditions(WMO,cycle,   days, Bool, DEPTH, min_d, max_d , TI_3, tmp):
     """  compute the trend if time-nans and vals for the selected day
          are satisfied. 
          CASE_* is a Boolean 
@@ -60,11 +60,11 @@ def trend_conditions(WMO,  days, Bool, DEPTH, min_d, max_d , TI_3, tmp):
        y = np.array(tmp.VAR)
        x = np.array(tmp.index)
        LIST_APP_TRENDS = Trend_RANSAC_ThSEN(x , y)
-       lst   = [WMO ,DEPTH,  days , min_d ,  max_d,  LIST_APP_TRENDS[0][1], LIST_APP_TRENDS[1][1] ,np.nan,np.nan,np.nan]
+       lst   = [WMO,cycle ,DEPTH, days , min_d ,  max_d,  LIST_APP_TRENDS[0][1], LIST_APP_TRENDS[1][1] ,np.nan,np.nan,np.nan]
     elif CASE_2:
-       lst   = [WMO ,DEPTH,  days , np.nan , np.nan , np.nan ,               np.nan, np.nan,np.nan,np.nan] 
+       lst   = [WMO,cycle ,DEPTH, days, np.nan, np.nan, np.nan, np.nan, np.nan,np.nan,np.nan] 
     else: 
-       lst   = [WMO ,DEPTH,  days , min_d ,  max_d,   np.nan ,               np.nan, np.nan,np.nan,np.nan]
+       lst   = [WMO,cycle ,DEPTH, days, min_d, max_d, np.nan, np.nan, np.nan,np.nan,np.nan]
     return (lst)
 
 def drift_coding(WMO,Bool ,df0,df1):

src/bitsea/Float/oxygen_saturation.py

@@ -14,13 +14,16 @@ def oxy_sat(p):
     * O2o * a concentration of oxygen in mmol/m3
             nan if there are no TEMP and PSAL values with pressure less than 5m.
     '''
-
     if p.has_adjusted:
-        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)#p._my_float.adjusted('TEMP'))
-        PresS, Sali, QcS = p.read('PSAL', read_adjusted=False)#p._my_float.adjusted('PSAL'))
+        try:
+            PresT, Temp, QcT = p.read('TEMP', read_adjusted=True)
+            PresS, Sali, QcS = p.read('PSAL', read_adjusted=True)
+        except TypeError: # dovrebbe entrare se uso superfloat wrapper
+            PresT, Temp, QcT = p.read('TEMP')
+            PresS, Sali, QcS = p.read('PSAL')
     else:
-        PresT, Temp, QcT = p.read('TEMP')
-        PresS, Sali, QcS = p.read('PSAL')
+        PresT, Temp, QcT = p.read('TEMP', read_adjusted=False)
+        PresS, Sali, QcS = p.read('PSAL', read_adjusted=False)
 
     ii = PresT<=depth_threshold
     if ii.sum() == 0: