****CDL representation of QC'ed CTD ocean profile data at RAN METOC Data Centre*****

The following is the CDL for a netCDF file describing
a quality controlled vertical profile of pressure, temperature and salinity collected 
by a CTD instrument. The CDL follows the netCDF CF coventions version 1.6 
at ref (2). The CTD data is classed
as featureType 'profile' as per Chapter 9 'Discrete sampling geometries' in
ref (2).

The quality control (QC) flagging scheme follows the 2 level
QC flag scheme proposed by IODE in the ref (1) below. In this QC scheme the Level 1
flag is the primary QC flag. Level 2 flags are secondary indicating results
of specified QC tests. Level 2 flags use a numerical code i.e 0 (passed),1 (failed) or 2
(unknown) to indicate test status of a specified test. 


The netCDF file contains a set of global attributes sufficient to generate a metadata
record in ISO-19115/19139-MCP format. For this purpose attribute names recommended by the 
"NetCDF Attribute Convention for Dataset Discovery" (NACDD) from UniData 
are used, see ref. (3).
There are also global attributes specific to the CTD instrument used and technical
data such as Data Parameters,Cruise Number, Serial No, Scan rate, cast direction etc.
The Data Parameter names used are from the CF standard names list, see ref (4).
All global attributes are specified below.


References

(1) Konovalov et. al (March 2012), Proposal to adopt a quality flag scheme standard
for oceanographic and marine meteorological data, Version 1.2.

(2) http://cf-pcmdi.llnl.gov/documents/cf-conventions

(3) http://www.unidata.ucar.edu/software/netcdf-java/formats/DataDiscoveryAttConvention.html

(4) http://cf-pcmdi.llnl.gov/documents/cf-standard-names/

dimensions:
	
	pressure = UNLIMITED ; // (9 currently)
	
variables:
	double time ;
		time:standard_name = "time" ;
		time:units = "days since 1950-01-01 00:00:00Z" ;
		time:axis = "T" ;
		time:valid_min = 0 ;
		time:valid_max = 999999 ;
		
	byte time_qc_flag;	
		time_qc_flag:long_name = "quality control flag for time (Level 1 flag)" ;
		time_qc_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		time_qc_flag:valid_min = 1 ;
		time_qc_flag:valid_max = 9 ;
		time_qc_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		time_qc_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
				
	double latitude ;
		latitude:standard_name = "latitude" ;
		latitude:units = "degrees_north" ;
		latitude:axis = "Y" ;
		latitude:valid_min = -90 ;
		latitude:valid_max = 90 ;
		
	double longitude ;
		longitude:standard_name = "longitude" ;
		longitude:units = "degrees_east" ;
		longitude:axis = "X" ;
		longitude:valid_min = -180 ;
		longitude:valid_max = 180 ;
		
	byte position_qc_flag;	
		position_qc_flag:long_name = "quality control flag for position (Level 1 flag)" ;
		position_qc_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		position_qc_flag:valid_min = 1 ;
		position_qc_flag:valid_max = 9 ;
		position_qc_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		position_qc_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
	
	
	double pressure(pressure) ;
		pressure:standard_name = "sea_water_pressure" ;
		pressure:units = "decibars" ;
		pressure:axis = "Z" ;
		pressure:valid_min = 0 ;
		pressure:valid_max = 12000 ;
		pressure:positive = "down" ;
		
	double temperature(pressure) ;
		temperature:_FillValue = -99.99 ;
		temperature:standard_name = "sea_water_temperature" ;
		temperature:units = "degrees_C" ;
		temperature:valid_min = -2 ;
		temperature:valid_max = 40 ;
		temperature:ancillary_variables = "temperature_whole_profile_flag temperature_qc_flag temperature_sd_test"  ;
		temperature:coordinates = "time latitude longitude pressure" ;
		
	byte temperature_whole_profile_flag ;
		temperature_whole_profile_flag:long_name = "qc flag for whole temperature profile (primary L1 flag)" ;
		temperature_whole_profile_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		temperature_whole_profile_flag:valid_min = 1 ;
		temperature_whole_profile_flag:valid_max = 9 ;
		temperature_whole_profile_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		temperature_whole_profile_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
		
	byte temperature_qc_flag(pressure) ;
		temperature_qc_flag:long_name = "quality control flag for temperature (primary Level 1 flag)" ;
		temperature_qc_flag:standard_name = "sea_water_temperature status_flag" ;
		temperature_qc_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		temperature_qc_flag:valid_min = 1 ;
		temperature_qc_flag:valid_max = 9 ;
		temperature_qc_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		temperature_qc_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
		temperature_qc_flag:coordinates = "time latitude longitude pressure" ;
		
	byte temperature_sd_test(pressure) ;
		temperature_sd_test:long_name = "qc flag for monthly temperature standard deviation test (secondary L2 flag)"
		temperature_sd_test:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		temperature_sd_test:valid_min = 0 ;
		temperature_sd_test:valid_max = 2 ;
		temperature_sd_test:flag_values = 0b, 1b, 2b ;
		temperature_sd_test:flag_meanings = "passed failed unknown" ;
		temperature_sd_test:coordinates = "time latitude longitude pressure" ;
	
				
	double salinity(pressure) ;
		salinity:_FillValue = -99.99 ;
		salinity:standard_name = "sea_water_practical_salinity" ;
		salinity:units = "psu" ;
		salinity:valid_min = 0 ;
		salinity:valid_max = 45 ;
		salinity:ancillary_variables = "salinity_whole_profile_flag salinity_qc_flag salinity_sd_test" 
		salinity:coordinates = "time latitude longitude pressure" ;
		
	byte salinity_whole_profile_flag ;
		salinity_whole_profile_flag:long_name = "qc flag for whole salinity profile (primary L1 flag)" ;
		salinity_whole_profile_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		salinity_whole_profile_flag:valid_min = 1 ;
		salinity_whole_profile_flag:valid_max = 9 ;
		salinity_whole_profile_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		salinity_whole_profile_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
		
	byte salinity_qc_flag(pressure) ;
		salinity_qc_flag:long_name = "quality control flag for salinity (primary Level 1 flag)" ;
		salinity_qc_flag:standard_name = "sea_water_practical_salinity status_flag" ;
		salinity_qc_flag:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		salinity_qc_flag:valid_min = 1 ;
		salinity_qc_flag:valid_max = 9 ;
		salinity_qc_flag:flag_values = 1b, 2b, 3b, 4b, 9b ;
		salinity_qc_flag:flag_meanings = "good not_evaluated_or_unknown suspect bad missing" ;
		salinity_qc_flag:coordinates = "time latitude longitude pressure" ;
		
	byte salinity_sd_test(pressure) ;
		salinity_sd_test:long_name = "qc flag for monthly salinity standard deviation test (secondary L2 flag)"
		salinity_sd_test:quality_control_convention = "Proposed IODE qc scheme March 2012" ;
		salinity_sd_test:valid_min = 0 ;
		salinity_sd_test:valid_max = 2 ;
		salinity_sd_test:flag_values = 0b, 1b, 2b ;
		salinity_sd_test:flag_meanings = "passed failed unknown" ;
		salinity_sd_test:coordinates = "time latitude longitude pressure" ;
			
	int profile ; //Unique integer to identify each profile
		profile:long_name = "profile identifier"
		profile:cf_role = "profile_id";
		profile:comment = "8 digit profile identifier in format nnnncccc where nnnn=numerical cruise id, cccc=consecutive number in cruise";


// global attributes:

//Attributes From CF coventions ref (2)

:Conventions = "CF-1.6" ;
:featureType = "profile" ;

// Attributes from NetCDF Attribute Convention for Dataset Discovery (NACDD - ref (3) )

:title = "RAN CTD data Ship:HMAS Leeuwin Cruise:HIXXX ProfileId:131414" {ProfileID is the 'profile' variables value an integer}
:summary = The summary (abstract) will be different depending if the data
is off the MVP CTD or the HS-CTD. Use text as approp. below

***For HS-CTD

"This file contains a vertical profile of pressure, temperature and salinity measured by a Conductivity, Temperature and Depth (CTD) probe. This CTD is fitted within an inductive type conductivity cell, a platinum thermometer and silicon pressure sensor. 

The CTD temperature sensor is calibrated on the ITS-90 temperature scale against a master CTD using a controlled temperature bath. The pressure sensor is calibrated using a Druck Pressure Calibrator. The conductivity sensor is calibrated in a temperature controlled bath against seawater samples of known conductivity.

The CTD is lowered and raised by a hand winch sampling at a rate of 1.83 Hz. Data files were recorded in the downcast and upcast direction but most data is from downcasts due to higher quality. Raw CTD pressure data is not always monotonic due to transient up/down motion of winch/ship. Raw pressure data is subsetted to give a monotonic (increasing) sequence and then linearly interpolated to 1 decibar pressure intervals. Data is then flagged with quality control flags after visual inspection and comparison to average climatology and historical CTD casts."

***For MVP200-CTD

"This file contains a vertical profile of pressure, temperature and salinity measured by a Conductivity, Temperature and Depth (CTD) probe. This CTD is fitted with a 4 electrode platinized conductivity cell, thermistor temperature sensor and a semiconductor strain gauge pressure sensor.

The CTD sensors are calibrated at the manufacturer Applied Microsystems Limited on a 12-18 month schedule. The CTD temperature sensor is calibrated against 'Hart' temperature standards. The pressure sensor is calibrated using 'Budenburg Deadweight' standards. The conductivity sensor is calibrated using 'Hart' temperature standards and seawater samples of known conductivity.

The CTD sensors are mounted to a fish-shape probe. The probe is controlled by an electric winch (MVP200 type). For downcasts the fish is allowed to free-fall (winch is in 'free-wheel' mode) under its own weight at about 2-3 ms-1 and then is winched back. Data is recorded in downcast and usually in upcast direction at a sampling rate of 25 Hz. The downcast data is of higher quality because sensors encounter undisturbed seawater that flows through the nose of the fish. On the upcast the fish is flipped around (tail first) and the sensors in the nose encounter disturbed flow from the fish tail.

This type of CTD is prone to a phenomenon called 'salinity spiking' caused by a mismatch between the response times of the temperature and conductivity sensor. Data undergoes a salinity de-spiking routine to correct for this. See the history metadata for further details on the salinity de-spiking process.

Raw CTD pressure data is not always monotonic due to transient up/down motion of winch/ship. Raw pressure data is subsetted to give a monotonic (increasing) sequence and then linearly interpolated to 1 decibar pressure intervals. Data is then flagged with quality control flags after visual inspection and comparison to average climatology and historical CTD casts."


:keywords = "Oceans | Ocean Temperature | Water Temperature,Oceans | Salinity/Density | Salinity"
:id = type 4 random uuid used to id metadata record e.g "c3ab1cd1-cd8e-44f0-994e-93ecc5da9d84" use python uuid function
:naming_authority = "metoc.gov.au"
:keywords_vocabulary = "NASA/Global Change Master Directory (GCMD) Earth Science Keywords. Version  5.3.8 (2006)"

:cdm_data_type = "Profile" ;

:history = History record details Processing and QC details. Use diffrent text for 
HS-CTD or MVP-200 CTD

***HS-CTD

"Initial Processing

Incoming data undergoes gross range checks and removal of any corrupted or unrealistic data. Some CTD data is subject to a pressure offset error generally less than 1 decibar. A pressure offset error occurs when the pressure sensor does not read 0 dbar in air. In cases where the pressure offset error was observed before the CTD cast the pressure readings have been corrected for the offset which is stored in the global netCDF attribute named 'CTD_pressure_offset'. In some cases there is a separate recording for the downcast and upcast. The downcast data were selected as priority for better quality data. In rarer cases only an upcast was available so it is used by default.

Salinity on the PSS-78 scale is computed from conductivity, temperature and pressure using the UNESCO (1983) algorithm with temperature corrected from the ITS-90 scale to the ITS-68 scale used by the algorithm.

The raw CTD data contains a non-monotonic sequence of pressures due to up/down motion of ship/winch. A sequence of unique monotonic pressures up to the maximum value is extracted and then linearly interpolated to 1 decibar pressure levels with the corresponding interpolated temperature and salinity. Finally the interpolated files are converted to a set of netCDF files ready for QC.

Quality Control

Quality control (QC) involves viewing location of the data on a map, visual inspection of each profile of temperature and salinity and comparison with nearest neighbours and climatology. CTD profiles are checked for any density inversions i.e. density decreasing with depth by computing sigma-t density. A density inversion may indicate unrealistic salinity and/or temperature values. QC flags are applied to indicate whether data is good, suspect, bad or not tested. QC flags may apply at the 'whole profile' level or individual pressure levels.
 
CTD cast positions and times undergo a land and ship speed check. Casts with position on land or unrealistic speed between casts are flagged as failed position or time. Temperature and salinity profiles are then visually compared against a 3 standard deviation envelope from the CSIRO-CARS (2009) atlas. Profiles or segments of profiles with data outside the envelope are flagged as doubtful or failed. Profiles are also checked for consistency or doubtful features by comparing with previous/next casts (buddies) and also by comparison with historical CTD casts taken in the same area and season."

****MVP200-CTD

"Initial Processing

Incoming data undergoes gross range checks and removal of any corrupted or unrealistic data. Some CTD data is subject to a pressure offset error generally less than 1 decibar. A pressure offset error occurs when the pressure sensor does not read 0 dbar in air. In cases where the pressure offset error was observed before the CTD cast the pressure readings have been corrected for the offset which is stored in the global netCDF attribute named 'CTD_pressure_offset'.

Salinity on the PSS-78 scale is computed from conductivity, temperature and pressure using the UNESCO (1983) algorithm with temperature corrected from the ITS-90 scale to the ITS-68 scale used by the algorithm.

For this CTD model the response time (time constant) of the temperature sensor (100 ms)is slower than the conductivity cell (25ms). This leads to a phenomenon called 'salinity spiking' in the salinity data. Spiking is caused by the fact that calculated salinity is a function of conductivity, temperature and pressure [S=f(C,T,z)]. The temperature response lags the conductivity response resulting an error in the calculation of salinity. This is evident as noticeable spikes in the salinity profiles especially at depths where temperature is rapidly changing e.g in the ocean thermocline. The spiking was reduced reduced by applying a 2.5 sample time shift (0.1 s) to the temperature values and using the shifted temperature value in the re-computation of the salinity. An array of raw pressure, temperature and de-spiked salinity is then passed to the next processing stage.

The raw CTD data contains a non-monotonic sequence of pressures due to up/down motion of ship/winch. A sequence of unique monotonic pressures up to the maximum value is extracted and then linearly interpolated to 1 decibar pressure levels with the corresponding interpolated temperature and salinity. Finally the interpolated files are converted to a set of netCDF files ready for QC.

Quality Control

Quality control (QC) involves viewing location of the data on a map, visual inspection of each profile of temperature and salinity and comparison with nearest neighbours and climatology. CTD profiles are checked for any density inversions i.e. density decreasing with depth by computing sigma-t density. A density inversion may indicate unrealistic salinity and/or temperature values. QC flags are applied to indicate whether data is good, suspect, bad or not tested. QC flags may apply at the 'whole profile' level or individual pressure levels.
 
CTD cast positions and times undergo a land and ship speed check. Casts with position on land or unrealistic speed between casts are flagged as failed position or time. Temperature and salinity profiles are then visually compared against a 3 standard deviation envelope from the CSIRO-CARS (2009) atlas. Profiles or segments of profiles with data outside the envelope are flagged as doubtful or failed. Profiles are also checked for consistency or doubtful features by comparing with previous/next casts (buddies) and also by comparison with historical CTD casts taken in the same area and season."


:date_created = "1991-06-19T20:02:01" Local time (ISO8601) when netCDF file created.
:date_modified = "2012-05-30T02:59:54Z" Local time file modified through QC, update each time QC is run
:qc_status = 0 (0 for no QC, 1 for QC commenced, 2 for QC complete)

:creator_name = "Data Management Officer"
:creator_url = http://www.metoc.gov.au/
:creator_email = dm@metoc.gov.au
:institution = "Royal Australian Navy Hydrography and METOC Branch"
:acknowledgment = "Royal Australian Navy Hydrography and Metoc Branch,CSIRO Marine and Atmospheric Research"

:geospatial_lat_min = -35.12345 ; (as per cast location)
:geospatial_lat_max = -35.12345 ;
:geospatial_lon_min = 151.82345 ;
:geospatial_lon_max = 151.82345 ;
:geospatial_vertical_min = 1.00 ; (as per 1st pressure in final .dat file)
:geospatial_vertical_max = 475.00 ; (as per last pressure)
:geospatial_vertical_units = "decibars"
:time_coverage_start = "1991-06-19T20:02:01Z" ; (cast date/time)
:time_coverage_end = "1991-06-19T20:02:01Z" ;


:standard_name_vocabulary = "CF standard names list at http://cf-pcmdi.llnl.gov/documents/cf-standard-names/"
:license = "Creative Commons - Attribution (CC-BY) at http://creativecommons.org/licenses/by/3.0/au/"

//Extra Attributes - particular to data centre, processing  and CTD data type

:coordinateReferenceSystem = "EPSG:4326" ;
:shipname: = HMAS Leeuwin
:cruise_id = "HIxxx"

:water_depth = "500" for HS-CTD not available, leave this attribute out
:ship_speed = "1.8" for HS-CTD not avaialble, leave this attribute out

:source_filename = svp_0008.raw
:data_parameters = "sea_water_temperature,sea_water_salinity"

:CTD_instrument_manufacturer = "Falmouth Scientific" (HS-CTD) or "Applied Microsystems Limited" (MVPCTD)
:CTD_instrument_model = "2-inch Micro CTD" (HS-CTD) or "Micro CTD" (MVPCTD) 
:CTD_serial_no = "XXXX" if not given leave this attribute out

:CTD_pressure_offset = Not recorded yet, leave as "0.0" for now

:CTD_MinRawPressure = "1.20" (as per source .dat file )
:CTD_MaxRawPressure = "12.20"
:CTD_MinPressureforInterp = "3.30"
:CTD_MaxPressureforInterp = "12.20"

:CTD_cast_direction = "Down" or "Up"
:CTD_scan_rate = "1.83 Hz" (for HS-CTD)
:CTD_scan_rate = "25 Hz" (for MVP-CTD)

		data:

 time = 22509.0081018517 ;
 time_qc_flag = 1;

 latitude = -10.5688 ;

 longitude = 142.11713 ;
 
 position_qc_flag = 1;

 pressure = 4, 5, 6, 7, 8, 9, 10, 11, 12 ;

 temperature = 24.737, 24.737, 24.739, 24.738, 24.737, 24.736, 24.7375, 
    24.737, 24.735 ;
	
 temperature_whole_profile_flag = 1;
	
 temperature_qc_flag = 1,1,1,1,1,1,1,1,1;
 
 temperature_sd_test = 0,0,0,0,0,0,0,0,0;

 salinity = 35.2028, 35.2032, 35.2013, 35.2018, 35.2022, 35.2019, 35.2019, 
    35.2024, 35.204 ;
	
 salinity_whole_profile_flag = 1;
	
 salinity_qc_flag = 1,1,1,1,1,1,1,1,1;
 
 salinity_sd_test = 0,0,0,0,0,0,0,0,0;
 
 profile = 1234567 ;
 