Add WOUDC Data Registry's Parsers

Parser from WOUDC data registry added into woudc-extcsv. Reader and Writer classes modified to used the ExtendedCSV class. Includes warning and error reporting for WOUDC's extcsv format.

Author: victoriarspada

LICENSE.md

@@ -19,7 +19,7 @@ are asked to read the 3rd Party Licenses referenced with those assets.
 The MIT License (MIT)
 =====================
 
-Copyright (c) 2015 Government of Canada
+Copyright (c) 2022 Government of Canada
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.md

@@ -5,19 +5,21 @@
 
 # WOUDC Extended CSV library
 
-Python package providing read/write support of the
+Python package providing read/write support for files of the
 [WOUDC](https://woudc.org) Extended CSV format.
 
 ## Installation
 
 ### Requirements
 
-woudc-extcsv requires Python 3 and [pywoudc](https://github.com/woudc/pywoudc).
+woudc-extcsv requires Python 3 or above. See `requirements.txt`.
 
-### Dependencies
+### Development
 
-See `requirements.txt`
-- [pywoudc](https://github.com/woudc/pywoudc)
+```bash
+# install dev requirements
+pip install -r requirements-dev.txt
+```
 
 ### Installing the Package
 
@@ -69,10 +71,7 @@ ecsv.add_table('TIMESTAMP')
 # Add fields
 ecsv.add_field('TIMESTAMP', 'UTCOffset,Date,Time')
 # Add data
-ecsv.add_data('TIMESTAMP', '+08:38:47', field='UTCOffset')
-# Add more data
-ecsv.add_data('TIMESTAMP', '1991-01-01', field='Date')
-ecsv.add_data('TIMESTAMP', '06:38:47', field='Time')
+ecsv.add_data('TIMESTAMP', '+08:38:47,1991-01-01,06:38:47')
 
 # Add new table, fields, and data at the same time
 ecsv.add_data('GLOBAL_SUMMARY',
@@ -85,6 +84,7 @@ ecsv.add_data('GLOBAL',
               '290.5,0.000E+00')
 ecsv.add_data('GLOBAL',
               '291.0,0.000E+00')
+ecsv.add_table_comment('GLOBAL', 'This is a table level comment', index=1)
 # Add table for new groupings
 ecsv.add_data('TIMESTAMP',
               '+08:38:46,1991-01-01,07:38:46',
@@ -95,7 +95,7 @@ ecsv.add_data('GLOBAL_SUMMARY',
               '07:38:46,2.376E-02,3.984E-01,82.92,6.75,122.69,100000,999',
               field='Time,IntACGIH,IntCIE,ZenAngle,MuValue,AzimAngle,Flag,TempC',
               index=2, table_comment='This is a table level comment.')
-
+ecsv.add_table_comment('GLOBAL_SUMMARY', 'This is another table level comment', index=2)
 # Write to string
 ecsvs = woudc_extcsv.dumps(ecsv)
 
@@ -114,28 +114,54 @@ ecsv = woudc_extcsv.load('file.csv')
 # load from string into Reader object
 ecsv = woudc_extcsv.loads(my_ecsv_string)
 # dump to file from Writer object
-ecsv = woudc_extcsv.dump('file.csv')
+ecsv = woudc_extcsv.dump(ecsv_writer, 'file.csv')
 # dump to string from Writer object
-ecsv = woudc_extcsv.dumps(my_ecsv_string)
+ecsv = woudc_extcsv.dumps(ecsv_writer)
+```
+
+### ExtendedCSV Objects
+The ExtendedCSV class is a parser class used in the Reader and Writer classes, and can be used to both read and write to an Extended CSV object.
+
+```python
+from woudc_extcsv import ExtendedCSV
+# read from file
+with open('file.csv', 'r') as ff:
+    ecsv = ExtendedCSV(ff.read())
+# read from string
+ecsv = Extended(my_ecsv_string)
+
+# Add a file-level comment
+ecsv.add_comment('This is a file level comment')
+# Add new table to object
+ecsv.init_table('GLOBAL', ['Wavelength', 'S-Irradiance'], line_num)
+# Add another field to the new table
+ecsv.add_field_to_table('GLOBAL', ['Time'])
+# Add data to table
+ecsv.add_values_to_table('GLOBAL', ['290.5', '0.000E+00', ''], line_num)
+# Add a table comment
+ecsv.add_table_comment('GLOBAL', 'This is a table level comment')
+# Remove a table
+ecsv.remove_table('GLOBAL')
+# Validate Extended CSV and collimate the tables
+# Check metadata tables
+ecsv.validate_metadata_tables()
+# Check dataset-specific tables
+ecsv.validate_dataset_tables()
 ```
 
 ### Error Handling
 
-```pyhon
-from woudc_extcsv import loads, WOUDCExtCSVReaderError
+```python
+from woudc_extcsv import ExtendedCSV, NonStandardDataError, MetadataValidationError
 
 try:
-    loads('bad content!')
-except WOUDCExtCSVReaderError as err:
+    ecsv = ExtendedCSV('bad content!')
+except (NonStandardDataError, MetadataValidationError) as err:
     print(err.message)
     for error in err.errors:
          print(error)
 ```
 
-## Examples
-
-See the `examples/` directory for sample scripts.
-
 ## Development
 
 For development environments, install

requirements.txt

@@ -0,0 +1,2 @@
+jsonschema
+pyyaml

setup.py

@@ -18,7 +18,7 @@
 # those files. Users are asked to read the 3rd Party Licenses
 # referenced with those assets.
 #
-# Copyright (c) 2015 Government of Canada
+# Copyright (c) 2022 Government of Canada
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -126,7 +126,10 @@ def get_package_version():
     url=URL,
     install_requires=INSTALL_REQUIRES,
     packages=find_packages('.'),
-    package_data={'woudc_extcsv': ['table_configuration.csv']},
+    package_data={
+        'woudc_extcsv': ['tables-schema.json', 'tables-backfilling.yml',
+                         'errors-backfilling.csv']
+    },
     scripts=SCRIPTS,
     classifiers=[
         'Development Status :: 4 - Beta',

tests/data/broad-band/20080101.Kipp_Zonen.UV-S-E-T.000560.PMOD-WRC.csv

@@ -0,0 +1,31 @@
+#CONTENT
+Class,Category,Level,Form
+WOUDC,Broad-band,1.0,1
+
+#DATA_GENERATION
+Date,Agency,Version,ScientificAuthority
+2009-12-08,PMOD-WRC,3.0,Julian Groebner
+
+#INSTRUMENT
+Name,Model,Number
+Kipp_Zonen,UV-S-E-T,000560
+
+#PLATFORM
+Type,ID,Name,Country,GAW_ID
+STN,501,DAVOS,CHE
+
+#LOCATION
+Latitude,Longitude,Height
+46.82,9.85,1590
+
+#TIMESTAMP
+UTCOffset,Date
++00:00:00,2008-01-01
+
+#GLOBAL
+Time,Irradiance
+00:01:02,0.000000
+00:03:02,0.000000
+00:05:02,0.000000
+00:07:02,0.000000
+00:09:02,0.000001

tests/data/broad-band/20100109.Kipp_Zonen.UV-S-B-C.020579.ASM-ARG.csv

@@ -0,0 +1,31 @@
+#CONTENT
+Class,Category,Level,Form
+WOUDC,Broad-band,1.0,1
+
+#DATA_GENERATION
+Date,Agency,Version,ScientificAuthority
+2010-02-04,ASM-ARG,1.0,Dr. A. Aculinin
+
+#PLATFORM
+Type,ID,Name,Country,GAW_ID
+STN,455,Kishinev,MDA
+
+#INSTRUMENT
+Name,Model,Number
+Kipp_Zonen,UV-S-B-C,020579
+
+#LOCATION
+Latitude,Longitude,Height
+47.0,28.82,205
+
+#TIMESTAMP
+UTCOffset,Date,Time
++00:00:00,2010-01-09
+
+#DIFFUSE
+Time,Irradiance
+06:00:00,0.002
+06:01:00,0.002
+06:02:00,0.002
+06:03:00,0.002
+06:04:00,0.002

tests/data/general/ecsv-comments.csv

@@ -0,0 +1,65 @@
+* These data were originally received by the WOUDC in SHADOZ file format and
+* have been translated into extCSV file format for WOUDC archiving.
+* This translation process re-formats these data to comply with WOUDC standards.
+
+* This file was generated by SHADOZ_TO_CSX - version 3.1.
+* 'na' is used where Instrument Model or Number are not available.
+
+* Source File: ronbrown_20040709_L1.txt
+
+* NASA/GSFC/INTEX-NA/IONS04 Archive
+* Created 12 March, 2007, SHADOZ V4 Format
+* STATION                          : R/V R.H. Brown
+* Station Principal Investigator   : Samuel J. Oltmans (NOAA/ESRL, CO, US)
+* Latitude (deg)                   : 43.4400
+* Longitude (deg)                  : -66.5700
+* Elevation (m)                    : m.s.l.
+* Launch Date                      : 20040709
+* Launch Time (UT)                 : 14:00
+* Sonde Type/KI Solution           : 2Z EnSci, 0.5% buffered
+* Applied pump corrections         : NOAA/ESRL measured
+* Burst Pressure (hPa)             : 2.60
+* Integrated O3 (DU)               : 372.57
+* Sonde/Sage Climatology(1988-2002): 7.4
+* Const. Mixing Ratio Residual (DU): 27.1
+
+#CONTENT
+Class,Category,Level,Form
+IONS,OzoneSonde,1.0,1
+
+#DATA_GENERATION
+Date,Agency,Version,ScientificAuthority
+2007-03-12,NOAA-CMDL,1.0,"Samuel J. Oltmans (NOAA/ESRL, CO, US)"
+* IONS Project Co-ordinator: Anne M. Thompson (NASA/GSFC)
+
+#PLATFORM
+Type,ID,Name,Country,GAW_ID
+SHP,440,R H BROWN RESEARCH SHIP,*IW,
+
+#INSTRUMENT
+Name,Model,Number
+ECC,2Z,2ZL1
+
+#LOCATION
+Latitude,Longitude,Height
+43.44,-66.57,2
+
+#TIMESTAMP
+UTCOffset,Date,Time
++00:00:00,2004-07-09,14:00:00
+
+#FLIGHT_SUMMARY
+IntegratedO3,CorrectionCode,SondeTotalO3,CorrectionFactor,TotalO3,WLcode,ObsType,Instrument,Number
+372.57,,,,,,,,
+
+#AUXILIARY_DATA
+MeteoSonde,ib1,ib2,PumpRate,BackgroundCorr,SampleTemperatureType,MinutesGroundO3
+,,,,,PUMP,
+
+#PROFILE
+Pressure,O3PartialPressure,Temperature,WindSpeed,WindDirection,LevelCode,Duration,GPHeight,RelativeHumidity,SampleTemperature
+1007.70,2.540,9.50,1.6,18.4,,,0,94,
+1008.00,2.450,9.30,2.2,68.0,,,20,94,
+1006.40,2.630,9.20,3.1,109.8,,,30,95,
+1004.90,2.620,9.10,3.6,137.2,,,40,95,
+1004.90,2.540,9.40,5.3,145.7,,,40,96,