.gitignore

 venv/
+testing/
+logs/
 .ipynb_checkpoints
 .gitignore
 .idea/
 .git/
-__pycache__
\ No newline at end of file
+__pycache__
+.DS_Store
+.notes
+packages/config.py

nc2icartt.py

@@ -568,7 +568,7 @@ def get_launch_information(filename, in_dir):
     try:
         file_list = [f for f in os.listdir(in_dir)]
     except FileNotFoundError as fnfe: #todo remove?
-        traceback.print_tb(fnfe, 2)
+        print("file not found when tryig to list contents of in_dir: {}".format(in_dir))
 
     # In this case, the file we are splitting is assumed to be the normal ADI netCDF filename format,
     # e.g. sgpaafccn200F1.a1.20160411.20582.nc
@@ -722,6 +722,8 @@ def main(ncfile, vars, in_dir, out_dir):
 
     # Read in the netCDF file that is to be converted.
     datagrp = Dataset(ncfile, "r")
+    if 'all' in vars:
+        vars = datagrp.variables.keys()
 
     dimensions, datalength = get_dimensions(datagrp)
 
@@ -829,13 +831,15 @@ def nc2icartt(main_args):
     files = [os.path.join(main_args.in_dir, x) for x in files]
 
     print('opening {}'.format(main_args.var_list))
-    try:
+    if main_args.var_list:
         with open(main_args.var_list) as open_file:
             print('reading variables')
             variables = open_file.readlines()
-    except [TypeError, FileNotFoundError]:
-        print("nc2icartt: File error - var_list = {}".format(main_args.var_list))
-        return(1)
+    else:
+        # todo make this default to all variables if no file found
+        variables = ['all']
+        # print("nc2icartt: File error - var_list = {}".format(main_args.var_list))
+        # return(1)
 
     print('striping variables')
 

netcdf2ascii

@@ -44,6 +44,7 @@ import sys
 import json
 from nc2icartt import nc2icartt
 from nc2csv import nc2csv
+import logging
 
 # Input parsing
 import argparse
@@ -51,34 +52,17 @@ import argparse
 sys.path.insert(0, "/apps/base/python3.5/lib/python3.5/site-packages")
 sys.path.insert(1, "/apps/adc/retrievals/ARM-utils/packages")
 
-# Configuration and communication library
-# FIXME: remove these if you don't have them
+logging.basicConfig(level=logging.WARNING,
+                    format='[%(asctime)s] %(levelname)s (%(funcName)s %(lineno)s) : %(message)s',
+                    datefmt='%Y-%m-%d %H:%M')
+main_logger = logging.getLogger("main")
 try:
     import comms
+    from config import *
 except Exception:
     print("Could not import comms/config. No monitoring messages will be sent.")
 
 
-def is_arial(file_name):
-    print(file_name)
-    print(file_name.split("."))
-    print(file_name.split(".")[0])
-    print(file_name.split(".")[0][-2])
-    facility = file_name.split(".")[0][-2]
-    return "F" == facility or "U" == facility
-
-def get_first_file(file_list):
-    files = list()
-    if os.path.isfile(file_list):
-        for file_name in open(file_list, "r"):
-            infile_path = os.path.join(main_args.in_dir, file_name.strip("\n"))
-            if os.path.isfile(infile_path):
-                return file_name
-            else:
-                raise FileNotFoundError
-    else:
-        raise FileNotFoundError
-
 help_description = """
 This program converts netcdf files into csv files.
 """
@@ -112,43 +96,33 @@ Returns:
 
 
 def parse_arguments():
+    '''Instantiates an argument parser and returns loads in artument flags.
+
+    :return (argparse.Namespace): Ojbect with arguments and flags as dot properties.
+    '''
     parser = argparse.ArgumentParser(description=help_description, epilog=example,
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     requiredArguments = parser.add_argument_group("required arguments")
+    optionalArguments = parser.add_argument_group("optional arguments")
 
-    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", help="The directory where input files are.")
-    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir",
+    # The type is a function to validate that the path exists, only for the required arguments.
+    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", required=True,
+                                   help="The directory where input files are.")
+    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir", required=True,
                                    help="The directory to put output file in.")
-    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list", default=None,
-                                   help="The name of a file that contains the names of variables to extract.")
-    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list",
+    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list", required=True,
                                    help="The name of a file that contains the names of files to convert.")
+    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list",
+                                   help="The name of a file that contains the names of variables to extract.")
 
-    parser.add_argument("--mergedoutput", action="store_true", dest="merged_output",
+    optionalArguments.add_argument("--mergedoutput", action="store_true", dest="merged_output",
                         help="merge files into one, possibly with max size")
-    parser.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
+    # DQR filter can take the following forms; --DQRfilter nofilter,incorrect,suspect
+    optionalArguments.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
                         help="filter out data with dqr, <suspect> or <incorrect> or <suspect,incorrect>")
-
-    parser.add_argument("-d", "--delimiter", type=str, dest="delimiter", default=",",
-                        help="delimiter, ' ', ',' '-', '\t' or 'tab' ")
-    parser.add_argument("-e", "--eol", type=str, dest='eol', default='lf',
-                        help="end of line character, line feed <lf> \n, carrage return <cr> \r or both <cr/lf>")
-    parser.add_argument("--timestamp", type=str, dest="timeStamp", default='calendar',
-                        help="type of timestamp format to add. none, calendar, ordinal")
-    parser.add_argument("-p", "--maxProcesses", type=int, dest="maxProcesses", default=6,
-                        help="Maximum number of processes to spawn.")
-    parser.add_argument("--replaceMissing", action="store_true", dest="replaceMissing",
-                        help="replace missing values (-9999, -9995) with blank value.")
-    parser.add_argument("--orderid", dest="orderID", default=None, help="ANDERS legacy attribute")
-    parser.add_argument("--outmaxMB", type=int, dest="outMaxMB", default=0,
-                        help="ANDERS legacy attribute, used to be the maximum size of the merged output in MB")
-    parser.add_argument("--allFields", action="store_true", dest="allFields",
-                        help="ANDERS legacy attribute,this would signify all variables."
-                             "here that is done if there is no var_list, var_list is empty or contains all variables")
-    parser.add_argument("--asciiblocks", action="store_true", dest="asciiBlocks", help="ANDERS legacy attribute")
-
-    parser.add_argument("-D", "--Debug", action="store_true", dest="debug", help="enables debug printing")
+    optionalArguments.add_argument("-mp", type=int, dest="maxProcesses", default=3,
+                        help="number of processes in multiprocessing pool")
 
     args, unknownArgs = parser.parse_known_args()
     if len(sys.argv) <= 1:
@@ -162,33 +136,77 @@ success_msg = "Successful {}: {} data extracted\n" \
               "main_args: {}"
 
 def netcdf2ascii(main_args):
+    main_logger.debug("main_args: {}".format(main_args))
     try:
-        print("main_args: {}".format(main_args))
-        if is_arial(get_first_file(main_args.file_list)):
-            print('Processing arial files...')
+        if is_arial(get_first_file(main_args.file_list, main_args.in_dir)):
+            main_logger.info('Processing arial files...')
             proc_type = "arial"
             exit_code = nc2icartt(main_args)
-            print("exit({})".format(exit_code))
+            main_logger.info("netcdf2ascii.icart: exit({})".format(exit_code))
         else:
-            print('Processing netcdf files...')
+            main_logger.info('Processing netcdf files...')
             proc_type = "netcdf"
             exit_code = nc2csv(main_args)
-            print("exit({})".format(exit_code))
-        # try:
-        #     comms.slack_send_direct(success_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
-        # except Exception:
-        #     print("No monitoring message sent.")
-        exit(exit_code)
-    except Exception as e:
-        print("ERROR: {}".format(e))
+            main_logger.info("netcdf2ascii.nc2csv: exit({})".format(exit_code))
+    except (FileNotFoundError, NotADirectoryError, ConnectionError):
+        main_logger.info("netcdf2ascii.nc2csv: exit(1)")
+        try:
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
+        except Exception:
+            pass
+        exit(1)
+    except BlockingIOError:
+        main_logger.critical("Unknown exception: exit(7)")
         try:
-            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
         except Exception:
-            print("No monitoring message sent.")
-        exit(2)
+            pass
+        exit(7)
+
+def valid_argument(arg: str) -> str:
+    '''Checks to make sure the input argument exists. If not raise argparse.ArgumentTypeError.
+
+    :param arg (str): One of the required input arguments.
+    :return str: The input argument path as a string if it exists.
+    '''
+    if os.path.exists(arg):
+        return arg
+    else:
+        raise argparse.ArgumentTypeError("Input argument does not exist: {}".format(arg))
+
+def is_arial(file_name):
+    main_logger.debug(file_name)
+    main_logger.debug(file_name.split("."))
+    main_logger.debug(file_name.split(".")[0])
+    main_logger.debug(file_name.split(".")[0][-2])
+    facility = file_name.split(".")[0][-2]
+    return "F" == facility or "U" == facility
+
+def get_first_file(file_list, in_dir):
+    files = list()
+    main_logger.debug('file_list: {}'.format(file_list))
+    try:
+        if os.path.isfile(file_list):
+            if os.path.isdir(in_dir):
+                for file_name in open(file_list, "r"):
+                    infile_path = os.path.join(in_dir, file_name.strip("\n"))
+                    if os.path.isfile(infile_path):
+                        return file_name
+                    else:
+                        main_logger.warning("File to convert does not exist: {}".format(infile_path))
+                main_logger.critical("No valid files in file list.")
+                raise FileNotFoundError
+            else:
+                main_logger.critical("Input directory does not exist.")
+                raise NotADirectoryError
+        else:
+            main_logger.critical("File list does not exist.")
+            raise FileNotFoundError
+    except TypeError:
+        main_logger.critical("Missing argument; file_list and/or in_dir are null.")
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         main_args,unknownArgs = parse_arguments()
 
-        netcdf2ascii(main_args)
+        netcdf2ascii(main_args)
\ No newline at end of file

packages/logging_config.yaml

@@ -22,6 +22,10 @@ logging:
       handlers: [console, debug_file_handler]
       qualname: nc2csv_logger
       propagate: false
+    nc2csv_test_logger:
+      handlers: [test_console]
+      qualname: nc2csv_test_logger
+      propagate: false
 
   formatters:
       simple:
@@ -35,6 +39,11 @@ logging:
         level: INFO
         formatter: simple
         stream: ext://sys.stdout
+      test_console:
+        class: logging.StreamHandler
+        level: DEBUG
+        formatter: simple
+        stream: ext://sys.stdout
       debug_file_handler:
         class: logging.handlers.RotatingFileHandler
         level: DEBUG

requirements.txt

-cftime==1.0.0
-Cython==0.28.3
-netCDF4==1.4.0
-numpy==1.14.3
-pandas==0.23.0
-python-dateutil==2.7.3
-pytz==2018.4
-six==1.11.0