.gitignore

 venv/
+testing/
+logs/
 .ipynb_checkpoints
 .gitignore
 .idea/
 .git/
-__pycache__
\ No newline at end of file
+__pycache__

netcdf2ascii

@@ -44,6 +44,7 @@ import sys
 import json
 from nc2icartt import nc2icartt
 from nc2csv import nc2csv
+import logging
 
 # Input parsing
 import argparse
@@ -51,34 +52,17 @@ import argparse
 sys.path.insert(0, "/apps/base/python3.5/lib/python3.5/site-packages")
 sys.path.insert(1, "/apps/adc/retrievals/ARM-utils/packages")
 
-# Configuration and communication library
-# FIXME: remove these if you don't have them
+logging.basicConfig(level=logging.WARNING,
+                    format='[%(asctime)s] %(levelname)s (%(funcName)s %(lineno)s) : %(message)s',
+                    datefmt='%Y-%m-%d %H:%M')
+main_logger = logging.getLogger("main")
 try:
     import comms
+    from config import *
 except Exception:
     print("Could not import comms/config. No monitoring messages will be sent.")
 
 
-def is_arial(file_name):
-    print(file_name)
-    print(file_name.split("."))
-    print(file_name.split(".")[0])
-    print(file_name.split(".")[0][-2])
-    facility = file_name.split(".")[0][-2]
-    return "F" == facility or "U" == facility
-
-def get_first_file(file_list):
-    files = list()
-    if os.path.isfile(file_list):
-        for file_name in open(file_list, "r"):
-            infile_path = os.path.join(main_args.in_dir, file_name.strip("\n"))
-            if os.path.isfile(infile_path):
-                return file_name
-            else:
-                raise FileNotFoundError
-    else:
-        raise FileNotFoundError
-
 help_description = """
 This program converts netcdf files into csv files.
 """
@@ -112,43 +96,33 @@ Returns:
 
 
 def parse_arguments():
+    '''Instantiates an argument parser and returns loads in artument flags.
+
+    :return (argparse.Namespace): Ojbect with arguments and flags as dot properties.
+    '''
     parser = argparse.ArgumentParser(description=help_description, epilog=example,
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     requiredArguments = parser.add_argument_group("required arguments")
+    optionalArguments = parser.add_argument_group("optional arguments")
 
-    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", help="The directory where input files are.")
-    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir",
+    # The type is a function to validate that the path exists, only for the required arguments.
+    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", required=True,
+                                   help="The directory where input files are.")
+    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir", required=True,
                                    help="The directory to put output file in.")
-    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list", default=None,
-                                   help="The name of a file that contains the names of variables to extract.")
-    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list",
+    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list", required=True,
                                    help="The name of a file that contains the names of files to convert.")
+    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list",
+                                   help="The name of a file that contains the names of variables to extract.")
 
-    parser.add_argument("--mergedoutput", action="store_true", dest="merged_output",
+    optionalArguments.add_argument("--mergedoutput", action="store_true", dest="merged_output",
                         help="merge files into one, possibly with max size")
-    parser.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
+    # DQR filter can take the following forms; --DQRfilter nofilter,incorrect,suspect
+    optionalArguments.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
                         help="filter out data with dqr, <suspect> or <incorrect> or <suspect,incorrect>")
-
-    parser.add_argument("-d", "--delimiter", type=str, dest="delimiter", default=",",
-                        help="delimiter, ' ', ',' '-', '\t' or 'tab' ")
-    parser.add_argument("-e", "--eol", type=str, dest='eol', default='lf',
-                        help="end of line character, line feed <lf> \n, carrage return <cr> \r or both <cr/lf>")
-    parser.add_argument("--timestamp", type=str, dest="timeStamp", default='calendar',
-                        help="type of timestamp format to add. none, calendar, ordinal")
-    parser.add_argument("-p", "--maxProcesses", type=int, dest="maxProcesses", default=6,
-                        help="Maximum number of processes to spawn.")
-    parser.add_argument("--replaceMissing", action="store_true", dest="replaceMissing",
-                        help="replace missing values (-9999, -9995) with blank value.")
-    parser.add_argument("--orderid", dest="orderID", default=None, help="ANDERS legacy attribute")
-    parser.add_argument("--outmaxMB", type=int, dest="outMaxMB", default=0,
-                        help="ANDERS legacy attribute, used to be the maximum size of the merged output in MB")
-    parser.add_argument("--allFields", action="store_true", dest="allFields",
-                        help="ANDERS legacy attribute,this would signify all variables."
-                             "here that is done if there is no var_list, var_list is empty or contains all variables")
-    parser.add_argument("--asciiblocks", action="store_true", dest="asciiBlocks", help="ANDERS legacy attribute")
-
-    parser.add_argument("-D", "--Debug", action="store_true", dest="debug", help="enables debug printing")
+    optionalArguments.add_argument("-mp", type=int, dest="maxProcesses", default=3,
+                        help="number of processes in multiprocessing pool")
 
     args, unknownArgs = parser.parse_known_args()
     if len(sys.argv) <= 1:
@@ -162,33 +136,77 @@ success_msg = "Successful {}: {} data extracted\n" \
               "main_args: {}"
 
 def netcdf2ascii(main_args):
+    main_logger.debug("main_args: {}".format(main_args))
     try:
-        print("main_args: {}".format(main_args))
-        if is_arial(get_first_file(main_args.file_list)):
-            print('Processing arial files...')
+        if is_arial(get_first_file(main_args.file_list, main_args.in_dir)):
+            main_logger.info('Processing arial files...')
             proc_type = "arial"
             exit_code = nc2icartt(main_args)
-            print("exit({})".format(exit_code))
+            main_logger.info("netcdf2ascii.icart: exit({})".format(exit_code))
         else:
-            print('Processing netcdf files...')
+            main_logger.info('Processing netcdf files...')
             proc_type = "netcdf"
             exit_code = nc2csv(main_args)
-            print("exit({})".format(exit_code))
-        # try:
-        #     comms.slack_send_direct(success_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
-        # except Exception:
-        #     print("No monitoring message sent.")
-        exit(exit_code)
-    except Exception as e:
-        print("ERROR: {}".format(e))
+            main_logger.info("netcdf2ascii.nc2csv: exit({})".format(exit_code))
+    except (FileNotFoundError, NotADirectoryError, ConnectionError):
+        main_logger.info("netcdf2ascii.nc2csv: exit(1)")
+        try:
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
+        except Exception:
+            pass
+        exit(1)
+    except BlockingIOError:
+        main_logger.critical("Unknown exception: exit(7)")
         try:
-            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
         except Exception:
-            print("No monitoring message sent.")
-        exit(2)
+            pass
+        exit(7)
+
+def valid_argument(arg: str) -> str:
+    '''Checks to make sure the input argument exists. If not raise argparse.ArgumentTypeError.
+
+    :param arg (str): One of the required input arguments.
+    :return str: The input argument path as a string if it exists.
+    '''
+    if os.path.exists(arg):
+        return arg
+    else:
+        raise argparse.ArgumentTypeError("Input argument does not exist: {}".format(arg))
+
+def is_arial(file_name):
+    main_logger.debug(file_name)
+    main_logger.debug(file_name.split("."))
+    main_logger.debug(file_name.split(".")[0])
+    main_logger.debug(file_name.split(".")[0][-2])
+    facility = file_name.split(".")[0][-2]
+    return "F" == facility or "U" == facility
+
+def get_first_file(file_list, in_dir):
+    files = list()
+    main_logger.debug('file_list: {}'.format(file_list))
+    try:
+        if os.path.isfile(file_list):
+            if os.path.isdir(in_dir):
+                for file_name in open(file_list, "r"):
+                    infile_path = os.path.join(in_dir, file_name.strip("\n"))
+                    if os.path.isfile(infile_path):
+                        return file_name
+                    else:
+                        main_logger.warning("File to convert does not exist: {}".format(infile_path))
+                main_logger.critical("No valid files in file list.")
+                raise FileNotFoundError
+            else:
+                main_logger.critical("Input directory does not exist.")
+                raise NotADirectoryError
+        else:
+            main_logger.critical("File list does not exist.")
+            raise FileNotFoundError
+    except TypeError:
+        main_logger.critical("Missing argument; file_list and/or in_dir are null.")
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         main_args,unknownArgs = parse_arguments()
 
-        netcdf2ascii(main_args)
+        netcdf2ascii(main_args)
\ No newline at end of file

requirements.txt

-cftime==1.0.0
-Cython==0.28.3
-netCDF4==1.4.0
-numpy==1.14.3
-pandas==0.23.0
-python-dateutil==2.7.3
-pytz==2018.4
-six==1.11.0