.gitignore

 venv/
+testing/
+logs/
 .ipynb_checkpoints
 .gitignore
 .idea/
 .git/
-__pycache__
\ No newline at end of file
+__pycache__

netcdf2ascii

@@ -44,6 +44,7 @@ import sys
 import json
 from nc2icartt import nc2icartt
 from nc2csv import nc2csv
+import logging
 
 # Input parsing
 import argparse
@@ -51,8 +52,10 @@ import argparse
 sys.path.insert(0, "/apps/base/python3.5/lib/python3.5/site-packages")
 sys.path.insert(1, "/apps/adc/retrievals/ARM-utils/packages")
 
-# Configuration and communication library
-# FIXME: remove these if you don't have them
+logging.basicConfig(level=logging.WARNING,
+                    format='[%(asctime)s] %(levelname)s (%(funcName)s %(lineno)s) : %(message)s',
+                    datefmt='%Y-%m-%d %H:%M')
+main_logger = logging.getLogger("main")
 try:
     import comms
     from config import *
@@ -60,26 +63,6 @@ except Exception:
     print("Could not import comms/config. No monitoring messages will be sent.")
 
 
-def is_arial(file_name):
-    print(file_name)
-    print(file_name.split("."))
-    print(file_name.split(".")[0])
-    print(file_name.split(".")[0][-2])
-    facility = file_name.split(".")[0][-2]
-    return "F" == facility or "U" == facility
-
-def get_first_file(file_list):
-    files = list()
-    if os.path.isfile(file_list):
-        for file_name in open(file_list, "r"):
-            infile_path = os.path.join(main_args.in_dir, file_name.strip("\n"))
-            if os.path.isfile(infile_path):
-                return file_name
-            else:
-                raise FileNotFoundError
-    else:
-        raise FileNotFoundError
-
 help_description = """
 This program converts netcdf files into csv files.
 """
@@ -113,43 +96,33 @@ Returns:
 
 
 def parse_arguments():
+    '''Instantiates an argument parser and returns loads in artument flags.
+
+    :return (argparse.Namespace): Ojbect with arguments and flags as dot properties.
+    '''
     parser = argparse.ArgumentParser(description=help_description, epilog=example,
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     requiredArguments = parser.add_argument_group("required arguments")
+    optionalArguments = parser.add_argument_group("optional arguments")
 
-    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", help="The directory where input files are.")
-    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir",
+    # The type is a function to validate that the path exists, only for the required arguments.
+    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", required=True,
+                                   help="The directory where input files are.")
+    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir", required=True,
                                    help="The directory to put output file in.")
-    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list", default=None,
-                                   help="The name of a file that contains the names of variables to extract.")
-    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list",
+    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list", required=True,
                                    help="The name of a file that contains the names of files to convert.")
+    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list",
+                                   help="The name of a file that contains the names of variables to extract.")
 
-    parser.add_argument("--mergedoutput", action="store_true", dest="merged_output",
+    optionalArguments.add_argument("--mergedoutput", action="store_true", dest="merged_output",
                         help="merge files into one, possibly with max size")
-    parser.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
+    # DQR filter can take the following forms; --DQRfilter nofilter,incorrect,suspect
+    optionalArguments.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
                         help="filter out data with dqr, <suspect> or <incorrect> or <suspect,incorrect>")
-
-    parser.add_argument("-d", "--delimiter", type=str, dest="delimiter", default=",",
-                        help="delimiter, ' ', ',' '-', '\t' or 'tab' ")
-    parser.add_argument("-e", "--eol", type=str, dest='eol', default='lf',
-                        help="end of line character, line feed <lf> \n, carrage return <cr> \r or both <cr/lf>")
-    parser.add_argument("--timestamp", type=str, dest="timeStamp", default='calendar',
-                        help="type of timestamp format to add. none, calendar, ordinal")
-    parser.add_argument("-p", "--maxProcesses", type=int, dest="maxProcesses", default=6,
-                        help="Maximum number of processes to spawn.")
-    parser.add_argument("--replaceMissing", action="store_true", dest="replaceMissing",
-                        help="replace missing values (-9999, -9995) with blank value.")
-    parser.add_argument("--orderid", dest="orderID", default=None, help="ANDERS legacy attribute")
-    parser.add_argument("--outmaxMB", type=int, dest="outMaxMB", default=0,
-                        help="ANDERS legacy attribute, used to be the maximum size of the merged output in MB")
-    parser.add_argument("--allFields", action="store_true", dest="allFields",
-                        help="ANDERS legacy attribute,this would signify all variables."
-                             "here that is done if there is no var_list, var_list is empty or contains all variables")
-    parser.add_argument("--asciiblocks", action="store_true", dest="asciiBlocks", help="ANDERS legacy attribute")
-
-    parser.add_argument("-D", "--Debug", action="store_true", dest="debug", help="enables debug printing")
+    optionalArguments.add_argument("-mp", type=int, dest="maxProcesses", default=3,
+                        help="number of processes in multiprocessing pool")
 
     args, unknownArgs = parser.parse_known_args()
     if len(sys.argv) <= 1:
@@ -163,33 +136,77 @@ success_msg = "Successful {}: {} data extracted\n" \
               "main_args: {}"
 
 def netcdf2ascii(main_args):
+    main_logger.debug("main_args: {}".format(main_args))
     try:
-        print("main_args: {}".format(main_args))
-        if is_arial(get_first_file(main_args.file_list)):
-            print('Processing arial files...')
+        if is_arial(get_first_file(main_args.file_list, main_args.in_dir)):
+            main_logger.info('Processing arial files...')
             proc_type = "arial"
             exit_code = nc2icartt(main_args)
-            print("exit({})".format(exit_code))
+            main_logger.info("netcdf2ascii.icart: exit({})".format(exit_code))
         else:
-            print('Processing netcdf files...')
+            main_logger.info('Processing netcdf files...')
             proc_type = "netcdf"
             exit_code = nc2csv(main_args)
-            print("exit({})".format(exit_code))
+            main_logger.info("netcdf2ascii.nc2csv: exit({})".format(exit_code))
+    except (FileNotFoundError, NotADirectoryError, ConnectionError):
+        main_logger.info("netcdf2ascii.nc2csv: exit(1)")
         try:
-            comms.slack_send_direct(success_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
         except Exception:
-            print("No monitoring message sent.")
-        exit(exit_code)
-    except Exception as e:
-        print("ERROR: {}".format(e))
+            pass
+        exit(1)
+    except BlockingIOError:
+        main_logger.critical("Unknown exception: exit(7)")
         try:
-            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
+            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
         except Exception:
-            print("No monitoring message sent.")
-        exit(2)
+            pass
+        exit(7)
+
+def valid_argument(arg: str) -> str:
+    '''Checks to make sure the input argument exists. If not raise argparse.ArgumentTypeError.
+
+    :param arg (str): One of the required input arguments.
+    :return str: The input argument path as a string if it exists.
+    '''
+    if os.path.exists(arg):
+        return arg
+    else:
+        raise argparse.ArgumentTypeError("Input argument does not exist: {}".format(arg))
+
+def is_arial(file_name):
+    main_logger.debug(file_name)
+    main_logger.debug(file_name.split("."))
+    main_logger.debug(file_name.split(".")[0])
+    main_logger.debug(file_name.split(".")[0][-2])
+    facility = file_name.split(".")[0][-2]
+    return "F" == facility or "U" == facility
+
+def get_first_file(file_list, in_dir):
+    files = list()
+    main_logger.debug('file_list: {}'.format(file_list))
+    try:
+        if os.path.isfile(file_list):
+            if os.path.isdir(in_dir):
+                for file_name in open(file_list, "r"):
+                    infile_path = os.path.join(in_dir, file_name.strip("\n"))
+                    if os.path.isfile(infile_path):
+                        return file_name
+                    else:
+                        main_logger.warning("File to convert does not exist: {}".format(infile_path))
+                main_logger.critical("No valid files in file list.")
+                raise FileNotFoundError
+            else:
+                main_logger.critical("Input directory does not exist.")
+                raise NotADirectoryError
+        else:
+            main_logger.critical("File list does not exist.")
+            raise FileNotFoundError
+    except TypeError:
+        main_logger.critical("Missing argument; file_list and/or in_dir are null.")
 
 if __name__ == "__main__":
     if len(sys.argv) > 1:
         main_args,unknownArgs = parse_arguments()
 
-        netcdf2ascii(main_args)
+        netcdf2ascii(main_args)
\ No newline at end of file

packages/comms.py

+import os
+from config import *
+
+def slack_send_direct(message):
+    from slackclient import SlackClient
+
+    slack_token = SLACK_ARM_TOKEN
+    sc = SlackClient(slack_token)
+
+    api_call = sc.api_call("im.list")
+
+    if api_call.get('ok'):
+        for im in api_call.get('ims'):
+            if im.get('user') == SLACK_USER_ID and im.get("is_im") == True:
+                im_channel = im.get("id")
+                sc.api_call("chat.postMessage",
+                            channel=im_channel,
+                            text=message,
+                            user=SLACK_USER_ID)
+
+def send_email(subject:str, message:str, recipients:list):
+    try:
+        import smtplib
+        from email.mime.text import MIMEText
+        executable = os.path.basename(__file__)
+        host = os.uname().__getattribute__('nodename')
+        sender = '{}@{}'.format(executable, host)
+
+        stars = 100*"*"
+        msg = "Sent from {0}. Variable extraction.\n\n" \
+              "{2}\n\t{1}\n{2}".format(host, message, stars)
+        msg = MIMEText(msg)
+
+        msg['Subject'] = "*** {} ***".format(subject)
+        msg['From'] = sender
+        msg['To'] = ", ".join(recipients)
+        s = smtplib.SMTP('localhost')
+        s.sendmail(sender, recipients, msg.as_string())
+    except Exception:
+        print("Email not sent to {}".format(", ".join(recipients)))
+
+def main():
+    print('** Test some stuff\nSend a slack message to Michael')
+    slack_send_direct("test message from python")
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

packages/logging_config.yaml

+# Generic configuration
+# ---------------------
+
+# Config for current application
+# ------------------------------
+
+nc2csv:
+  foo: bar
+  bar: [1, 2]
+
+# Config for logging
+# ------------------
+# From https://gist.github.com/glenfant/4358668
+# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema
+
+logging:
+  version: 1
+  disable_existing_loggers: False
+
+  loggers:
+    nc2csv_logger:
+      handlers: [console, debug_file_handler]
+      qualname: nc2csv_logger
+      propagate: false
+    nc2csv_test_logger:
+      handlers: [test_console]
+      qualname: nc2csv_test_logger
+      propagate: false
+
+  formatters:
+      simple:
+          format: "%(levelname)s — %(funcName)s:%(lineno)d — %(message)s"
+      lineInfo:
+          format: "%(asctime)s — %(name)s — %(levelname)s — %(funcName)s:%(lineno)d — %(message)s"
+
+  handlers:
+      console:
+        class: logging.StreamHandler
+        level: INFO
+        formatter: simple
+        stream: ext://sys.stdout
+      test_console:
+        class: logging.StreamHandler
+        level: DEBUG
+        formatter: simple
+        stream: ext://sys.stdout
+      debug_file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: DEBUG
+        formatter: lineInfo
+        filename: logs/debug.log
+        maxBytes: 10485760 # 10MB
+        backupCount: 10
+        encoding: utf8
+      info_file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: INFO
+        formatter: simple
+        filename: logs/info.log
+        maxBytes: 10485760 # 10MB
+        backupCount: 10
+        encoding: utf8
+      error_file_handler:
+        class: logging.handlers.RotatingFileHandler
+        level: ERROR
+        formatter: simple
+        filename: logs/errors.log
+        maxBytes: 10485760 # 10MB
+        backupCount: 10
+        encoding: utf8
+  root:
+    level: DEBUG
+    handlers: [console, info_file_handler, error_file_handler, debug_file_handler]

requirements.txt

-cftime==1.0.0
-Cython==0.28.3
-netCDF4==1.4.0
-numpy==1.14.3
-pandas==0.23.0
-python-dateutil==2.7.3
-pytz==2018.4
-six==1.11.0