netcdf2ascii

#!/apps/base/python3/bin/python3

"""
Author: Michael Giansiracusa
Email: giansiracumt@ornl.gov

Purpose:
    This is a wrapper to decide which nc 2 ascii script to call, icartt for arial data, cdf for all others

Arguments
    required arguments
        in_dir : str
            fully qualified path to files that will be processed.
        out_dir : str
            fully qualified path where to put output files.
        var_list : str
            fully qualified path to a file that contains all
            variables to extract each on it's own line.
        file_list : str
            fully qualified path to a file that contains all
            files in the in_dir to process.
    optional arguments
        merged_output : bool flag
            output files merged into one if provided
        replaceMissing : bool flag
            replace missing values (-9999, -9995) with ""/None
        DQRfilter : str
            filter data by variable using dqr status.
            should be suspect, incorrect, or suspect,incorrect


Output:
    One cdf file for each file in input directory is put in the out_dir or
    one merged file with all rows and columns converted from input cdf files.
    The first row of each file will be the column names; the first 3 column
    names will always be time_stamp, basetime, and time_offset. The time_stamp
    column is calculated by adding the basetime and time_offset and converting
    to a calendar date of the form YYYY-mm-dd HH:MM:SS
"""

# Operations
import os
import sys
import json
from nc2icartt import nc2icartt
from nc2csv import nc2csv
import logging

# Input parsing
import argparse

sys.path.insert(0, "/apps/base/python3.5/lib/python3.5/site-packages")
sys.path.insert(1, "/apps/adc/retrievals/ARM-utils/packages")

logging.basicConfig(level=logging.WARNING,
                    format='[%(asctime)s] %(levelname)s (%(funcName)s %(lineno)s) : %(message)s',
                    datefmt='%Y-%m-%d %H:%M')
main_logger = logging.getLogger("main")
try:
    import comms
    from config import *
except Exception:
    print("Could not import comms/config. No monitoring messages will be sent.")


help_description = """
This program converts netcdf files into csv files.
"""

example = """
EXAMPLE: py3 netcdf2ascii.py
            --in_dir "/path_to/input"
            --out_dir "/path_to/output"
            --var_list "/path_to/var_list.txt"
            --file_list "/path_to/file_list.txt"
"""

""" setup_arguments summary

Author: Michael Giansiracusa
Email: giansiracumt@ornl.gov

Purpose:
    Parse arguments from command line.

Args:
    None

Returns:
    args object with attributes
            in_dir : str
            out_dir : str
            var_list : str
            file_list : str
"""


def parse_arguments():
    '''Instantiates an argument parser and returns loads in artument flags.

    :return (argparse.Namespace): Ojbect with arguments and flags as dot properties.
    '''
    parser = argparse.ArgumentParser(description=help_description, epilog=example,
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    requiredArguments = parser.add_argument_group("required arguments")
    optionalArguments = parser.add_argument_group("optional arguments")

    # The type is a function to validate that the path exists, only for the required arguments.
    requiredArguments.add_argument("-i", "--indir", type=str, dest="in_dir", required=True,
                                   help="The directory where input files are.")
    requiredArguments.add_argument("-o", "--outdir", type=str, dest="out_dir", required=True,
                                   help="The directory to put output file in.")
    requiredArguments.add_argument("-f", "--filelist", type=str, dest="file_list", required=True,
                                   help="The name of a file that contains the names of files to convert.")
    requiredArguments.add_argument("-v", "--varlist", type=str, dest="var_list",
                                   help="The name of a file that contains the names of variables to extract.")

    optionalArguments.add_argument("--mergedoutput", action="store_true", dest="merged_output",
                        help="merge files into one, possibly with max size")
    # DQR filter can take the following forms; --DQRfilter nofilter,incorrect,suspect
    optionalArguments.add_argument("--DQRfilter", type=str, dest="DQRfilter", default="",
                        help="filter out data with dqr, <suspect> or <incorrect> or <suspect,incorrect>")
    optionalArguments.add_argument("-mp", type=int, dest="maxProcesses", default=3,
                        help="number of processes in multiprocessing pool")

    args, unknownArgs = parser.parse_known_args()
    if len(sys.argv) <= 1:
        parser.print_help()

    return args, unknownArgs

error_msg = "Error {}: Problem with {} extraction.\n" \
            "main_args: {}"
success_msg = "Successful {}: {} data extracted\n" \
              "main_args: {}"

def netcdf2ascii(main_args):
    main_logger.debug("main_args: {}".format(main_args))
    try:
        if is_arial(get_first_file(main_args.file_list, main_args.in_dir)):
            main_logger.info('Processing arial files...')
            proc_type = "arial"
            exit_code = nc2icartt(main_args)
            main_logger.info("netcdf2ascii.icart: exit({})".format(exit_code))
        else:
            main_logger.info('Processing netcdf files...')
            proc_type = "netcdf"
            exit_code = nc2csv(main_args)
            main_logger.info("netcdf2ascii.nc2csv: exit({})".format(exit_code))
    except (FileNotFoundError, NotADirectoryError, ConnectionError):
        main_logger.info("netcdf2ascii.nc2csv: exit(1)")
        try:
            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
        except Exception:
            pass
        exit(1)
    except BlockingIOError:
        main_logger.critical("Unknown exception: exit(7)")
        try:
            comms.slack_send_direct(error_msg.format(exit_code, proc_type, main_args))
        except Exception:
            pass
        exit(7)

def valid_argument(arg: str) -> str:
    '''Checks to make sure the input argument exists. If not raise argparse.ArgumentTypeError.

    :param arg (str): One of the required input arguments.
    :return str: The input argument path as a string if it exists.
    '''
    if os.path.exists(arg):
        return arg
    else:
        raise argparse.ArgumentTypeError("Input argument does not exist: {}".format(arg))

def is_arial(file_name):
    main_logger.debug(file_name)
    main_logger.debug(file_name.split("."))
    main_logger.debug(file_name.split(".")[0])
    main_logger.debug(file_name.split(".")[0][-2])
    facility = file_name.split(".")[0][-2]
    return "F" == facility or "U" == facility

def get_first_file(file_list, in_dir):
    files = list()
    main_logger.debug('file_list: {}'.format(file_list))
    try:
        if os.path.isfile(file_list):
            if os.path.isdir(in_dir):
                for file_name in open(file_list, "r"):
                    infile_path = os.path.join(in_dir, file_name.strip("\n"))
                    if os.path.isfile(infile_path):
                        return file_name
                    else:
                        main_logger.warning("File to convert does not exist: {}".format(infile_path))
                main_logger.critical("No valid files in file list.")
                raise FileNotFoundError
            else:
                main_logger.critical("Input directory does not exist.")
                raise NotADirectoryError
        else:
            main_logger.critical("File list does not exist.")
            raise FileNotFoundError
    except TypeError:
        main_logger.critical("Missing argument; file_list and/or in_dir are null.")

if __name__ == "__main__":
    if len(sys.argv) > 1:
        main_args,unknownArgs = parse_arguments()

        netcdf2ascii(main_args)