nc2csv.py

@@ -49,6 +49,7 @@ from functools import partial
 import logging.config
 import os
 import re
+import requests
 import sys
 import time
 import warnings
@@ -67,7 +68,15 @@ import pandas as pd
 import numpy as np
 np.seterr(invalid='ignore')
 
-global_config = yaml.load(open(".config.yaml"))
+sys.path.insert(0, "/apps/base/python3.5/lib/python3.5/site-packages")
+sys.path.insert(1, "{}/packages".format(os.getcwd()))
+
+try:
+    import comms
+except Exception:
+    print("Could not import comms/config. No monitoring messages will be sent.")
+
+global_config = yaml.load(open("packages/logging_config.yaml"))
 logging.config.dictConfig(global_config['logging'])
 nc2csv_logger = logging.getLogger("nc2csv_logger")
 
@@ -225,15 +234,23 @@ class Setup():
         # query the web service and parse response into an array of arrays
         def get_timeblocks(url):
             # query web service https://www.archive.arm.gov/dqrws/
-            response = urlopen(url) #FIXME change this to use requests module
+            response = requests.get(url) #FIXME change this to use requests module
             # list to hold parsed response
             timeblocks = list()
             # if response was valid
-            if response.getcode() == 200:
+            if response.status_code == 200:
                 # for line in the response, each line is a dqr record
                 for line in response.readlines():
                     # decode line, parse line and append an array to timeblocks array
-                    timeblocks.append(line.decode().replace('\r\n', '').split('|'))
+                    timeblocks.append(line.decode().replace('\r\n', '').split('|')) # todo modify to work with requests response format, remove the decode and replace maybe?
+            # if bad response from web server
+            elif response.status_code == 500:
+                # try and send a slack message based on config file
+                try:
+                    comms.slack_send_direct("nc2csv Error: DQRWS response: {}".format(response.__dict__))
+                except NameError:
+                    nc2csv_logger.error("Name Error: No comms sent.")
+                raise ConnectionError
 
             return timeblocks
 
@@ -254,7 +271,13 @@ class Setup():
             for var in variables:
                 var_url = "".join((url, "&varname=", var))
                 nc2csv_logger.info("getting dqr timeblocks for {}\n\t{}".format(var, var_url))
-                timeblocks = get_timeblocks(var_url)
+                try:
+                    timeblocks = get_timeblocks(var_url)
+                except ConnectionError:
+                    # if dqrws is down then mark for all variables and break out of loop
+                    nc2csv_logger.error("Error with dqr web service.")
+                    dqr_results['all'] = True
+                    break
                 dqr_results[var] = timeblocks
 
         # write the dqr results to a file for reference
@@ -320,7 +343,6 @@ class ProcessManager:
 
         # if the flag for all variables is set
         if "all" in variables:
-            nc2csv_logger.info("Variables = {}".format(variables))
             # populate a temp var_list with all variables
             nc2csv_logger.info("populating list will all variables in current netCDF file")
             variables.remove("all")

netcdf2ascii

@@ -55,7 +55,6 @@ sys.path.insert(1, "/apps/adc/retrievals/ARM-utils/packages")
 # FIXME: remove these if you don't have them
 try:
     import comms
-    from config import *
 except Exception:
     print("Could not import comms/config. No monitoring messages will be sent.")
 
@@ -175,10 +174,10 @@ def netcdf2ascii(main_args):
             proc_type = "netcdf"
             exit_code = nc2csv(main_args)
             print("exit({})".format(exit_code))
-        try:
-            comms.slack_send_direct(success_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
-        except Exception:
-            print("No monitoring message sent.")
+        # try:
+        #     comms.slack_send_direct(success_msg.format(exit_code, proc_type, main_args)) #FIXME: remove if unimported
+        # except Exception:
+        #     print("No monitoring message sent.")
         exit(exit_code)
     except Exception as e:
         print("ERROR: {}".format(e))

packages/comms.py

+import os
+from config import *
+
+def slack_send_direct(message):
+    from slackclient import SlackClient
+
+    slack_token = SLACK_ARM_TOKEN
+    sc = SlackClient(slack_token)
+
+    api_call = sc.api_call("im.list")
+
+    if api_call.get('ok'):
+        for im in api_call.get('ims'):
+            if im.get('user') == SLACK_USER_ID and im.get("is_im") == True:
+                im_channel = im.get("id")
+                sc.api_call("chat.postMessage",
+                            channel=im_channel,
+                            text=message,
+                            user=SLACK_USER_ID)
+
+def send_email(subject:str, message:str, recipients:list):
+    try:
+        import smtplib
+        from email.mime.text import MIMEText
+        executable = os.path.basename(__file__)
+        host = os.uname().__getattribute__('nodename')
+        sender = '{}@{}'.format(executable, host)
+
+        stars = 100*"*"
+        msg = "Sent from {0}. Variable extraction.\n\n" \
+              "{2}\n\t{1}\n{2}".format(host, message, stars)
+        msg = MIMEText(msg)
+
+        msg['Subject'] = "*** {} ***".format(subject)
+        msg['From'] = sender
+        msg['To'] = ", ".join(recipients)
+        s = smtplib.SMTP('localhost')
+        s.sendmail(sender, recipients, msg.as_string())
+    except Exception:
+        print("Email not sent to {}".format(", ".join(recipients)))
+
+def main():
+    print('** Test some stuff\nSend a slack message to Michael')
+    slack_send_direct("test message from python")
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file