import ast
from builtins import range
import copy
import csv
from functools import cmp_to_key
import json
import logging
import mimetypes
import os
import re
import socket
import sys
import subprocess
import sendemail_common

if sys.version_info >= (3, 0):
    from io import (BytesIO, TextIOWrapper)
    import urllib.parse as urllib
else:
    from cStringIO import StringIO
    BytesIO = StringIO
    import urllib
import time
import splunk.clilib.cli_common as cli_common

from email import encoders, utils
from email.header import Header
from email.mime.application import MIMEApplication
from email.mime.base import MIMEBase
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

import splunk.safe_lxml_etree as et
from mako import template
import mako.filters as filters
import splunk.entity as entity
import splunk.Intersplunk
import splunk.mining.dcutils as dcu
import splunk.pdf.pdfgen_utils as pu
import splunk.search as search
import splunk.secure_smtplib as secure_smtplib
import splunk.ssl_context as ssl_context
from splunk.rest import simpleRequest
from splunk.saved import savedSearchJSONIsAlert
from splunk.util import normalizeBoolean, unicode, format_local_tzoffset

PDF_REPORT_SERVER_TIMEOUT = 600
PDFGEN_SIMPLE_REQUEST_TIMEOUT = 3600
EMAIL_DELIM = re.compile(r'\s*[,;]\s*')
EMAIL_FORMAT = re.compile(r'^[^\s@]+@[^\s@,;]+$')
CHARSET = "UTF-8"
IMPORTANCE_MAP = {
    "highest": "1 (Highest)",
    "high"   : "2 (High)",
    "low"    : "4 (Low)",
    "lowest" : "5 (Lowest)",
    "1" : "1 (Highest)",
    "2" : "2 (High)",
    "4" : "4 (Low)",
    "5" : "5 (Lowest)"
}

logger = dcu.getLogger()

class SendEmailException(Exception):
    pass

def unquote(val):
    if val is not None and len(val) > 1 and val.startswith('"') and val.endswith('"'):
       return val[1:-1]
    return val

def numsort(x, y):
    if y[1] > x[1]:
        return -1
    elif x[1] > y[1]:
        return 1
    else:
        return 0

def renderTime(results):
   for result in results:
      if "_time" in result:
         try:
              result["_time"] = time.ctime(float(result["_time"]))
         except:
              pass
def getEmailList(email):
    tmpList = EMAIL_DELIM.split(email)
    emailList = []
    for email_item in tmpList:
        if re.search(EMAIL_FORMAT, email_item):
            emailList.append(email_item)
        else:
            logger.warn("Invalid email recipient=%s, remove recipient=%s from recipients."
                        % (email_item, email_item))
    return emailList

# SPL-207377 - Normalize email address separators
def normalizeEmail(email, field, recipients):
    emailList = getEmailList(email[field])
    recipients.extend(emailList)
    stripped = ','.join([str(elem) for elem in emailList])
    email.replace_header(field, stripped)

def mail(email, argvals, ssContent, sessionKey):

    sender     = email['From']
    use_ssl    = normalizeBoolean(ssContent.get('action.email.use_ssl', False))
    use_tls    = normalizeBoolean(ssContent.get('action.email.use_tls', False))
    server     = ssContent.get('action.email.mailserver', 'localhost')

    username   = argvals.get('username', '')
    password   = argvals.get('password', '')
    recipients = []

    if email['To']:
        normalizeEmail(email, 'To', recipients)
    if email['Cc']:
        normalizeEmail(email, 'Cc', recipients)
    if email['Bcc']:
        recipients.extend(getEmailList(email['Bcc']))
        del email['Bcc']    # delete bcc from header after adding to recipients

    # Clear leading / trailing whitespace from recipients
    recipients = [r.strip() for r in recipients]
    validRecipients = []
    if ssContent.get('action.email.allowedDomainList') != "" and ssContent.get('action.email.allowedDomainList') != None:
        domains = []
        domains.extend(EMAIL_DELIM.split(ssContent['action.email.allowedDomainList']))
        domains = [d.strip() for d in domains]
        domains = [d.lower() for d in domains]
        recipients = [r.lower() for r in recipients]
        for recipient in recipients:
            dom = recipient.partition("@")[2]
            if not dom in domains:
                logger.error("For subject=%s, email recipient=%s is not among the alowedDomainList=%s in alert_actions.conf file. Removing it from the recipients list."
                             % (ssContent.get('action.email.subject'), recipient, ssContent.get('action.email.allowedDomainList')))
            else:
                validRecipients.append(recipient)
    else:
        validRecipients = recipients


    mail_log_msg = 'Sending email. subject="%s", encoded_subject="%s", results_link="%s", recipients="%s", server="%s"' % (
        ssContent.get('action.email.subject'),
        email['Subject'],
        ssContent.get('results_link'),
        str(validRecipients),
        str(server)
    )
    try:
        # make sure the sender is a valid email address
        if sender.find("@") == -1:
            sender = sender + '@' + socket.gethostname()
            if sender.endswith("@"):
              sender = sender + 'localhost'

        # setup the Open SSL Context
        sslHelper = ssl_context.SSLHelper()
        serverConfJSON = sslHelper.getServerSettings(sessionKey)
        # Pass in settings from alert_actions.conf into context
        ctx = sslHelper.createSSLContextFromSettings(
            sslConfJSON=ssContent.get('alertActions'),
            serverConfJSON=serverConfJSON,
            isClientContext=True)

        # send the mail
        if not use_ssl:
            smtp = secure_smtplib.SecureSMTP(host=server)
        else:
            smtp = secure_smtplib.SecureSMTP_SSL(host=server, sslContext=ctx)

        if use_tls:
            smtp.starttls(ctx)
        if len(username) > 0 and password is not None and len(password) >0:
            smtp.login(username, password)
        if ssContent.get('action.email.allowedDomainList') != "" and ssContent.get('action.email.allowedDomainList') != None:
            if len(validRecipients) == 0:
                raise Exception("The email domains of the recipients are not among those on the allowed domain list.")
        
        # Installed SMTP daemon may not support UTF8 as well as
        # it may throw other exceptions also.
        error = sendemail_common.sendEmailWithUTF8(smtp, sender, validRecipients, email.as_string())

        if (error is not None):
            logger.debug('send mail with utf8 failed. retrying without utf8 option. Error: %s', str(error))
            smtp.sendmail(sender, validRecipients, email.as_string())

        smtp.quit()

        if ssContent.get('action.email.allowedDomainList') != "" and ssContent.get('action.email.allowedDomainList') != None:
            if validRecipients != recipients:
                raise Exception("Not all of the recipient email domains are on the allowed domain list. Sending emails only to %s" % str(validRecipients))

        logger.info(mail_log_msg)

    except Exception as e:
        logger.error(mail_log_msg)
        raise

def sendEmail(results, settings, keywords, argvals):
    for key in argvals:
        if key != 'ssname':
            argvals[key] =  unquote(argvals[key])
    

    namespace       = settings['namespace']
    owner           = settings['owner']
    sessionKey      = settings['sessionKey']
    sid             = settings['sid']
    ssname          = argvals.get('ssname')
    isScheduledView = False
    is_stream_malert = normalizeBoolean(argvals.get('is_stream_malert'))

    # Use this way to GET the correct alert_actions.conf under corresponding app.
    entityClass = ['alerts', 'alert_actions', 'email']
    uri = entity.buildEndpoint(
        entityClass,
        namespace=namespace,
        owner=owner
    )
    responseHeaders, responseBody = simpleRequest(uri, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)
    alertEmail = json.loads(responseBody)
    alertContent = alertEmail['entry'][0]['content']
    alertContent['allowedDomainList'] = alertContent['allowedDomainList'].strip()

    if ssname: 
        # populate content with savedsearch
        if '_ScheduledView__' in ssname or argvals.get('pdfview'):
            if '_ScheduledView__' in ssname:
                ssname = ssname.replace('_ScheduledView__', '')
            else:
                ssname = argvals.get('pdfview')

            uri = entity.buildEndpoint(
                [ 'scheduled', 'views', ssname], 
                namespace=namespace, 
                owner=owner
            )
            isScheduledView = True

        else:
            if is_stream_malert:
                entityClass = ['alerts', 'metric_alerts']
            else:
                entityClass = ['saved', 'searches']
            entityClass.append(ssname)
            uri = entity.buildEndpoint(
                entityClass,
                namespace=namespace, 
                owner=owner
            )

        responseHeaders, responseBody = simpleRequest(uri, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)

        savedSearch = json.loads(responseBody)
        ssContent = savedSearch['entry'][0]['content']

        # set type of saved search
        if isScheduledView: 
            ssContent['type']  = 'view'
        elif is_stream_malert or savedSearchJSONIsAlert(savedSearch):
            ssContent['type']  = 'alert'
        else:
            ssContent['type']  = 'report'

        # remap needed attributes that are not already on the content
        ssContent['name']                 = ssname
        ssContent['app']                  = savedSearch['entry'][0]['acl'].get('app')
        ssContent['owner']                = savedSearch['entry'][0]['acl'].get('owner')

        # The footer.text key will always exist for type alert and report. 
        # It may not exist for scheduled views created before 6.1 therefore the schedule view default footer.text 
        # should be set if the key does not exist.
        # This can be removed once migration has happened to ensure scheduled views always have the footer.text attribute
        ssContent['action.email.footer.text'] = ssContent.get('action.email.footer.text', "If you believe you've received this email in error, please see your Splunk administrator.\r\n\r\nsplunk>")

        # The message key will always exist for type alert and report. 
        # It may not exist for scheduled views created before 6.1 therefore the schedule view default message 
        # should be set if the key does not exist.
        # This can be removed once migration has happened to ensure scheduled views always have the message.view attribute
        ssContent['action.email.message'] = ssContent.get('action.email.message.' + ssContent.get('type'), 'A PDF was generated for $name$')
        if normalizeBoolean(ssContent.get('action.email.useNSSubject', False)):
            ssContent['action.email.subject'] = ssContent['action.email.subject.' + ssContent.get('type')]

        # prior to 6.1 the results link was sent as the argval sslink, must check both results_link
        # and sslink for backwards compatibility
        ssContent['results_link'] = argvals.get('results_link', argvals.get('sslink', ''))
        if normalizeBoolean(ssContent['results_link']) and normalizeBoolean(ssContent['type']):
            split_results_path = urllib.splitquery(ssContent.get('results_link'))[0].split('/')
            view_path = '/'.join(split_results_path[:-1]) + '/'
            ssType = ssContent.get('type')
            useGoLink = False
            if '@go' in split_results_path:
                useGoLink = True
            if ssType == 'alert':
                if is_stream_malert:
                    # SPL-174186
                    # Note: this will generate link like this:  http://<host:webport>/en-US/app/search/analysis_workspace?s=/servicesNS/<user>/<app>/alerts/metric_alerts/<alert_name>
                    # but the current MAW working link is this: http://<host:webport>/en-US/app/search/analysis_workspace?s=/servicesNS/<user>/<app>/saved/searches/<alert_name>
                    # If MAW UI can create new links for stream alert, this python code will work out of the box. Otherwise, we need to fix it at here.
                    ssContent['view_link'] = view_path + 'analytics_workspace?' + urllib.urlencode({'s': savedSearch['entry'][0]['links'].get('alternate')})
                elif useGoLink:
                    ssContent['view_link'] = view_path + '@go?' + urllib.urlencode({'s': savedSearch['entry'][0]['links'].get('alternate'), 'dispatch_view': 'alert'})
                else:
                    ssContent['view_link'] = view_path + 'alert?' + urllib.urlencode({'s': savedSearch['entry'][0]['links'].get('alternate')})
            elif ssType == 'report':
                if useGoLink:
                    ssContent['view_link'] = view_path + '@go?' + urllib.urlencode({'s': savedSearch['entry'][0]['links'].get('alternate'), 'sid': sid, 'dispatch_view': 'report'})
                else:
                    ssContent['view_link'] = view_path + 'report?' + urllib.urlencode({'s': savedSearch['entry'][0]['links'].get('alternate'), 'sid': sid})
            elif ssType == 'view':
                ssContent['view_link'] = view_path + ssContent['name']
            else:
                ssContent['view_link'] = view_path + 'search'
    else:
        if is_stream_malert:
            entityClass = ['alerts', 'metric_alerts']
        else:
            entityClass = ['saved', 'searches']
        entityClass.append('_new')
        #assumes that if no ssname then called from searchbar or test email
        uri = entity.buildEndpoint(
                entityClass,
                namespace=namespace,
                owner=owner
            )

        responseHeaders, responseBody = simpleRequest(uri, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)

        savedSearch = json.loads(responseBody)
        ssContent = savedSearch['entry'][0]['content']
        searchCommandSSContent = {
            'type': 'searchCommand',
            'view_link': '',
            'action.email.subject': 'Splunk Results'
        }
        ssContent.update(searchCommandSSContent)

        if normalizeBoolean(argvals.get('sendtestemail')):
            ssContent['type'] = 'view'
            if argvals.get('pdfview'):
                ssContent['name'] = argvals.get('pdfview')
                isScheduledView = True
            if not argvals.get('message'):
                ssContent['action.email.message'] = 'Search results attached.'

    ssContent['trigger_date'] = None
    ssContent['trigger_timeHMS'] = None
    ssContent['trigger_time'] = argvals.get('trigger_time')
    if normalizeBoolean(ssContent['trigger_time']):
        try:
            triggerSeconds = time.localtime(float(ssContent['trigger_time']))
            ssContent['trigger_date'] = time.strftime("%B %d, %Y", triggerSeconds)
            # %Z is deprecated, it does not work consistently between OSes
            ssContent['trigger_timeHMS'] = time.strftime("%H:%M:%S ", triggerSeconds) + format_local_tzoffset()
        except Exception as e:
            logger.error(e)

    # layer in arg vals
    if argvals.get('to'):
        ssContent['action.email.to'] = argvals.get('to')
    if argvals.get('bcc'):
        ssContent['action.email.bcc'] = argvals.get('bcc')
    if argvals.get('cc'):
        ssContent['action.email.cc'] = argvals.get('cc')
    if argvals.get('format'):
        ssContent['action.email.format'] = argvals.get('format')
    if argvals.get('from'):
        ssContent['action.email.from'] = argvals.get('from')
    if argvals.get('inline'):
        ssContent['action.email.inline'] = normalizeBoolean(argvals.get('inline'))
    if argvals.get('sendresults'):
        ssContent['action.email.sendresults'] = normalizeBoolean(argvals.get('sendresults'))
    if argvals.get('sendpdf'):
        ssContent['action.email.sendpdf'] = normalizeBoolean(argvals.get('sendpdf'))
    if argvals.get('sendpng'):
        ssContent['action.email.sendpng'] = normalizeBoolean(argvals.get('sendpng'))
    if argvals.get('pdfview'):
        ssContent['action.email.pdfview'] = argvals.get('pdfview')
    if argvals.get('papersize') or ssContent.get('action.email.papersize'):
        ssContent['action.email.reportPaperSize'] = argvals.get('papersize') or ssContent.get('action.email.papersize')
    if argvals.get('paperorientation') or ssContent.get('action.email.paperorientation'):
        ssContent['action.email.reportPaperOrientation'] = argvals.get('paperorientation') or ssContent.get('action.email.paperorientation')
    if argvals.get('sendcsv'):
        ssContent['action.email.sendcsv'] = normalizeBoolean(argvals.get('sendcsv'))
    if argvals.get('allow_empty_attachment'):
        ssContent['action.email.allow_empty_attachment'] = normalizeBoolean(argvals.get('allow_empty_attachment'))
    if argvals.get('pdf.logo_path'):
        ssContent['action.email.pdf.logo_path'] = argvals.get('pdf.logo_path')
    if argvals.get('escapeCSVNewline'):
        ssContent['action.email.escapeCSVNewline'] = normalizeBoolean(argvals.get('escapeCSVNewline'))
    ## if 'server' in arg is different than the one set in alert_action.conf, don't use default credentials.
    ## SPL-135659
    setDefaultUserCredentials = 1
    if argvals.get('server'):
        alert_action_server = ssContent.get('action.email.mailserver', 'localhost')
        if alertContent.get('allowedDomainList') == "":
            ssContent['action.email.mailserver'] = argvals.get('server')
        else:
            logger.warn("When 'allowedDomainList' is setup, 'server' argument is not accepted in sendemail command. "
                        "The 'server' value is obtained from 'mailserver'=%s in alert_actions.conf." % alertContent.get('mailserver'))
        assigned_server = ssContent.get('action.email.mailserver', 'localhost')
        if str(alert_action_server) != str(assigned_server):
            setDefaultUserCredentials = 0
    if argvals.get('subject'):
        ssContent['action.email.subject'] = argvals.get('subject')
    if argvals.get('footer'):
        ssContent['action.email.footer.text'] = argvals.get('footer')
    if argvals.get('width_sort_columns'):
        ssContent['action.email.width_sort_columns'] = normalizeBoolean(argvals.get('width_sort_columns'))
    if argvals.get('message'):
        ssContent['action.email.message'] = argvals.get('message')
    else:
        if ssContent['type'] == 'searchCommand':
            # set default message for searchCommand emails
            if ssContent['action.email.sendresults'] or ssContent['action.email.sendpdf'] or ssContent['action.email.sendcsv']:
                if ssContent.get('action.email.inline') and not(ssContent.get('action.email.sendpdf') or ssContent.get('action.email.sendcsv')):
                    ssContent['action.email.message'] = 'Search results.'
                else:
                    ssContent['action.email.message'] = 'Search results attached.'
            else:
                ssContent['action.email.message'] = 'Search complete.'
    if argvals.get('priority'):
        ssContent['action.email.priority'] = argvals.get('priority')
    if argvals.get('use_ssl'):
        ssContent['action.email.use_ssl'] = normalizeBoolean(argvals.get('use_ssl'))
    if argvals.get('use_tls'):
        ssContent['action.email.use_tls'] = normalizeBoolean(argvals.get('use_tls'))
    if argvals.get('content_type'):
        ssContent['action.email.content_type'] = argvals.get('content_type')
    # parse the result id for per_result_alert
    if argvals.get('results_file'):
        dirname, fname = os.path.split(argvals.get('results_file'))
        if dirname.endswith('per_result_alert'):
            results_ids = re.findall(r'^tmp_(\d+)\.', fname)
            if results_ids and len(results_ids) > 0:
                ssContent['action.per_alert_result_id'] = results_ids[0]
                logger.debug('Parsed action.per_alert_result_id=%s' % (ssContent['action.per_alert_result_id']))
            else:
                logger.warn('Cannot parse results_id=%s for per result alert' % (fname))
    ssContent['graceful'] = normalizeBoolean(argvals.get('graceful', 0))

    #if there is no results_link then do not incude it
    if is_stream_malert or not normalizeBoolean(ssContent.get('results_link')):
        ssContent['action.email.include.results_link'] = False

    #need for backwards compatibility
    format = ssContent.get('action.email.format')

    if format == 'html' or format == 'plain':
        ssContent['action.email.format'] = 'table'
        ssContent['action.email.content_type'] = format
    elif format == 'text':
        ssContent['action.email.content_type'] = 'plain'
        ssContent['action.email.format'] = 'table'
    elif format == 'pdf':
        ssContent['action.email.content_type'] = 'html'
        ssContent['action.email.format'] = 'table'

    #fetch server info
    uriToServerInfo = entity.buildEndpoint(['server', 'info'])
    serverInfoHeaders, serverInfoBody = simpleRequest(uriToServerInfo, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)
    
    serverInfo        = json.loads(serverInfoBody)
    serverInfoContent = serverInfo['entry'][0]['content']

    #fetch job info
    jobResponseHeaders = {} 
    jobResponseBody = { 
        'entry': [
            {
                'content': {}
            }
        ]
    }
    if sid: 
        uriToJob = entity.buildEndpoint(
            [
                'search', 
                'jobs', 
                sid
            ], 
            namespace=namespace, 
            owner=owner
        )
        jobResponseHeaders, jobResponseBody = simpleRequest(uriToJob, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)
        searchJob         = json.loads(jobResponseBody)
        jobContent        = searchJob['entry'][0]['content']
    else:
        jobContent        = jobResponseBody['entry'][0]['content']

    #fetch view info
    viewResponseHeaders = {}
    viewResponseBody = {
        'entry': [
            {
                'content': {}
            }
        ]
    }

    if isScheduledView:
        uriToView = entity.buildEndpoint(
            [
                'data',
                'ui',
                'views',
                ssContent['name']
            ],
            namespace=namespace,
            owner=owner
        )
        viewResponseHeaders, viewResponseBody = simpleRequest(uriToView, method='GET', getargs={'output_mode':'json'}, sessionKey=sessionKey)
        viewResponseBody = json.loads(viewResponseBody)

    viewContent = viewResponseBody['entry'][0]['content']

    if ssContent.get('action.email.allowedDomainList'):
        ssContent['action.email.allowedDomainList'] = ssContent['action.email.allowedDomainList'].strip()
        if ssContent.get('action.email.allowedDomainList') != alertContent.get('allowedDomainList'):
            ssContent['action.email.allowedDomainList'] = alertContent['allowedDomainList']
            logger.warn("For alert=%s, the 'allowedDomainList' value is always obtained from alert_actions.conf."
                        "The allowedDomainList=%s" % (ssname, alertContent.get('allowedDomainList')))

    if alertContent.get('allowedDomainList') != "":
        if ssContent.get('action.email.mailserver') != alertContent.get('mailserver'):
            ssContent['action.email.mailserver'] = alertContent['mailserver']
            logger.warn("For alert=%s, if a 'allowedDomainList' is specified, it uses the 'mailserver'=%s in alert_actions.conf." %
                        (ssname, ssContent.get('action.email.mailserver')))

    valuesForTemplate = buildSafeMergedValues(ssContent, results, serverInfoContent, jobContent, viewContent, argvals.get('results_file'))
    realize(valuesForTemplate, ssContent, sessionKey, namespace, owner, argvals)
    #Creation of the email object that is to be populated in the build
    #prefixed methods.
    content_type = ssContent.get('action.email.content_type')
    isHtmlBasedContentType = content_type in ['html', 'html_pdf']
    inline_studio_png = normalizeBoolean(ssContent.get('action.email.sendpng', False))

    if isHtmlBasedContentType or inline_studio_png:
        email = MIMEMultipart('mixed')
        email.preamble = 'This is a multi-part message in MIME format.'
        emailBody = MIMEMultipart('alternative')
        email.attach(emailBody)
    else:
        email = emailBody = MIMEMultipart()

    #make time user readable
    resultsWithRenderedTime = copy.deepcopy(results)
    renderTime(resultsWithRenderedTime)

    #potentially mutate argvals if username/password are empty
    if setDefaultUserCredentials:
        setUserCredentials(argvals, settings)

    #build all the different email components
    jobCount = getJobCount(jobContent)
    #attachments must be built before body so body can inlclude errors cause by attachments but
    #must actually be attached after body
    toAttach = buildAttachments(settings, ssContent, resultsWithRenderedTime, email, jobCount, argvals)
    buildPlainTextBody(ssContent, resultsWithRenderedTime, settings, emailBody, jobCount)

    if isHtmlBasedContentType or inline_studio_png:
        buildHTMLBody(ssContent, resultsWithRenderedTime, settings, emailBody, email, jobCount, argvals)
    buildHeaders(argvals, ssContent, email, sid, serverInfoContent)
    #attach attachments
    for attachment in toAttach:
        email.attach(attachment)

    try:
        mail(email, argvals, ssContent, sessionKey)
    except Exception as e:
        errorMessage = str(e) + ' while sending mail to: ' + ssContent.get("action.email.to")
        logger.error(errorMessage)
        results = dcu.getErrorResults(results, ssContent['graceful'], errorMessage)
    return results

def buildSafeMergedValues(ssContent, results, serverInfoContent, jobContent, viewContent, results_file):
    mergedObject = {}
     #namespace the keys
    for key, value in jobContent.items():
        mergedObject['token.job.'+key] = value
    mergedObject['token.search_id'] = jobContent.get('sid')

    for key, value in ssContent.items():
        mergedObject['token.'+key] = value

    mergedObject['token.name'] = ssContent.get('name')
    mergedObject['token.app'] = ssContent.get('app')
    mergedObject['token.owner'] = ssContent.get('owner')

    for key, value in serverInfoContent.items():
        mergedObject['token.server.'+key] = value

    if viewContent:
        mergedObject['token.dashboard.title'] = mergedObject['token.dashboard.label'] = viewContent.get('label') if viewContent.get('label') else mergedObject['token.name']
        mergedObject['token.dashboard.id'] = mergedObject['token.name']
        mergedObject['token.dashboard.description'] = getDescriptionFromXml(viewContent.get('eai:data'))

    if results:
        r = results[0]
        for k in r:
            if k.startswith('__mv_'):
                continue
            if isinstance(r[k], list):
                mergedObject['token.result.'+k] = ' '.join(map(str,r[k]))
            else:
                mergedObject['token.result.'+k] = r[k]
        mergedObject['token.results.count'] = len(results)
    mergedObject['token.results.url'] = ssContent.get('results_link')
    mergedObject['token.results.file'] = results_file

    return mergedObject

def getDescriptionFromXml(xmlString):
    if xmlString:
        dashboardNode = et.fromstring(unicode(xmlString).encode('utf-8'))
        return dashboardNode.findtext('./description')
    else:
        return None

def realize(valuesForTemplate, ssContent, sessionKey, namespace, owner, argvals):
    stringsForPost = {}
    if ssContent.get('action.email.message'):
        stringsForPost['action.email.message'] = ssContent['action.email.message']

    if ssContent.get('action.email.cc'):
        stringsForPost['action.email.cc'] = ssContent['action.email.cc']

    if ssContent.get('action.email.bcc'):
        stringsForPost['action.email.bcc'] = ssContent['action.email.bcc']

    if ssContent.get('action.email.to'):
        stringsForPost['action.email.to'] = ssContent['action.email.to']

    if ssContent.get('action.email.subject'):
        stringsForPost['action.email.subject'] = ssContent['action.email.subject']

    if ssContent.get('action.email.footer.text'):
        # SPL-144752: allow footer to have no value at all if user choose to not have it.
        # This can be done by setting footer = " " (note the space in between)
        if len(ssContent['action.email.footer.text'].strip()) > 0:
            stringsForPost['action.email.footer.text'] = ssContent['action.email.footer.text']

    realizeURI = entity.buildEndpoint([ 'template', 'realize' ])

    postargs = valuesForTemplate
    postargs['output_mode'] = 'json'
    postargs['conf.recurse'] = 0
    try:
        for key, value in stringsForPost.items():
            if len(value.strip()) == 0:
                logger.warning('Token substitution may fail due to key:%s contains only whitespaces' % key)
            postargs['name'] = value
            headers, body = simpleRequest(
                realizeURI, 
                method='POST', 
                postargs=postargs, 
                sessionKey=sessionKey
            )
            body = json.loads(body)
            ssContent[key] = body['entry'][0]['content']['eai:data']
    except Exception as e:
        logger.error(e)
        # SPL-96721: email subject didn't get replaced, reset it to ssname
        if ssContent.get('action.email.subject') == stringsForPost.get('action.email.subject'):
            ssContent['action.email.subject'] = "Splunk Alert:"+argvals['ssname']
            ssContent['action.email.message'] = ssContent['action.email.message'] + "\n\nNOTE: The dynamic substitution of the email subject failed because of failed token substitution. A generic subject has been used instead. Please check splunkd.log for additional details."

def setUserCredentials(argvals, settings):
    username    = argvals.get("username" , "")
    password    = argvals.get("password" , "")

    # fetch credentials from the endpoint if none are supplied or password is encrypted
    if (len(username) == 0 and len(password) == 0) or (password.startswith('$1$') or password.startswith('$7$')) :
        namespace  = settings.get("namespace", None)
        sessionKey = settings['sessionKey']

        username, password = getCredentials(sessionKey, namespace)
         
        argvals['username'] = username
        argvals['password'] = password

def getJobCount(jobContent):
    if jobContent.get('statusBuckets') == 0 or (normalizeBoolean(jobContent.get('reportSearch')) and not re.match('sendemail', jobContent.get('reportSearch'))):
        return jobContent.get('resultCount')
    else:
        return jobContent.get('eventCount')

# takes header, html, text
def buildHeaders(argvals, ssContent, email, sid, serverInfoContent):

    sender  = ssContent.get("action.email.from", "splunk")
    to      = ssContent.get("action.email.to")
    cc      = ssContent.get("action.email.cc")
    bcc     = ssContent.get("action.email.bcc")
    subject = ssContent.get("action.email.subject")
    priority = ssContent.get("action.email.priority", '')

    # use the Header object to specify UTF-8 msg headers, such as subject, to, cc etc
    email['Subject'] = Header(subject, CHARSET)

    recipients = []
    if to:
        email['To'] = to
    if sender:
        email['From'] = sender
    if cc:
        email['Cc'] = cc
    if bcc:
        email['Bcc'] = bcc

    email['Date'] = utils.formatdate(localtime=True)

    if priority:
        # look up better name
        val = IMPORTANCE_MAP.get(priority.lower(), '')
        # unknown value, use value user supplied
        if not val:
            val = priority
        email['X-Priority'] = val

    # trace info
    if ssContent.get('name'):
        email['X-Splunk-Name'] = ssContent.get('name')
    if ssContent.get('owner'):
        email['X-Splunk-Owner'] = ssContent.get('owner')
    if ssContent.get('app'):
        email['X-Splunk-App'] = ssContent.get('app')
    email['X-Splunk-SID'] = sid
    email['X-Splunk-ServerName'] = serverInfoContent.get('serverName')
    email['X-Splunk-Version'] = serverInfoContent.get('version')
    email['X-Splunk-Build'] = serverInfoContent.get('build')


def buildHTMLBody(ssContent, results, settings, emailbody, email, jobCount, argvals):
    messageHTML  = re.sub(r'\r\n?|\\r\\n?|\n|\\n', '<br />\r\n', htmlMessageTemplate().render(msg=ssContent.get('action.email.message')))
    body_png = ''
    resultsHTML = ''
    metaDataHTML = ''
    errorHTML = ''
    if ssContent['type'] == 'view':
        if normalizeBoolean(ssContent.get('action.email.include.view_link')) and ssContent.get('view_link'):
            metaDataHTML = htmlMetaDataViewTemplate().render(
                view_link=ssContent.get('view_link')
            )
        if should_inline_png(ssContent, argvals):
            try:
                name = ssContent.get('action.email.pdfview', '')
                name_png = name + '.png'
                body_png = inline_image_template().render(cid_name=name_png, name=name)
            except ValueError as e:
                logger.error('Failed to inline PNG due to {}'.format(e))
    else:
        if ssContent['type'] != 'searchCommand':
            metaDataHTML = htmlMetaDataSSTemplate().render(
                jobcount=jobCount,
                results_link=ssContent.get('results_link'),
                include_results_link=normalizeBoolean(ssContent.get('action.email.include.results_link')),
                view_link=ssContent.get('view_link'),
                include_view_link=normalizeBoolean(ssContent.get('action.email.include.view_link')),
                name=ssContent.get('name'),
                include_search=normalizeBoolean(ssContent.get('action.email.include.search')),
                ssquery=ssContent.get('search'),
                include_smaDefinition=normalizeBoolean(ssContent.get('action.email.include.smaDefinition')),
                smaIndexes=ssContent.get('metric_indexes'),
                smaFilter=ssContent.get('filter'),
                smaGroupby=ssContent.get('groupby'),
                smaCondition=ssContent.get('condition'),
                smaThreshold=ssContent.get('trigger.threshold'),
                smaSuppress=ssContent.get('trigger.suppress'),
                smaEvaluationPerGroup=ssContent.get('trigger.evaluation_per_group'),
                smaActionPerGroup=ssContent.get('trigger.action_per_group'),
                smaTriggerPrepare=ssContent.get('trigger.prepare'),
                alert_type=ssContent.get('alert_type'),
                include_trigger=normalizeBoolean(ssContent.get('action.email.include.trigger')),
                include_inline=normalizeBoolean(ssContent.get('action.email.inline')),
                include_trigger_time=normalizeBoolean(ssContent.get('action.email.include.trigger_time')),
                trigger_date=ssContent.get('trigger_date'),
                trigger_timeHMS=ssContent.get('trigger_timeHMS'),
                ssType=ssContent.get('type'),
            )
        # need to check aciton.email.sendresults for type searchCommand
        if normalizeBoolean(ssContent.get('action.email.inline')) and normalizeBoolean(
                ssContent.get('action.email.sendresults')):
            resultsHTML = htmlResultsTemplate().render(
                include_results_link=normalizeBoolean(ssContent.get('action.email.include.results_link')),
                results_link=ssContent.get('results_link'),
                truncated=normalizeBoolean(settings.get('truncated')),
                resultscount=len(results),
                jobcount=jobCount,
                hasjob=normalizeBoolean(settings.get('sid'))
            )
            format = ssContent.get('action.email.format')
            if format == 'table':
                resultsHTML += htmlTableTemplate().render(results=results)
            elif format == 'raw':
                resultsHTML += htmlRawTemplate().render(results=results)
            elif format == 'csv':
                resultsHTML += htmlCSVTemplate().render(results=results)

    footerHTML = htmlFooterTemplate().render(footer=ssContent.get('action.email.footer.text'), re=re, filters=filters)
    errorHTML = htmlErrorTemplate().render(errors=ssContent.get('errorArray'))
    wrapperHTML = htmlWrapperTemplate().render(body=messageHTML + body_png + metaDataHTML + errorHTML + resultsHTML + footerHTML)
    emailbody.attach(MIMEText(wrapperHTML, 'html', _charset=CHARSET))


def htmlWrapperTemplate():
    return template.Template('''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
    <head>
        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
    </head>
    <body style="font-size: 14px; font-family: helvetica, arial, sans-serif; padding: 20px 0; margin: 0; color: #333;">
        ${body}
    </body>
</html>
    ''')

def htmlMetaDataSSTemplate():
    return template.Template('''
<table cellpadding="0" cellspacing="0" border="0" class="summary" style="margin: 20px;">
    <tbody>
        % if view_link and name and include_view_link:
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">${ssType.capitalize()}:</th><td style="padding: 0 0 10px 0;"><a href="${view_link}" style=" text-decoration: none; margin: 0 40px 0 0; color: #006d9c;">${name|h}</a></td>
        </tr>
        % endif
        % if ssquery and include_search:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">Search String:</th><td style="padding: 0 0 10px 0;">${ssquery|h}</td>
        </tr>
        % endif
        % if smaIndexes and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">metric_indexes:</th><td style="padding: 0 0 10px 0;">${smaIndexes|h}</td>
        </tr>
        % endif
        % if smaFilter and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">filter:</th><td style="padding: 0 0 10px 0;">${smaFilter|h}</td>
        </tr>
        % endif
        % if smaGroupby and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">groupby:</th><td style="padding: 0 0 10px 0;">${smaGroupby|h}</td>
        </tr>
        % endif
        % if smaCondition and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">condition:</th><td style="padding: 0 0 10px 0;">${smaCondition|h}</td>
        </tr>
        % endif
        % if smaThreshold and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">trigger.threshold:</th><td style="padding: 0 0 10px 0;">${smaThreshold|h}</td>
        </tr>
        % endif
        % if smaSuppress and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">trigger.suppress:</th><td style="padding: 0 0 10px 0;">${smaSuppress|h}</td>
        </tr>
        % endif
        % if smaEvaluationPerGroup and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">trigger.evaluation_per_group:</th><td style="padding: 0 0 10px 0;">${smaEvaluationPerGroup|h}</td>
        </tr>
        % endif
        % if smaActionPerGroup and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">trigger.action_per_group:</th><td style="padding: 0 0 10px 0;">${smaActionPerGroup|h}</td>
        </tr>
        % endif
        % if smaTriggerPrepare and include_smaDefinition:    
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">trigger.prepare:</th><td style="padding: 0 0 10px 0;">${smaTriggerPrepare|h}</td>
        </tr>
        % endif
        % if include_trigger and name and alert_type and ssType == "alert":
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">Trigger:</th><td style="padding: 0 0 10px 0;">Saved Search [${name|h}]: ${alert_type}
                % if alert_type == "number of events":
                    (${jobcount})
                % endif
                </td>
        </tr>
        % endif
        % if include_trigger_time and trigger_timeHMS and trigger_date and ssType == "alert":
        <tr>
            <th style="font-weight: normal; text-align: left; padding: 0 20px 10px 0;">Trigger Time:</th><td style="padding: 0 0 10px 0;">${trigger_timeHMS} on ${trigger_date}.</td>
        </tr>
        % endif
    </tbody>
</table>
% if not include_inline:
    % if include_results_link:
    <a href="${results_link|h}" style=" text-decoration: none; margin: 0 20px; color: #006d9c;">View results</a>
    % endif
% endif
''')

def htmlMetaDataViewTemplate():
    return template.Template('''
 <p><a href="${view_link}" style=" text-decoration: none; margin: 20px 40px 0 20px; color: #006d9c;">View dashboard</a></p>
''')

def htmlErrorTemplate():
    return template.Template('''
% if errors:
    <div style="border-top: 1px solid #c3cbd4;"></div>
    % for error in errors:
        <p style="margin: 20px;">${error|h}</p>
    % endfor
% endif
''')

def htmlResultsTemplate():
    return template.Template('''
<div style="margin-top: 10px; padding-top: 20px; border-top: 1px solid #c3cbd4;"></div>
% if truncated:
    %if jobcount:
<p style="margin: 0 20px;">Only the first ${resultscount} of ${jobcount} results are included below.
    %else:
<p style="margin: 0 20px;">Search results in this email have been truncated. 
    %endif
    %if include_results_link:
<a href="${results_link|h}" style=" text-decoration: none; margin: 0 0; color: #006d9c;">View all results</a> in Splunk.</p>
    %else:
</p>
    %endif
% elif include_results_link:
<div style="margin: 0 20px;">
    <a href="${results_link|h}" style=" text-decoration: none; color: #006d9c;">View results in Splunk</a>
</div>
% endif
''')

def htmlMessageTemplate():
    return template.Template('<div style="margin: 0 20px;">${msg|h}</div>')

def inline_image_template():
    return template.Template('<img style="width: 100%;" src="cid:${cid_name}" alt="dashboard image for ${name}">')

def htmlFooterTemplate():
    return template.Template('''
<div style="margin-top: 10px; border-top: 1px solid #c3cbd4;"></div>
<% footerEscaped = filters.html_entities_escape(footer) %>
<% footerBreaks = re.sub(r'\\r\\n?|\\n', '<br>', footerEscaped) %>
<p style="margin: 20px; font-size: 11px; color: #999;">${footerBreaks}</p>
''')

def htmlTableTemplate():
    return template.Template('''
% if len(results) > 0:
<div style="margin:0">
    <div style="overflow: auto; width: 100%;">
        <table cellpadding="0" cellspacing="0" border="0" class="results" style="margin: 20px;">
            <tbody>
                <% cols = [] %>
                <tr>
                % for key,val in results[0].items():
                    % if not key.startswith("_") or key == "_raw" or key == "_time":
                        <% cols.append(key) %>
                        <th style="text-align: left; padding: 4px 8px; margin-bottom: 0px; border-bottom: 1px dotted #c3cbd4;">${key|h}</th>
                    % endif
                % endfor
                </tr>
                % for result in results:
                    <tr valign="top">
                    % for col in cols:
                        <td style="text-align: left; padding: 4px 8px; margin-top: 0px; margin-bottom: 0px; border-bottom: 1px dotted #c3cbd4;">
                            % if isinstance(result.get(col), list):
                                % for val in result.get(col):
                                    <pre style="font-family: helvetica, arial, sans-serif; white-space: pre-wrap; margin:0px;">${val|h}</pre>
                                % endfor
                            % else:
                                <pre style="font-family: helvetica, arial, sans-serif; white-space: pre-wrap; margin:0px;">${result.get(col)|h}</pre>
                            % endif
                        </td>
                    % endfor
                    </tr>
                % endfor
            </tbody>
        </table>
    </div>
</div>
% else:
        <div class="results" style="margin: 20px;">No results found.</div>
% endif
''')

def htmlRawTemplate():
    return template.Template('''
% if len(results) > 0:
    % if results[0].get("_raw"):
        <div style="margin: 20px;" class="events">
        % for result in results:
            <div class="event"style="border-bottom: 1px dotted #c3cbd4; padding: 5px 0; font-family: monospace; word-break: break-all;"><pre style="white-space: pre-wrap;">${result.get("_raw", "")|h}</pre></div>
        % endfor
        </div>
    % else:
        <div> The results contain no "_raw" field.  Please choose another inline format (csv or table).</div>
    % endif
% else:
    <div class="results" style="margin: 20px;">No results found.</div>
% endif
''')

def htmlCSVTemplate():
    return template.Template('''
% if len(results) > 0:
    <div style="margin: 20px;">
    <% cols = [] %>
    % for key,val in results[0].items():
        % if not key.startswith("_") or key == "_raw" or key == "_time":
            <% cols.append(key) %>
        % endif
    % endfor
${','.join(cols)|h}<br/>
    % for result in results:
        <% vals = [] %>
        % for col in cols:
            % if isinstance(result.get(col), list):
                <% vals.append(' '.join(map(str,result.get(col)))) %>
            % else:
                <% vals.append(result.get(col))%>
            % endif
        % endfor
${','.join(vals)|h}<br/>
    % endfor
    </div>
% else:
    <div class="results" style="margin: 20px;">No results found.</div>
% endif
''')

def buildPlainTextBody(ssContent, results, settings, email, jobCount):
    plainTextMsg = buildPlainTextMessage().render(msg=ssContent.get('action.email.message'))
    plainResults = ''
    plainTextMeta = ''
    plainError = ''
    if ssContent['type'] == 'view':
        if normalizeBoolean(ssContent.get('action.email.include.view_link')) and ssContent.get('view_link'):
            plainTextMeta = buildPlainTextViewMetaData().render(
                view_link=ssContent.get('view_link')
            )
        plainError = buildPlainTextError().render(errors=ssContent.get('errorArray'))
    else:
        if ssContent['type'] != 'searchCommand':
            plainTextMeta    = buildPlainTextSSMetaData().render(
                jobcount=jobCount,
                results_link=ssContent.get('results_link'),
                include_results_link=normalizeBoolean(ssContent.get('action.email.include.results_link')),
                view_link=ssContent.get('view_link'),
                include_view_link=normalizeBoolean(ssContent.get('action.email.include.view_link')),
                name=ssContent.get('name'),
                include_search=normalizeBoolean(ssContent.get('action.email.include.search')),
                ssquery=ssContent.get('search'),
                include_smaDefinition=normalizeBoolean(ssContent.get('action.email.include.smaDefinition')),
                smaIndexes=ssContent.get('metric_indexes'),
                smaFilter=ssContent.get('filter'),
                smaGroupby=ssContent.get('groupby'),
                smaCondition=ssContent.get('condition'),
                smaThreshold=ssContent.get('trigger.threshold'),
                smaSuppress=ssContent.get('trigger.suppress'),
                smaEvaluationPerGroup=ssContent.get('trigger.evaluation_per_group'),
                smaActionPerGroup=ssContent.get('trigger.action_per_group'),
                smaTriggerPrepare=ssContent.get('trigger.prepare'),
                alert_type=ssContent.get('alert_type'),
                include_trigger=normalizeBoolean(ssContent.get('action.email.include.trigger')),
                include_inline=normalizeBoolean(ssContent.get('action.email.inline')),
                include_trigger_time=normalizeBoolean(ssContent.get('action.email.include.trigger_time')),
                trigger_date=ssContent.get('trigger_date'),
                trigger_timeHMS=ssContent.get('trigger_timeHMS'),
                ssType=ssContent.get('type')
            )
        plainError = buildPlainTextError().render(errors=ssContent.get('errorArray'))
        # need to check aciton.email.sendresults for type searchCommand
        if normalizeBoolean(ssContent.get('action.email.inline')) and normalizeBoolean(ssContent.get('action.email.sendresults')):
            plainResults = plainResultsTemplate().render(
                include_results_link=normalizeBoolean(ssContent.get('action.email.include.results_link')),
                results_link=ssContent.get('results_link'),
                truncated=normalizeBoolean(settings.get('truncated')),
                resultscount=len(results),
                jobcount=jobCount,
                hasjob=normalizeBoolean(settings.get('sid'))
            )
            format = ssContent.get('action.email.format')
            if format == 'table':
                plainResults += plainTableTemplate(results, ssContent)
            elif format == 'raw':
                plainResults += plainRawTemplate().render(results=results)
            elif format == 'csv':
                plainResults += plainCSVTemplate(results)

    plainFooter = plainFooterTemplate().render(footer=ssContent.get('action.email.footer.text'))

    email.attach(MIMEText(plainTextMsg + plainTextMeta + plainError + plainResults + plainFooter, 'plain', _charset=CHARSET))

def buildPlainTextSSMetaData():
    return template.Template('''
 % if view_link and name and include_view_link:
${ssType.capitalize()} Title:      ${name}
${ssType.capitalize()} Location:   ${view_link}
% endif
% if ssquery and include_search:    
Search String:    ${ssquery}
% endif
% if smaIndexes and include_smaDefinition:    
metric_indexes:    ${smaIndexes}
% endif
% if smaFilter and include_smaDefinition:    
filter:    ${smaFilter}
% endif
% if smaGroupby and include_smaDefinition:    
groupby:    ${smaGroupby}
% endif
% if smaCondition and include_smaDefinition:    
condition:    ${smaCondition}
% endif
% if smaThreshold and include_smaDefinition:    
trigger.threshold:    ${smaThreshold}
% endif
% if smaSuppress and include_smaDefinition:    
trigger.suppress:    ${smaSuppress}
% endif
% if smaEvaluationPerGroup and include_smaDefinition:    
trigger.evaluation_per_group:    ${smaEvaluationPerGroup}
% endif
% if smaActionPerGroup and include_smaDefinition:    
trigger.action_per_group:    ${smaActionPerGroup}
% endif
% if smaTriggerPrepare and include_smaDefinition:    
trigger.prepare:    ${smaTriggerPrepare}
% endif
% if include_trigger and name and alert_type and ssType == "alert":
    % if alert_type == "number of events":
Trigger:          Saved Search [${name}]: ${alert_type} (${jobcount})
    % else:
Trigger:          Saved Search [${name}]: ${alert_type}
    %endif
% endif
% if include_trigger_time and trigger_timeHMS and trigger_date and ssType == "alert":
Trigger Time:     ${trigger_timeHMS} on ${trigger_date}.
% endif
% if not include_inline:
    % if include_results_link:
View results:     ${results_link}
    % endif
%endif
''')

def buildPlainTextViewMetaData():
    return template.Template('''

View dashboard:     ${view_link}

''')

def buildPlainTextMessage():
    return template.Template('${msg}')

def plainFooterTemplate():
    return template.Template('''
------------------------------------------------------------------------

${footer}
''')

def buildPlainTextError():
    return template.Template('''
% if errors:
------------------------------------------------------------------------
    % for error in errors:
${error}
    % endfor
% endif
''')

def plainResultsTemplate():
    return template.Template('''
------------------------------------------------------------------------
% if truncated:
    % if jobcount:
Only the first ${resultscount} of ${jobcount} results are included below.
    % else:
Search results in this email have been truncated.
    % endif
    % if include_results_link:
View all results in Splunk: ${results_link}
    % endif
% elif include_results_link:
View results in Splunk: ${results_link}
% endif
''')

# sort columns from shortest to largest
def getSortedColumns(results, width_sort_columns):
    if len(results) == 0:
        return []

    columnMaxLens = {}
    for result in results:
        for k,v in result.items():
            # ignore attributes that start with "_"
            if k.startswith("_") and k!="_raw" and k!="_time":
                continue
            if isinstance(v, list):
                v = getLongestString(v)

            newLen = len(str(v))
            oldMax = columnMaxLens.get(k, -1)
            
            #initialize the column width to the length of header (name)
            if oldMax == -1:
                columnMaxLens[k] = oldMax = len(k)
            if newLen > oldMax:
                columnMaxLens[k] = newLen

    colsAndCounts = []
    # sort columns iff asked to
    if width_sort_columns:
       colsAndCounts = sorted(columnMaxLens.items(), key=cmp_to_key(numsort))
    else:
       for k,v in results[0].items():
          if k in columnMaxLens:
             colsAndCounts.append([k, columnMaxLens[k]])

    return colsAndCounts

def getLongestString(valuesList):
    return max(map(str,valuesList), key=len)

def plainTableTemplate(results, ssContent):
    if len(results) > 0:
        width_sort_columns = normalizeBoolean(ssContent.get('action.email.width_sort_columns', True))
        columnMaxLens = getSortedColumns(results, width_sort_columns)
        text = "\n"
        space = " "*8
        
        # output column names
        for col, maxlen in columnMaxLens:
            val = col
            padsize = maxlen - len(val)
            text += val + ' '*padsize + space

        rowBorder = "-"*len(text)
        text += "\n" + rowBorder + "\n"

        # output each result's values
        for result in results:
            # maximum number of multivalue fields in any column for a given result
            maxFields = -1
            for col, maxlen in columnMaxLens:
                val = result.get(col, "")
                if isinstance(val, list):
                    maxFields = len(val) if len(val) > maxFields else maxFields
                    val = val[0]
                padsize = maxlen - len(val)
                # left justify ALL the columns
                text += val + ' '*padsize + space
            text += "\n"

            # add remaining multivalue items, if any
            if maxFields > -1:
                for row in range(1, maxFields):
                    for col, maxlen in columnMaxLens:
                        val = result.get(col, "")
                        if isinstance(val, list) and len(val) > row:
                            val = str(val[row])
                        else:
                            # no value in this row if it isn't a list
                            val = ""
                        padsize = maxlen - len(val)
                        text += val + ' '*padsize + space
                    text += "\n"

            text += rowBorder + "\n"

    else:
        text = "No results found."
    return text

def plainCSVTemplate(results):
    text = ""
    if len(results) > 0:
        cols = []
        for key,val in results[0].items():
            if not key.startswith("_") or key == "_raw" or key == "_time":
                cols.append(key)   
        text = ','.join(cols) +'\n'   
        for result in results:
            vals = []
            for col in cols:
                val = result.get(col)
                if isinstance(val, list):
                    val = ' '.join(map(str,val))
                vals.append(val)
            text += ','.join(vals) + '\n'
    else:
        text = "No results found."
    return text

def plainRawTemplate():
    return template.Template('''
% if len(results) > 0:
    % if results[0].get('_raw'):
        % for result in results:
${result.get("_raw", "")}\n
        % endfor
    % else:
The results contain no "_raw" field.  Please choose another inline format (csv or table).
    % endif
% else:
No results found.
% endif
''')

def should_inline_png(ssContent, argvals):
    sendpng = normalizeBoolean(ssContent.get('action.email.sendpng'), False)
    no_errors = len(ssContent.get('errorArray', [])) == 0
    inline_specified = normalizeBoolean(ssContent.get('action.email.inline'), False) or normalizeBoolean(argvals.get('inline'), False)
    return sendpng and no_errors and inline_specified

def buildAttachments(settings, ssContent, results, email, jobCount, argvals):
    toAttach    = []
    ssContent['errorArray'] = []
    sendpdf     = normalizeBoolean(ssContent.get('action.email.sendpdf', False))
    sendcsv     = normalizeBoolean(ssContent.get('action.email.sendcsv', False))
    sendpng     = normalizeBoolean(ssContent.get('action.email.sendpng', False))
    allowEmpty  = normalizeBoolean(ssContent.get('action.email.allow_empty_attachment', False))
    sendresults = normalizeBoolean(ssContent.get('action.email.sendresults', False))
    inline      = normalizeBoolean(ssContent.get('action.email.inline', False))
    inlineFormat= ssContent.get('action.email.format')
    input_type  = ssContent['type']

    namespace   = settings['namespace']
    owner       = settings['owner']
    sessionKey  = settings['sessionKey']
    searchid    = settings.get('sid')
    
    pdfview            = ssContent.get('action.email.pdfview', '')
    subject            = ssContent.get("action.email.subject")
    ssName             = ssContent.get("name")
    server             = ssContent.get('action.email.mailserver', 'localhost')
    results_link       = ssContent.get('results_link')

    paperSize        = ssContent.get('action.email.reportPaperSize', 'letter')
    paperOrientation = ssContent.get('action.email.reportPaperOrientation', 'portrait')
    pdfLogoPath  = ssContent.get('action.email.pdf.logo_path')

    # Despite being generically called a "view", a pdfview is _always_ the ID of
    # a splunk dashboard. Also see src/scheduler/SavedSearchAdminHandler.cpp's
    # definition of getDefaultScheduledView()
    isDashboard  = normalizeBoolean(pdfview)
    pdf_attachment   = None
    png_attachment = None
    export_type = ''
    alertActions = getAlertActions(sessionKey)
    if alertActions:
        # Take ssContent first, then fallback to alertActions.
        ss_filename = ssContent.get('action.email.reportFileName', None)
        aa_filename = alertActions.get('reportFileName', None)
        fileName = ss_filename if ss_filename else aa_filename
        # Ideally we would retrieve the alert actions conf settings in mail()
        # which is where these settings are used. But we are already making
        # a REST call to get these settings, so we save these settings for later
        ssContent['alertActions'] = alertActions
    else:
        fileName = None

    if sendpdf or sendpng:
        sendtestemail = normalizeBoolean(argvals.get('sendtestemail', False))
        # Dashboard PDF export *always* overrides allowEmpty (action.email.allow_empty_attachment)
        if len(results) == 0 and not allowEmpty and not sendtestemail and not isDashboard:
            logger.info("Not attaching pdf file due to no matching results and allow_empty_attachment=%s" % str(allowEmpty))
        else:
            import splunk.pdf.availability as pdf_availability
            pdfgen_available = pdf_availability.is_available(session_key=sessionKey)
            logger.info("pdfgen_available = %s" % pdfgen_available)

            try:
                if pdfgen_available:
                    result_id = None
                    if ssContent.get('action.per_alert_result_id'):
                        result_id = ssContent.get('action.per_alert_result_id')
                    
                    if sendpdf:
                        export_type = 'pdf'
                        # will raise an Exception on error
                        pdf_attachment = generate_attachment(server, subject, searchid, settings, pdfview, ssName, paperSize,
                                            paperOrientation, pdfLogoPath, result_id, export_type, ssContent['errorArray'])
                    if sendpng:
                        export_type = 'png'
                        png_attachment = generate_attachment(server, subject, searchid, settings, pdfview, ssName, paperSize,
                                            paperOrientation, pdfLogoPath, result_id, export_type, ssContent['errorArray'])

            except Exception as e:
                logger.error("An error occurred while generating a %s: %s" % (export_type.upper(), e))
                if len(ssContent['errorArray']) == 0:
                    # default error message (should never reach here for Studio dashboards)
                    base_message = "An error occurred while generating the %s." % export_type.upper()
                    pdf_message = base_message + " " + "See python.log and pdfgen.log for errors."
                    png_message = base_message + " " + "Only Dashboard Studio supports PNG exports. Migrate to Dashboard Studio to schedule a PNG export."
                    error_message = png_message if export_type == 'png' else pdf_message
                    ssContent['errorArray'].append(error_message)

            # build up filename to use with attachments
            props = {
                "owner": owner or 'nobody',
                "app": namespace,
                "type": "dashboard" if isDashboard else input_type or "report",
                "name": pdfview or ssName
            }
            if pdf_attachment:
                pdf_filename = pu.makeReportName(pattern=fileName, type='pdf', reportProps=props)
                pdf_attachment.add_header('content-disposition', 'attachment', filename=pdf_filename)
                toAttach.append(pdf_attachment)
            if png_attachment:
                png_filename = pu.makeReportName(pattern=fileName, type='png', reportProps=props)
                extension = '.png'
                content_id = "<" + pdfview + extension + ">"

                png_attachment.add_header('Content-Type', mimetypes.types_map[extension])
                png_attachment.add_header("Content-Disposition", "inline; filename=" + png_filename)

                if should_inline_png(ssContent, argvals):
                    # attaches inline
                    png_attachment.add_header("Content-ID", content_id)
                    email.attach(png_attachment)
                # attaches externally
                toAttach.append(png_attachment)


    # (input_type == searchCommand and sendresults and not inline) needed for backwards compatibility
    # (sendresults and not(sendcsv or sendpdf or inline) and inlineFormat == 'csv') 
    #       needed for backwards compatibility when we did not have sendcsv pre 6.1 SPL-79561
    if len(results) == 0 and not allowEmpty:
        logger.info("Not attaching csv file due to no matching results and allow_empty_attachment=%s" % str(allowEmpty))
    elif (sendcsv or (input_type == 'searchCommand' and sendresults and not inline) or (sendresults and not(sendcsv or sendpdf or inline) and inlineFormat == 'csv')):
        csvAttachment = MIMEBase("text", "csv")
        # SPL-179427 add choice whether to escape newlines
        escapeCSVNewline = normalizeBoolean(ssContent.get('action.email.escapeCSVNewline', True))
        csvAttachment.set_payload(generateCSVResults(results, escapeCSVNewline))
        encoders.encode_base64(csvAttachment)
        props = {
            "owner": owner or 'nobody',
            "app": namespace,
            "type": input_type,
            "name": ssName
        }
        filename = pu.makeReportName(pattern=fileName, type="csv", reportProps=props)
        csvAttachment.add_header('Content-Disposition', 'attachment', filename=filename)
        toAttach.append(csvAttachment)
        if normalizeBoolean(settings.get('truncated')):
            if normalizeBoolean(len(results)) and normalizeBoolean(jobCount):
                ssContent['errorArray'].append("Only the first %s of %s results are included in the attached csv." %(len(results), jobCount))
            else:
                ssContent['errorArray'].append("Attached csv results have been truncated.")

    return toAttach

def esc(val):
    return val.replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")

def generateCSVResults(results, escapeCSVNewline):
    if len(results) == 0:
        return 'No matching events found.'

    header = []
    s = BytesIO()
    if sys.version_info >= (3, 0):
        # SPL-204962 Scheduled Jobs sendemail csv file Generated Extra Blank Rows
        # Per Python 3 csv documentation https://docs.python.org/3/library/csv.html, csv file objects should be opened with newline=''.
        #
        # Also in the **footnotes** of the documentation
        #   If newline='' is not specified, newlines embedded inside quoted fields will not be interpreted correctly,
        #   and on platforms that use \r\n linendings on write an extra \r will be added. It should always be safe to specify
        #   newline='', since the csv module does its own (universal) newline handling.
        t = TextIOWrapper(s, write_through = True, encoding='utf-8', errors='backslashreplace', newline='')
        w = csv.writer(t)
    else:
        w = csv.writer(s)
    
    
    if "_time" in results[0] : header.append("_time")
    if "_raw"  in results[0] : header.append("_raw")
    
    # for backwards compatibility remove all internal fields except _raw and _time
    for k in results[0].keys():
       if k.startswith("_") :
          continue
       header.append(k)
        

    w.writerow(header)
    # output each result's values
    for result in results:
        row = []
        for col in header:
            val = result.get(col,"")
            if isinstance(val, list):
                val = ' '.join(map(str,val))
            if (escapeCSVNewline):
                row.append(esc(val))
            else:
                row.append(val)

        w.writerow(row)
    return s.getvalue()

# call separated for testing purposes
def call_pdfgen_render(sessionKey, parameters):
    response, content = simpleRequest("pdfgen/render", sessionKey = sessionKey, postargs = parameters, timeout = PDFGEN_SIMPLE_REQUEST_TIMEOUT)
    return response, content

def handle_png_error_messages(content, export_type, error_array):
    if export_type != 'png':
        return
    
    error_messages = get_error_messages(content)
    if error_messages is not None:
        error_array.extend(error_messages)

def attempt_json_formatting(decoded_content):
    try:
        formatted_json_content = json.dumps(ast.literal_eval(decoded_content))
        json_data = json.loads(formatted_json_content)
        return json_data.get('error_messages', None)
    except (ValueError, TypeError, json.JSONDecodeError) as e:
        logger.error("Failed to format JSON for error message due to %s" % repr(e))
        return None

def get_error_messages(content: bytes):
    try:
        decoded_content = content.decode('utf-8')
        json_data = json.loads(content)
        return json_data.get('error_messages', None)
    except (UnicodeDecodeError, json.JSONDecodeError):
        return attempt_json_formatting(decoded_content)

def generate_attachment(serverURL, subject, sid, settings, pdfViewID, ssName, paperSize, paperOrientation, pdfLogoPath, resultId, export_type, error_array):
    """
    Reach out and retrieve a PDF copy of the search results if possible
    and return the MIME attachment
    """
    sessionKey = settings.get('sessionKey', None)
    owner = settings.get('owner', 'nobody')
    upper_export_type = export_type.upper()
    if not sessionKey:
        raise SendEmailException("Can't attach %s - sessionKey unavailable" % upper_export_type)

    # build up parameters to the PDF server
    parameters = {}
    parameters['namespace'] = settings["namespace"]
    parameters['owner'] = owner
    if pdfViewID:
        parameters['input-dashboard'] = pdfViewID
        if export_type == 'png':
            parameters['type'] = export_type
    else:
        if ssName:
            parameters['input-report'] = ssName
            parameters['timezone'] = settings.get('timezone')
        elif sid:
            # in the event where sendemail is called from search
            # and we need to generate pdf re-run the search 
            job = search.getJob(sid, sessionKey=sessionKey)
            jsonJob = job.toJsonable(timeFormat='unix')

            searchToRun = jsonJob.get('search').strip()
            if searchToRun.lower().startswith('search '):
                searchToRun = searchToRun[7:]

            sendemailRegex = r'\|\s*sendemail'
            if (re.findall(sendemailRegex, searchToRun)):
                parameters['input-search'] = re.split(sendemailRegex, searchToRun)[0]
                parameters['et'] = jsonJob.get('earliestTime')
                parameters['lt'] = jsonJob.get('latestTime')
            else:
                raise SendEmailException("Can't attach %s - ssName and pdfViewID unavailable" % upper_export_type)

    if sid:
        if type(sid) is dict:
            for sidKey in sid:
                parameters[sidKey] = sid[sidKey]
        else:    
            parameters['sid'] = sid
    
    if paperSize and len(paperSize) > 0:
        if paperOrientation and paperOrientation != "portrait":
            parameters['paper-size'] = "%s-%s" % (paperSize, paperOrientation)
        else:
            parameters['paper-size'] = paperSize

    if pdfLogoPath:
        parameters['pdf.logo_path'] = pdfLogoPath

    if resultId:
        parameters['result_id'] = resultId

    # determine if we should set an effective dispatch "now" time for this job
    scheduledJobEffectiveTime = getEffectiveTimeOfScheduledJob(settings.get("sid", ""))
    logger.info("sendemail:mail effectiveTime=%s" % scheduledJobEffectiveTime) 
    if scheduledJobEffectiveTime != None:
        parameters['now'] = scheduledJobEffectiveTime
    try:
        response, content = call_pdfgen_render(sessionKey, parameters)

    except splunk.SplunkdConnectionException as e:
        raise SendEmailException("Failed to fetch %s (SplunkdConnectionException): %s" % (upper_export_type, repr(e)))

    except Exception as e:
        raise SendEmailException("Failed to fetch %s (Exception type=%s): %s" % (upper_export_type, repr(type(e)), repr(e)))

    if response['status']!='200':
        raise SendEmailException("Failed to fetch %s (status = %s): %s" % (upper_export_type, repr(response['status']), repr(content)))
    
    if export_type == 'png' and response['content-type'] == 'application/json':
        handle_png_error_messages(content, export_type, error_array)
        return None

    if (export_type == 'pdf' and response['content-type']!='application/pdf') or (export_type == 'png' and response['content-type']!='image/png'):
        raise SendEmailException("Failed to fetch %s (content-type = %s): %s" % (upper_export_type, repr(response['content-type']), repr(content)))

    mpart = None
    if export_type == 'png':
        mpart = MIMEImage(content, _subtype=export_type)
    else:
        mpart = MIMEApplication(content, export_type)
    logger.info('Generated %s for email' % upper_export_type)
    return mpart

def getEffectiveTimeOfScheduledJob(scheduledJobSid):
    """ parse out the effective time from the sid of a scheduled job
        if no effective time specified, then return None
        scheduledJobSid is of form: scheduler__<owner>__<namespace>_<hash>_at_<epoch seconds>_<mS> """
    scheduledJobSidParts = scheduledJobSid.split("_")
    effectiveTime = None
    if "scheduler" in scheduledJobSidParts and len(scheduledJobSidParts) > 4 and scheduledJobSidParts[-3] == "at":
        secondsStr = scheduledJobSidParts[-2]
        try:
            effectiveTime = int(secondsStr)
        except:
            pass

    return effectiveTime


def getCredentials(sessionKey, namespace):
   try:
      ent = entity.getEntity('admin/alert_actions', 'email', namespace=namespace, owner='nobody', sessionKey=sessionKey)
      if 'auth_username' in ent and 'clear_password' in ent:
          encrypted_password = ent['clear_password']
          splunkhome = os.environ.get('SPLUNK_HOME')
          if splunkhome == None:
              logger.error('getCredentials - unable to retrieve credentials; SPLUNK_HOME not set')
              return None
          clear_password = cli_common.decrypt(encrypted_password, setEnv=True)
          return ent['auth_username'], clear_password
   except Exception as e:
      logger.error("Could not get email credentials from splunk, using no credentials. Error: %s" % (str(e)))

   return '', ''


def getAlertActions(sessionKey):
    settings = None
    try:
        settings = entity.getEntity('/configs/conf-alert_actions', 'email', sessionKey=sessionKey)

        logger.debug("sendemail.getAlertActions conf file settings %s" % settings)
    except Exception as e:
        logger.error("Could not access or parse email stanza of alert_actions.conf. Error=%s" % str(e))

    return settings

def sendHealthAlertEmail(results, settings):
    """
    This function will only be called to send health report alerting emails.
    Email parameters will be passed through settings/results.
    """

    if not results or len(results) < 1:
        logger.error("There is no email content specified.")
        return

    if not settings.get('to') and not settings.get('cc') and not settings.get('bcc'):
        logger.error("There are no recipients specified")
        return

    sessionKey = ""
    alertConfig = {}
    email = MIMEMultipart()
    if settings.get('to'):
        email['To'] = settings.get('to')
    if settings.get('cc'):
        email['Cc'] = settings.get('cc')
    if settings.get('bcc'):
        email['Bcc'] = settings.get('bcc')

    if settings.get('from'):
        email['From'] = settings.get('from')
    alertConfig['action.email.use_ssl'] = normalizeBoolean(settings.get('use_ssl'))
    alertConfig['action.email.use_tls'] = normalizeBoolean(settings.get('use_tls'))
    if settings.get('mailserver'):
        alertConfig['action.email.mailserver'] = settings.get('mailserver')

    if settings.get('auth_username'):
        argvals['username'] = settings.get('auth_username')
    if settings.get('auth_password'):
        argvals['password'] = settings.get('auth_password')

    email['Subject'] = Header(results[0].get('subject'), CHARSET)
    plainMsg = results[0].get('plain_msg')
    email.attach(MIMEText(plainMsg, 'plain', _charset=CHARSET))

    try:
        mail(email, argvals, alertConfig, sessionKey)
    except Exception as e:
        errorMessage = 'Error sending Health Report alert. Error="%s".' % e
        logger.error(errorMessage)

    # For health report email alert, don't need return results.
    return None

def createOrganizedResultBuffer(buffer):
    # creates the readable buffer for reading search results into the send email script
    return TextIOWrapper(buffer, encoding='utf-8', errors='backslashreplace')

if __name__ == '__main__':
    if sys.version_info >= (3, 0):
        # We need to support data of non utf-8 encodings coming into this script, to do that,
        # we allow utf-8 errors 
        input_buf = createOrganizedResultBuffer(sys.stdin.buffer)
    else:
        input_buf = sys.stdin
    results, dummyresults, settings = splunk.Intersplunk.getOrganizedResults(input_buf)
    try:
        keywords, argvals = splunk.Intersplunk.getKeywordsAndOptions(CHARSET)

        logger.debug('SENDEMAIL keywords: %s, argvals: %s' % (keywords, argvals))

        if 'is_health_alert' in argvals:
            results = sendHealthAlertEmail(results, settings)
        else:
            if results or 'ssname' in argvals:
                results = sendEmail(results, settings, keywords, argvals)
            elif 'sendtestemail' in argvals:
                if normalizeBoolean(argvals.get('sendtestemail')):
                    results = sendEmail(results, settings, keywords, argvals)
            else:
                logger.warn("search results is empty, no email will be sent")
    except Exception as e:
        logger.exception(e)
    splunk.Intersplunk.outputResults(results)