################################################################################ # Remote Application Management # ################################################################################ from contextlib import closing import splunk import splunk.bundle as bundle import splunk.clilib.bundle_paths as bundle_paths import splunk.clilib.cli_common as cli_common import splunk.rest import splunk.rest.format as format import json import logging as logger import splunk.safe_lxml_etree as etree import os import sys import platform from future.moves.urllib.parse import urlencode, quote, unquote from future.moves.urllib.error import HTTPError, URLError from future.moves.urllib.request import urlopen, Request, URLopener # overridden by settings in server.conf -> [applicationsManagement] DEFAULT_URL = "https://apps.splunk.com/api/apps" LOGIN_URL = "https://apps.splunk.com/api/account:login/" VETTED_APPS_URI = "/services/appsbrowser/v1/app/" VETTED_APP_INSTALL_METHOD_SINGLE = "simple" VETTED_APP_INSTALL_METHOD_DISTRIBUTED = "appmgmt_phase" INSTANCE_TYPE_CLOUD = "cloud" HTTP_ACTION = "action" HTTP_ACTION_INSTALL = "install" HTTP_ACTION_DOWNLOAD = "download" HTTP_GET_COUNT = "count" HTTP_GET_OFFSET = "offset" HTTP_GET_QUERY = "q" HTTP_GET_SORTBY = "sort_by" HTTP_GET_SORTDIR = "sort_dir" HTTP_AUTH_TOKEN = "auth" HTTP_AUTH_HEADER = "X-Auth-Token" PRODUCT_TYPE_LITE = "lite" # Gets sent to SplunkBase. GET_ARG_PRODTYPE = "product" NSMAP = { 'a' : format.ATOM_NS, 's' : format.SPLUNK_NS, 'opensearch': format.OPENSEARCH_NS } URLOPEN_TIMEOUT = 15 def isCloud(sessionKey): """ Returns true if running on a cloud stack i.e instanceType == 'cloud' """ server_conf = bundle.getConf('server', sessionKey) if ('instanceType' in server_conf['general'] and server_conf['general']['instanceType'] == INSTANCE_TYPE_CLOUD): return True return False def getAppInstallMethod() -> str: server_config = cli_common.getConfStanza("server", "applicationsManagement") return server_config.get("filterAppInstallMethod", VETTED_APP_INSTALL_METHOD_SINGLE) class RemoteAppsHandlerList(splunk.rest.BaseRestHandler): """ Generate links to remote applications and categories. """ def handle_GET(self): links = { } links["entries"] = "Remote Applications" links["categories"] = "Remote Application Categories" return links class RemoteAppsSetup(splunk.rest.BaseRestHandler): """ Prepare remote applications management based on configuration settings. """ def __init__(self, method, requestInfo, responseInfo, sessionKey): splunk.rest.BaseRestHandler.__init__(self, method, requestInfo, responseInfo, sessionKey) # Default values self._allowRemote = True self._login = LOGIN_URL self._base = DEFAULT_URL self._agent = None self._platformInfo = None self._supportInProductInstall = True self._sslpol = bundle_paths.SSLPolicy() try: platform_info = platform.platform() os_name = platform.system() arch = platform.machine() py_ver = URLopener().version with open(os.path.join(bundle_paths.etc(), "splunk.version")) as f: for i in f: if i.startswith("VERSION"): version = i.split("=")[1].strip().strip('"') elif i.startswith("BUILD"): build = i.split("=")[1].strip() self._agent = "Splunkd/%s (%s; version=%s; arch=%s; build=%s; %s)" % (version, os_name, platform_info, arch, build, py_ver) self._platformInfo = {'version': version, 'platform': os_name} except Exception as e: logger.exception(e) # Manual overrides in server.conf try: conf = bundle.getConf("server", self.sessionKey) s = conf["applicationsManagement"] if not s.isDisabled(): if "allowInternetAccess" in s: self._allowRemote = bundle_paths.parse_boolean(s["allowInternetAccess"]) if "loginUrl" in s: self._login = s["loginUrl"] if "url" in s: self._base = s["url"] if "useragent" in s: self._agent = s["useragent"] if "caCertFile" in s: self._sslpol._cafile = bundle_paths.expandvars(s["caCertFile"]) if "sslCommonNameList" in s: self._sslpol._sslCommonNameList = bundle_paths.expandvars(s["sslCommonNameList"]) if "cipherSuite" in s: self._sslpol._cipherSuite = bundle_paths.expandvars(s["cipherSuite"]) s = conf["shclustering"] if not s.isDisabled(): self._supportInProductInstall = False except Exception as e: logger.exception(e) logger.debug("applicationsManagement.allowInternetAccess = %s" % str(self._allowRemote)) logger.debug("applicationsManagement.loginUrl = %s" % self._login) logger.debug("applicationsManagement.url = %s" % self._base) logger.debug("applicationsManagement.useragent = %s" % self._agent) logger.debug("applicationsManagement.supportInProductInstall = %s" % str(self._supportInProductInstall)) if self._sslpol._cafile is None: logger.debug("applicationsManagement.caCertFile = %s" % str(self._sslpol._cafile)) if self._sslpol._sslCommonNameList is None: logger.debug("applicationsManagement.sslCommonNameList = %s" % str(self._sslpol._sslCommonNameList)) if self._sslpol._cipherSuite is None: logger.debug("applicationsManagement.cipherSuite = %s" % str(self._sslpol._cipherSuite)) def verifyAllowRemote(self): if not self._allowRemote: raise splunk.RESTException(503, "Internet access is disabled") class RemoteAppsLogin(RemoteAppsSetup): """ Handle login to remote applications provider. """ def handle_POST(self): self.verifyAllowRemote() try: post_args = urlencode(self.request["form"]) if sys.version_info >= (3, 0): post_args = post_args.encode() logger.debug("Logging into %s" % self._login) bundle_paths.BundleInstaller().validate_server_cert(self._login, self._sslpol) # Forward post arguments, including username and password. with closing(urlopen(self._login, post_args, URLOPEN_TIMEOUT)) as f: root = etree.parse(f).getroot() token = root.xpath("a:id", namespaces=NSMAP)[0].text if self.request["output_mode"] == "json": self.response.setHeader('content-type', 'application/json') sessDict = {"response" : { "sessionKey" : token } } self.response.write(json.dumps(sessDict)) else: # Generate response. response = etree.Element("response") sessionKey = etree.SubElement(response, "sessionKey") sessionKey.text = token self.response.setHeader('content-type', 'text/xml') self.response.write(etree.tostring(response, pretty_print=True)) logger.debug("Login successful") except HTTPError as e: if e.code in [401, 405]: # Returning 401 logs off current session # Splunkbase retuns 405 when only password is submitted raise splunk.RESTException(400, e.msg) raise splunk.RESTException(e.code, e.msg) except Exception as e: logger.exception(e) raise splunk.AuthenticationFailed class RemoteAppsManager(RemoteAppsSetup): """ Interact with remote applications provider. """ # /services/apps/remote/ -> "base depth" of 3 URL parts BASE_DEPTH = 3 def handle_GET(self): """ Respond to an HTTP GET with information about remote applications. """ self.verifyAllowRemote() try: url = self._native_to_foreign_url() get_args = {} # The Apps site returns specific apps for products such as "lite". # Choosing to be selective, as it may not respect all prod types... # (But startswith() so we match lite and litefree..) if self.getProductType().startswith(PRODUCT_TYPE_LITE): get_args[GET_ARG_PRODTYPE] = "lite" # Create a new ElementTree based on the root element of the remote # feed. This strips things like processing instructions and the XML # manifest. tree = etree.ElementTree(self._get_feed_root(url, extra_get_args=get_args)) root = tree.getroot() self._transform_feed(root) for h in format.getAtomStyleNodes(): root.addprevious(h) str = etree.tostring(tree, xml_declaration=True, encoding='UTF-8', pretty_print=True) self.response.setHeader('content-type', 'text/xml') self.response.write(str.decode('UTF-8')) except Exception as e: if len(self.pathParts) == self.BASE_DEPTH: # If we're handling the base endpoint, suppress exceptions. # They probably aren't caused by user error here. logger.exception(e) # We can't get to the remote source, or it isn't providing # parseable XML back to us. Display no remote apps. return { } else: # Otherwise, allow exceptions to reach user, as they are likely # caused by a bad URL or application name. raise def handle_POST(self): """ Install a remote application in response to an HTTP POST. """ self.verifyAllowRemote() parts = len(self.pathParts) if parts == self.BASE_DEPTH + 2: default_version = True elif parts == self.BASE_DEPTH + 3: default_version = False else: raise splunk.BadRequest if HTTP_AUTH_TOKEN not in self.args: raise splunk.BadRequest("Missing argument: %s" % HTTP_AUTH_TOKEN) if HTTP_ACTION not in self.args: raise splunk.BadRequest("Missing argument: %s" % HTTP_ACTION) if self.args[HTTP_ACTION] not in (HTTP_ACTION_INSTALL, HTTP_ACTION_DOWNLOAD): raise splunk.BadRequest("Invalid value '%s' for argument '%s'" % (self.args[HTTP_ACTION], HTTP_ACTION)) if isCloud(self.sessionKey): app_name = self.pathParts[self.BASE_DEPTH + 1] getargs = {'appid': app_name, 'offset': 0, 'limit': 1} try: # TODO: pass `app_version` to Splunkbase app_version = self.pathParts[self.BASE_DEPTH + 2] except IndexError: app_version = "" logger.info("querying vetted app with args: %s" % getargs) serverResponse, serverContent = splunk.rest.simpleRequest(VETTED_APPS_URI, self.sessionKey, getargs, keepTrailingSlash=True) if serverResponse.status != 200: raise splunk.BadRequest('Error while querying Splunkbase. Splunkd returned %s' % serverContent) vetted_apps = json.loads(serverContent).get('results', []) if len(vetted_apps) == 0 or vetted_apps[0]['appid'] != app_name \ or not self._is_valid_install_method(vetted_apps[0]): raise splunk.BadRequest('App %s is not vetted for Splunk Cloud.' % app_name) url = self._native_to_foreign_url() root = self._get_feed_root(url) if default_version: root = self._get_latest_version(root) href = self._parse_link(root) try: # Package up a Request with auth information. req = Request(href) # XXX: Converting the auth token from a POST arg to a header # requires us to unquote() it. If the client did not correctly # quote() the token, login will fail. req.add_header(HTTP_AUTH_HEADER, unquote(self.args[HTTP_AUTH_TOKEN])) # Install using this Request object. installer = bundle_paths.BundleInstaller() if self.args[HTTP_ACTION] == HTTP_ACTION_INSTALL: b, status = installer.install_from_url(req, sslpol=self._sslpol) self.response.setStatus(status) if ((status == bundle_paths.BundleInstaller.STATUS_INSTALLED) or (status == bundle_paths.BundleInstaller.STATUS_UPGRADED)): # Migrate old-style bundles. logger.debug("Configuring application contents") try: b.migrate() except Exception as e: logger.exception(e) self.addMessage("WARN", "Error during configuration: %s" % e) # Redirect to local application. self.response.setHeader("Location", self._redirect_to_local(b)) # Let splunkd know about newly-installed app. logger.debug("Notifying splunkd that app has been installed") splunk.rest.simpleRequest('apps/local/_reload', sessionKey=self.sessionKey) if status == bundle_paths.BundleInstaller.STATUS_INSTALLED: self.addMessage("INFO", "Installed application: %s" % b.name()) elif status == bundle_paths.BundleInstaller.STATUS_UPGRADED: self.addMessage("INFO", "Upgraded application: %s" % b.name()) else: self.addMessage("WARN", "Could not install application: %s" % b.name()) else: assert self.args[HTTP_ACTION] == HTTP_ACTION_DOWNLOAD downloaded = installer.download_from_url(req, sslpol=self._sslpol) self.addMessage("INFO", "Downloaded application file: %s" % downloaded) self.response.setHeader('content-type', 'application/json') response_json = {"downloaded": downloaded} self.response.write(json.dumps(response_json)) except splunk.ResourceNotFound: raise except splunk.AuthorizationFailed: raise except splunk.InternalServerError: raise except Exception as e: logger.exception(e) raise splunk.InternalServerError(e) def _native_to_foreign_url(self): """ Convert this endpoint's URL into a remote-provider URL. """ url = self._base for part in self.pathParts[self.BASE_DEPTH:]: url += "/" + part return url def _foreign_to_native_url(self, url): """ Convert a remote-provider URL into a URL pointing to this endpoint. """ if not url.startswith(self._base): return url converted_base = splunk.mergeHostPath() for part in self.pathParts[:self.BASE_DEPTH]: converted_base += '/' + part return converted_base + url[len(self._base):] def _get_feed_root(self, url, extra_get_args={}): """ Get an Atom feed of application information from the remote provider. """ try: target_url = url # Forward GET arguments, and add user-agent. args_dict = {} headers = {} args_dict.update(self.request["query"]) if (len(extra_get_args) > 0): args_dict.update(extra_get_args) if self._platformInfo: args_dict.update(self._platformInfo) args = urlencode(args_dict) if args != "": target_url += ("?" + args) logger.debug("Getting feed from: %s" % target_url) if self._agent: headers["User-Agent"] = self._agent bundle_paths.BundleInstaller().validate_server_cert(target_url, self._sslpol) req = Request(target_url, None, headers) f = urlopen(req, None, URLOPEN_TIMEOUT) except HTTPError as e: raise splunk.RESTException(e.code, e.msg) except URLError as e: logger.exception(e) raise splunk.RESTException(503, "Splunk is unable to connect to the Internet to find more apps.") except Exception as e: logger.exception(e) raise splunk.RESTException(404, "Resource not found") try: root = etree.parse(f).getroot() f.close() return root except Exception as e: raise splunk.InternalServerError(e) # XXX: This is very brittle. Assumptions: # - The local apps endpoint is named 'local'. # - If the remote apps endpoint is at: /services/apps/remote, # then the local apps endpoint is at: /services/apps/local # - Local apps are located at /services/apps/local/ def _redirect_to_local(self, b): url = splunk.mergeHostPath() for part in self.pathParts[:(self.BASE_DEPTH - 1)]: url += '/' + part url += '/' + 'local' url += '/' + quote(b.prettyname()) return url def _is_valid_install_method(self, vetted_app): """ Validate install method based on server configuration """ if getAppInstallMethod() == VETTED_APP_INSTALL_METHOD_DISTRIBUTED: return vetted_app['install_method_distributed'] == VETTED_APP_INSTALL_METHOD_DISTRIBUTED return vetted_app['install_method_single'] == VETTED_APP_INSTALL_METHOD_SINGLE def _transform_feed(self, xml): """ Make an Atom feed from the remote provider look as though we generated it by rewriting certain URLs in the feed. """ try: self._convert_remote_elements(xml) for entry in xml.xpath("//a:entry", namespaces=NSMAP): self._convert_remote_elements(entry) except Exception as e: logger.exception(e) raise splunk.InternalServerError(e) def _convert_remote_elements(self, root): """ Make URLs that point to the remote provider point to us instead. """ # Rewrite URL in ID. for id in root.xpath("a:id", namespaces=NSMAP): id.text = self._foreign_to_native_url(id.text) # Rewrite hrefs in links. for link in root.xpath("a:link", namespaces=NSMAP): href = link.get("href") rel = link.get("rel") # Leave download links alone. if href and (rel != "download"): link.set("href", self._foreign_to_native_url(href)) if not self._supportInProductInstall: # XXX: Prevent in-product app install -- make all apps non-free. for price in root.xpath("//a:entry/a:content/s:dict/s:key[@name='price']", namespaces=NSMAP): price.text = 'other' def _get_latest_version(self, xml): """ Given a feed of version entries for an application, get the feed for the latest version. """ try: entry = self._get_latest_version_entry(xml) href = entry.xpath("a:link/@href", namespaces=NSMAP)[0] return self._get_feed_root(href) except Exception as e: logger.exception(e) msg = "Could not find latest version of application" raise splunk.ResourceNotFound(msg) def _get_latest_version_entry(self, xml): """ Given a feed of version entries for an application, get the entry for the latest version. """ for entry in xml.xpath("//a:entry", namespaces=NSMAP): try: contents = self._convert_content(entry) if contents["islatest"] == "True": return entry except Exception as e: logger.exception(e) return None def _parse_link(self, xml): """ Given a feed of application files, get the URL of the installer. """ msg = "Could not find application download location" try: for entry in xml.xpath("//a:entry", namespaces=NSMAP): try: contents = self._convert_content(entry) if contents["fileclass"] in ["bundle", "other"]: return entry.xpath("a:link/@href", namespaces=NSMAP)[0] except Exception as e: logger.exception(e) raise splunk.ResourceNotFound(msg) except Exception as e: logger.exception(e) raise splunk.ResourceNotFound(msg) def _convert_content(self, xml): """ Convert an Atom entry's node into a Python datastructure. """ try: return format.nodeToPrimitive(xml.xpath("a:content", namespaces=NSMAP)[0][0]) except Exception as e: logger.exception(e) return None def _get_from_xml(self, xml, location_path, contents, key): """ Set the value for contents[key] to the text at location_path in xml. """ try: contents[key] = xml.xpath(location_path, namespaces=NSMAP)[0].text except Exception as e: logger.exception(e)