You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
122 lines
4.4 KiB
122 lines
4.4 KiB
5 months ago
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
|
||
|
|
import splunk.clilib.bundle_paths as bundle_paths
|
||
|
|
import splunk.clilib.cli_common as comm
|
||
|
|
from splunk.appbuilder import PACKAGE_PATH
|
||
|
|
|
||
|
|
logger = logging.getLogger('splunk.spl2')
|
||
|
|
|
||
|
|
FILELOCK_SERVICE_FAILED = "Service temporarily unavailable. Retry in 30 seconds."
|
||
|
|
|
||
|
|
DOWNLOAD_MODULES_ENDPOINT = "/spl2/modules/download?app={app_name}"
|
||
|
|
|
||
|
|
|
||
|
|
def sync_spl2_objects(handler_args, object_type, session_key, app_name):
|
||
|
|
if object_type == "spl2-modules":
|
||
|
|
dest = handler_args.get('dest', ["default"])[0]
|
||
|
|
|
||
|
|
module_filter = []
|
||
|
|
arg_data = handler_args.data
|
||
|
|
if 'filter' in arg_data and arg_data['filter'][0] is not None:
|
||
|
|
for entry in arg_data['filter'][0].split(','):
|
||
|
|
entry = entry.strip()
|
||
|
|
if entry:
|
||
|
|
module_filter.append(entry)
|
||
|
|
|
||
|
|
app_modules = _get_all_modules(app_name, session_key)
|
||
|
|
if not app_modules:
|
||
|
|
logger.debug("No SPL2 modules found for app '{}'".format(app_name))
|
||
|
|
return
|
||
|
|
|
||
|
|
bundle_paths.maybe_makedirs(PACKAGE_PATH, throw_exceptions=True)
|
||
|
|
# TODO: Add concurrency mechanism - SCP-69093
|
||
|
|
create_module_files(app_name, dest, app_modules, module_filter)
|
||
|
|
else:
|
||
|
|
raise NotImplementedError("No handler for object_type {}".format(object_type))
|
||
|
|
|
||
|
|
|
||
|
|
def create_module_files(app, dest, modules, module_filter):
|
||
|
|
app_path = bundle_paths.get_bundle(app).location()
|
||
|
|
|
||
|
|
temp_path = os.path.join(PACKAGE_PATH, 'SPL2_DELETEME_' + app)
|
||
|
|
spl2_path = os.path.join(app_path, dest, "data", "spl2")
|
||
|
|
|
||
|
|
# delete stale temp_path if exists from previous CLI invoke and re-create dirs
|
||
|
|
bundle_paths.safe_remove(temp_path)
|
||
|
|
bundle_paths.maybe_makedirs(temp_path)
|
||
|
|
|
||
|
|
# if ..app/{dest}/data/spl2 exists, copy all contents to temp_path
|
||
|
|
if os.path.lexists(spl2_path):
|
||
|
|
comm.mergeDirs(spl2_path, temp_path)
|
||
|
|
|
||
|
|
# delete all .spl2 files from temp_path
|
||
|
|
_delete_spl2_modules(temp_path)
|
||
|
|
# write module files to the temp_path
|
||
|
|
_write_module_files(temp_path, modules, module_filter)
|
||
|
|
|
||
|
|
# remove ..app/{dest}/data/spl2 and mv temp_path contents to /data/spl2
|
||
|
|
bundle_paths.safe_remove(spl2_path)
|
||
|
|
comm.moveItem(temp_path, spl2_path)
|
||
|
|
logger.info("Moved downloaded modules to {}".format(spl2_path))
|
||
|
|
|
||
|
|
|
||
|
|
def _write_module_files(des, modules, module_filter):
|
||
|
|
modules_written = []
|
||
|
|
|
||
|
|
for module in modules:
|
||
|
|
name = module['name']
|
||
|
|
dirs = module['namespace'].split('.')[2:] # remove apps.<app_name.
|
||
|
|
content = module['definition']
|
||
|
|
mod_resource_name = module['namespace'] + '.' + module['name']
|
||
|
|
folder_path = os.path.join(des, *dirs)
|
||
|
|
file_path = os.path.join(folder_path, name + ".spl2")
|
||
|
|
|
||
|
|
file_ns_name = ".".join(dirs + [name])
|
||
|
|
|
||
|
|
if module_filter:
|
||
|
|
if file_ns_name in module_filter:
|
||
|
|
_write_module(content, file_path, folder_path)
|
||
|
|
modules_written.append(mod_resource_name)
|
||
|
|
else:
|
||
|
|
logger.debug("Skipped writing module {}. Applied filter: {}".format(file_ns_name, module_filter))
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
_write_module(content, file_path, folder_path)
|
||
|
|
modules_written.append(mod_resource_name)
|
||
|
|
|
||
|
|
if modules_written:
|
||
|
|
logger.info("Wrote {} module(s) to {}. Modules: {}".format(len(modules_written), des, modules_written))
|
||
|
|
|
||
|
|
|
||
|
|
def _write_module(content, file_path, folder_path):
|
||
|
|
os.makedirs(folder_path, exist_ok=True)
|
||
|
|
with open(file_path, "w") as file:
|
||
|
|
file.write(content)
|
||
|
|
|
||
|
|
|
||
|
|
def _get_all_modules(app, auth):
|
||
|
|
from splunk import rest
|
||
|
|
download_path = DOWNLOAD_MODULES_ENDPOINT.format(app_name=app)
|
||
|
|
response, content = rest.simpleRequest(download_path, sessionKey=auth, raiseAllErrors=True)
|
||
|
|
if content:
|
||
|
|
modules = json.loads(content)
|
||
|
|
logger.debug("Downloaded {} modules from app: {} using REST".format(len(modules), app))
|
||
|
|
return modules
|
||
|
|
|
||
|
|
|
||
|
|
def _delete_spl2_modules(path, extension=".spl2"):
|
||
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
||
|
|
for file in files:
|
||
|
|
if file.endswith(extension):
|
||
|
|
file_path = os.path.join(root, file)
|
||
|
|
os.remove(file_path)
|
||
|
|
|
||
|
|
# Now check and delete empty directories
|
||
|
|
for root, dirs, files in os.walk(path, topdown=False):
|
||
|
|
for dir_name in dirs:
|
||
|
|
dir_path = os.path.join(root, dir_name)
|
||
|
|
if not os.listdir(dir_path):
|
||
|
|
os.rmdir(dir_path)
|