|
|
# coding=utf-8
|
|
|
|
|
|
#
|
|
|
# streambag search operator, for generating test data in streaming manner
|
|
|
|
|
|
import os, sys, time, datetime
|
|
|
import splunk.util as util
|
|
|
import splunk.Intersplunk as isp
|
|
|
import logging as logger
|
|
|
import traceback
|
|
|
from builtins import range
|
|
|
|
|
|
DEFAULT_ARGS = {
|
|
|
'chunks': 10,
|
|
|
'chunkdelay': 0,
|
|
|
'rowcount': 100,
|
|
|
'multiline': False,
|
|
|
'distribution': 'flat',
|
|
|
|
|
|
'timeformat': 1,
|
|
|
'interval': 60 * 17,
|
|
|
'basetime': time.time(),
|
|
|
'timeorder': 'reverse',
|
|
|
|
|
|
'fieldcount': 0,
|
|
|
'constant': '"double quotes" \'single quotes\' \\slashes\\ `~!@#$%^&*()-_=+{}|;:<>,./? [brackets]',
|
|
|
'host': 'HAL_9000',
|
|
|
'source': 'SpaceOdyssey',
|
|
|
'sourcetype': 'fictional',
|
|
|
'punct': '`~!@#$%^&*()-_=+\\"\'{}|;:<>,./?[]',
|
|
|
|
|
|
'filler': '',
|
|
|
'utf8': False,
|
|
|
'mvfield': ''
|
|
|
}
|
|
|
|
|
|
ARG_SEPARATOR = ';'
|
|
|
|
|
|
utf8Samples = [
|
|
|
('Albanian', 'Unë mund të ha qelq dhe nuk më gjen gjë.'),
|
|
|
('Arabic', 'أنا قادر على أكل الزجاج و هذا لا يؤلمني.', 'ltr'),
|
|
|
('Armenian', 'Կրնամ ապակի ուտել և ինծի անհանգիստ չըներ։'),
|
|
|
('Chinese', ' 我能吞下玻璃而不傷身體'),
|
|
|
('Danish', 'Jeg kan spise glas, det gør ikke ondt på mig.'),
|
|
|
('Euro', '€.'),
|
|
|
('French', 'Je peux manger du verre, ça ne me fait pas de mal.'),
|
|
|
('Georgian', 'მინას ვჭამ და არა მტკივა.'),
|
|
|
('Greek', 'Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα.'),
|
|
|
('Hawaiian', 'Hiki iaʻu ke ʻai i ke aniani; ʻaʻole nō lā au e ʻeha.'),
|
|
|
('Hebrew', 'אני יכול לאכול זכוכית וזה לא מזיק לי.', 'ltr'),
|
|
|
('Hindi', 'मैं काँच खा सकता हूँ और मुझे उससे कोई चोट नहीं पहुंचती.'),
|
|
|
('Hindi', 'मैं काँच खा सकता हूँ, मुझे उस से कोई पीडा नहीं होती.'),
|
|
|
('Icelandic', 'Ég get etið gler án þess að meiða mig.'),
|
|
|
('Japanese', '私はガラスを食べられます。それは私を傷つけません'),
|
|
|
('Korean', '나는 유리를 먹을 수 있어요. 그래도 아프지 않아요'),
|
|
|
('Macedonian', 'Можам да јадам стакло, а не ме штета.'),
|
|
|
('Mongolian', 'Би шил идэй чадна, надад хортой биш'),
|
|
|
('Old Norse', 'Ek get etið gler án þess að verða sár.'),
|
|
|
('Polish', 'Mogę jeść szkło, i mi nie szkodzi.'),
|
|
|
('Romanian', 'Pot să mănânc sticlă și ea nu mă rănește.'),
|
|
|
('Serbian', 'Mogu jesti staklo a da mi ne škodi.'),
|
|
|
('Tamil', 'நான் கண்ணாடி சாப்பிடுவேன், அதனால் எனக்கு ஒரு கேடும் வராது.'),
|
|
|
('Thai', 'ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ'),
|
|
|
('Ukrainian', 'Я можу їсти шкло, й воно мені не пошкодить.'),
|
|
|
('Ukrainian', 'Я можу їсти шкло, й воно мені не пошкодить.'),
|
|
|
('Vietnamese', 'Tôi có thể ăn thủy tinh mà không hại gì.'),
|
|
|
('Yiddish', 'איך קען עסן גלאָז און עס טוט מיר נישט װײ.', 'ltr')
|
|
|
]
|
|
|
|
|
|
|
|
|
def generateData(chunknum, **kwargs):
|
|
|
|
|
|
output = []
|
|
|
|
|
|
basetime = float(kwargs['basetime'])
|
|
|
|
|
|
hosts = kwargs['host'].split(ARG_SEPARATOR)
|
|
|
sources = kwargs['source'].split(ARG_SEPARATOR)
|
|
|
sourcetypes = kwargs['sourcetype'].split(ARG_SEPARATOR)
|
|
|
|
|
|
for i in range(kwargs['rowcount']):
|
|
|
|
|
|
rowset = {}
|
|
|
|
|
|
# set time
|
|
|
timedir = -1
|
|
|
if kwargs['timeorder'] == 'forward': timedir = 1
|
|
|
timestamp = basetime + (kwargs['interval'] * (i + chunknum * kwargs['rowcount'])) * timedir
|
|
|
raw = [datetime.datetime.fromtimestamp(timestamp).isoformat()]
|
|
|
rowset['_time'] = timestamp
|
|
|
|
|
|
# set chunk number
|
|
|
raw.append('CHUNK %d' % chunknum)
|
|
|
rowset['chunk'] = chunknum
|
|
|
|
|
|
# set row number
|
|
|
raw.append('POSITION %d' % i)
|
|
|
rowset['position'] = i
|
|
|
|
|
|
# set primary stuff
|
|
|
rowset['host'] = hosts[i % len(hosts)]
|
|
|
rowset['source'] = sources[i % len(sources)]
|
|
|
rowset['sourcetype'] = sourcetypes[i % len(sourcetypes)]
|
|
|
rowset['_cd'] = 234
|
|
|
|
|
|
# insert unicode samples
|
|
|
if kwargs['utf8']:
|
|
|
if kwargs['multiline']:
|
|
|
sample = []
|
|
|
for j in range(i, i + 5):
|
|
|
sample.append(utf8Samples[j % (len(utf8Samples)-1)][1])
|
|
|
raw.append('\n'.join(sample))
|
|
|
else:
|
|
|
sample = utf8Samples[i % (len(utf8Samples)-1)]
|
|
|
raw.append('lang=%s sample="%s"' % (sample[0], sample[1]))
|
|
|
rowset['lang'] = sample[0]
|
|
|
rowset['sample'] = sample[1]
|
|
|
|
|
|
# add high-bit field name
|
|
|
sample = utf8Samples[i % (len(utf8Samples)-1)]
|
|
|
rowset[sample[1].split(' ')[0]] = 'field value in %s' % sample[0]
|
|
|
|
|
|
# set filler
|
|
|
if kwargs['filler']:
|
|
|
raw.append(kwargs['filler'])
|
|
|
|
|
|
# gen odd fields
|
|
|
# if i % 2 != 0:
|
|
|
# rowset['odd'] = i
|
|
|
# raw.append('odd=%s' % i)
|
|
|
|
|
|
# gen constant field value
|
|
|
# rowset['fancy_constant_field'] = kwargs['constant']
|
|
|
# raw.append('constant=%s' % kwargs['constant'])
|
|
|
|
|
|
# gen script injection test
|
|
|
# rowset['<script>alert("field_name_unescaped!")</script>'] = 'field_name_exploit_test'
|
|
|
# rowset['field_value_exploit_test'] = '<script>alert("field_value_unescaped!")</script>'
|
|
|
# raw.append('<script>alert("raw event unescaped!")</script>')
|
|
|
|
|
|
# set fields
|
|
|
# for j in range(kwargs['fieldcount']):
|
|
|
# fieldName = 'field%s' % j
|
|
|
# value = (i + 2) * (j + 2)
|
|
|
# rowset[fieldName] = value
|
|
|
# raw.append('%s=%s' % (fieldName, value))
|
|
|
|
|
|
# construct the raw item
|
|
|
splitter = kwargs['multiline'] and '\n' or ' '
|
|
|
rowset['_raw'] = splitter.join(raw)
|
|
|
|
|
|
# add a multivalued field?
|
|
|
if kwargs['mvfield']:
|
|
|
if (i % 3 == 0):
|
|
|
rowset[kwargs['mvfield']] = [str(i)]
|
|
|
elif (i % 3 == 1):
|
|
|
rowset[kwargs['mvfield']] = [str(i),str(i*i)]
|
|
|
else:
|
|
|
rowset[kwargs['mvfield']] = [str(i),str(i*i),str(i*i*i)]
|
|
|
|
|
|
# add to result set
|
|
|
output.append(rowset)
|
|
|
|
|
|
return output
|
|
|
|
|
|
|
|
|
#
|
|
|
# main
|
|
|
#
|
|
|
|
|
|
# merge any passed args
|
|
|
args = DEFAULT_ARGS
|
|
|
for item in sys.argv:
|
|
|
kv = item.split('=')
|
|
|
if len(kv) > 1:
|
|
|
val = item[item.find('=') + 1:]
|
|
|
try:
|
|
|
val = int(val)
|
|
|
except:
|
|
|
pass
|
|
|
args[kv[0]] = util.normalizeBoolean(val)
|
|
|
|
|
|
# run generator
|
|
|
try:
|
|
|
for c in range(args['chunks']):
|
|
|
if (c > 0 and args['chunkdelay'] > 0):
|
|
|
time.sleep(args['chunkdelay'])
|
|
|
results = generateData(c, **args)
|
|
|
isp.outputStreamResults(results)
|
|
|
except:
|
|
|
sys.stdout.write("FAILED: \n")
|
|
|
traceback.print_exc()
|
