#! /usr/bin/env python3 import os import sys import time import uuid import shutil import random import datetime SYSLOG_DATE_FMT = "%b %d %H:%M:%S" ACCESS_LOG_DATE_FMT = "%d/%b/%Y:%H:%M:%S" GENERIC_DATE_FMT = "%Y-%m-%dT%H:%M:%S.%%s" TEST_ADDRESSES = ( ["192.0.2.55"] * 20 + ["192.0.2.44"] * 20 + ["192.0.2.3"] * 40 + ["192.0.2.100"] * 10 + ["192.0.2.122"] * 30 + ["192.0.2.42"] * 100 ) TEST_USERNAMES = [ "bob@example.com", "bob@example.com", "bob@example.com", "combatcarl@example.com", "combatcarl@example.com", "combatcarl@example.com", "combatcarl@example.com", "combatcarl@example.com", "combatcarl@example.com", "-", "-", "-", "-", "-", "-", ] TEST_METHODS = [ "GET", "GET", "GET", "GET", "GET", "GET", "PUT", ] PATH_COMPS = "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua".split() TEST_URLS = [ "/index.html", "/index.html", "/index.html", "/features.html", "/images/compass.jpg", "/obj/1234", "/obj/1235?foo=bar", "/obj/1236?search=demo&start=1", ] for index in range(0, 50): path_list = [] for count in range(random.randint(2, 8)): path_list.append(random.choice(PATH_COMPS)) TEST_URLS.append("/".join(path_list)) TEST_VERSIONS = [ "HTTP/1.0", "HTTP/1.0", "HTTP/1.1", ] TEST_STATUS = [ 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 404, 404, 404, 404, 403, 403, 403, 500 ] TEST_REFERRERS = [ "-", "-", "-", "-", "-", "-", "http://lnav.org/download.html", ] TEST_AGENTS = [ "-", "-", "-", "-", "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.3 (java 1.5)", "Apache-HttpClient/4.2.3 (java 1.5)", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Roku4640X/DVP-7.70 (297.70E04154A)", "Roku4640X/DVP-7.70 (297.70E04154A)", "Roku4640X/DVP-7.70 (297.70E04154A)", ] START_TIME = datetime.datetime.fromtimestamp(1692700000) ACCESS_LOG_CURR_TIME = START_TIME SYSLOG_LOG_CURR_TIME = START_TIME def access_log_msgs(): global ACCESS_LOG_CURR_TIME while True: ACCESS_LOG_CURR_TIME += datetime.timedelta(seconds=random.randrange(1, 5)) yield '%s - %s [%s +0000] "%s %s %s" %s %s "%s" "%s"\n' % ( random.choice(TEST_ADDRESSES), random.choice(TEST_USERNAMES), ACCESS_LOG_CURR_TIME.strftime(ACCESS_LOG_DATE_FMT), random.choice(TEST_METHODS), random.choice(TEST_URLS), random.choice(TEST_VERSIONS), random.choice(TEST_STATUS), int(random.lognormvariate(1, 1) * 1000), random.choice(TEST_REFERRERS), random.choice(TEST_AGENTS) ) TEST_PROCS = [ "server[123]", "server[123]", "server[123]", "server[121]", "server[124]", "server[123]", "worker[61456]", "worker[61456]", "worker[61457]", ] TEST_MSGS = [ "Handling request %s" % uuid.uuid4(), "Handling request %s" % uuid.uuid4(), "Handling request %s" % uuid.uuid4(), "Successfully started helper", "Reading from device: /dev/hda", "Received packet from %s" % random.choice(TEST_ADDRESSES), "Received packet from %s" % random.choice(TEST_ADDRESSES), "Received packet from %s" % random.choice(TEST_ADDRESSES), ] def syslog_msgs(): global SYSLOG_LOG_CURR_TIME while True: SYSLOG_LOG_CURR_TIME += datetime.timedelta(seconds=random.randrange(1, 5)) yield '%s frontend3 %s: %s\n' % ( SYSLOG_LOG_CURR_TIME.strftime(SYSLOG_DATE_FMT), random.choice(TEST_PROCS), random.choice(TEST_MSGS), ) try: shutil.rmtree("/tmp/demo") os.makedirs("/tmp/demo") except OSError: pass FILES = [ ("/tmp/demo/access_log", access_log_msgs()), ("/tmp/demo/messages", syslog_msgs()), ] COUNTER = 0 while COUNTER < 5000: loop_inc = datetime.timedelta(seconds=random.weibullvariate(1, 1.5) * 500) ACCESS_LOG_CURR_TIME += loop_inc SYSLOG_LOG_CURR_TIME += loop_inc for fname, gen in FILES: for i in range(random.randrange(4, 8)): COUNTER += 1 with open(fname, "a+") as fp: if random.uniform(0.0, 1.0) < 0.01: line = next(gen) prefix = line[:50] suffix = line[50:] fp.write(prefix) # time.sleep(random.uniform(0.5, 0.6)) fp.write(suffix) else: fp.write(next(gen)) # if random.uniform(0.0, 1.0) < 0.010: # fp.truncate(0) time.sleep(random.uniform(0.01, 0.02)) # if random.uniform(0.0, 1.0) < 0.001: # os.remove(fname)