I am trying to aggregate bigip request logs for a couple of virtual servers to influxd and graph them using grafana (graph the data from the retention policies to which i write via continuous queries in influxd). The idea is something like this: f5 → rsyslog (udp - writes user.info facility entries to every1min logrotating files on a tmpfs (to not be a bottleneck)) → telegraf logparser using bellow conf → influxd default retention policy ( → continuous queries → retention policies → grafana )
my problem is that the write performance in influxd is abysmal. As in it looks like a lot of events get dropped (see chronograph graph). I’ve tried changing the output to file and counted the input file entries / minute versus the output-logparser-parsed entries / minute and came to the conclusion that telegraf is not the bottleneck (the logparser and the processors.regex at least) -the ones commented bellow.
about 1000 requests / second, not necessarily in order but most are (some requests may take longer)
[global_tags]
[agent]
interval = “10s”
round_interval = true
metric_batch_size = 1000
metric_buffer_limit = 10000
collection_jitter = “0s”
flush_interval = “10s”
flush_jitter = “0s”
precision = “”
debug = false
quiet = false
logfile = “/var/log/telegraf/telegraf_error.log”
hostname = “”
omit_hostname = false
[[outputs.influxdb]]
namedrop = [“rqbipsysl”,“internal_“]
urls = [“http://127.0.0.1:8086”] # required
database = “telegraf” # required
retention_policy = “”
write_consistency = “any”
timeout = “5s”
[[inputs.cpu]]
percpu = true
totalcpu = true
collect_cpu_time = false
report_active = false
[[inputs.disk]]
ignore_fs = [“tmpfs”, “devtmpfs”, “devfs”]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]
[[inputs.logparser]]
files = [”/srv/accesslog_tmpfs/bigip_access_log"]
from_beginning = false
name_override = “rqbipsysl”
[inputs.logparser.grok]
patterns = [“%{BIGIPSYSLOG}”]
custom_patterns = ‘’’
BIGIPSYSLOG %{IP:bigipip:tag} [%{HTTPDATE:ts:ts-httpd}] %{IP:clientip} %{IP:virtual_ip:tag} %{DATA:virtual_name:tag} %{DATA:virtual_pool_name:tag} %{DATA:server:tag} %{NUMBER:server_port} “(?:%{WORD:verb:tag} %{NOTSPACE:request:tag}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})” %{NUMBER:resp_code:tag} %{NUMBER:bytes:int} %{NUMBER:response_ms:int} %{QS:referrer} %{QS:agent}
‘’’
[[processors.regex]]
namepass = [“rqbipsysl”]
[[processors.regex.tags]]
key = “resp_code”
pattern = “^(\d)\d\d$”
replacement = “${1}xx”
[[processors.regex.tags]]
key = “request”
pattern = "/?([[:alnum:]%=.:~+,-]+)/?([[:alnum:]%=.:~+,-]+)?/?([[:alnum:]%_=.:~+,-]+)?.”
replacement = “${1}”
result_key = “context”
[[processors.regex.tags]]
key = “request”
pattern = “/?([[:alnum:]%=.:~+,-]+)/?([[:alnum:]%=.:~+,-]+)?/?([[:alnum:]%=.:~+,-]+)?.*"
replacement = “${2}”
result_key = “subcontext”
[[processors.regex.tags]]
key = “request”
pattern = "/?([[:alnum:]%=.:~+,-]+)/?([[:alnum:]%=.:~+,-]+)?/?([[:alnum:]%=.:~+,-]+)?.*”
replacement = “${3}”
result_key = “subsubcontext”
[[outputs.influxdb]]
urls = [“http://127.0.0.1:8086”] # required
database = “bigipsyslog” # required
write_consistency = “any”
timeout = “5s”
namepass = [“rqbipsysl”]
tagexclude = [“request”]
retention_policy = “asis”
[outputs.influxdb.tagdrop]
virtual_name = [ “/Common/vsname_ssl_vs” ]
#[[outputs.file]]
files = [“/srv/accesslog_tmpfs/bubu7.txt”]
namepass = [“rqbipsysl”]
tagexclude = [“request”]
#[[inputs.internal]]
collect_memstats = true
#[[outputs.file]]