How to parse nested fields data to influx line protocol format

Telegraf
, ,
1

Hi All,

I am trying to send metrics data to influxdb through fluentd and the problem here is metricbeat data format is not fit for influxdb since it uses protocol and it doesn’t allow nested fields but metricbeat data consists of nested fields.

I need to format the data before I sending it to influxdb. However I am not sure how to format the data and please let me know any reference documentation or examples and I will do the corrections in my configuration.

Sample data:-

@metadata={\\\"beat\\\"=\\u003e\\\"metricbeat\\\", \\\"type\\\"=\\u003e\\\"doc\\\", \\\"version\\\"=\\u003e\\\"6.3.0\\\"},
metricset={\\\"name\\\"=\\u003e\\\"process\\\", \\\"module\\\"=\\u003e\\\"system\\\", \\\"rtt\\\"=\\u003e101917},
system={\\\"process\\\"=\\u003e{\\\"fd\\\"=\\u003e{\\\"open\\\"=\\u003e12, \\\"limit\\\"=\\u003e{\\\"hard\\\"=\\u003e8192, \\\"soft\\\"=\\u003e8192}}, \\\"pid\\\"=\\u003e28708, \\\"ppid\\\"=\\u003e1889, \\\"name\\\"=\\u003e\\\"apache2\\\", \\\"memory\\\"=\\u003e{\\\"share\\\"=\\u003e5668864, \\\"size\\\"=\\u003e387362816, \\\"rss\\\"=\\u003e{\\\"pct\\\"=\\u003e0.001, \\\"bytes\\\"=\\u003e9879552}}, \\\"cwd\\\"=\\u003e\\\"/\\\", \\\"cmdline\\\"=\\u003e\\\"/usr/sbin/apache2 -k start\\\", \\\"cpu\\\"=\\u003e{\\\"total\\\"=\\u003e{\\\"pct\\\"=\\u003e0.0007, \\\"norm\\\"=\\u003e{\\\"pct\\\"=\\u003e0.0003}, \\\"value\\\"=\\u003e26810}, \\\"start_time\\\"=\\u003e\\\"2018-07-04T06:25:01.000Z\\\"}, \\\"cgroup\\\"=\\u003e{\\\"memory\\\"=\\u003e{\\\"kmem_tcp\\\"=\\u003e{\\\"limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e9223372036854771712}, \\\"usage\\\"=\\u003e{\\\"max\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"bytes\\\"=\\u003e0}, \\\"failures\\\"=\\u003e0}, \\\"stats\\\"=\\u003e{\\\"hierarchical_memory_limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e9223372036854771712}, \\\"inactive_anon\\\"=\\u003e{\\\"bytes\\\"=\\u003e196608}, \\\"hierarchical_memsw_limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"rss\\\"=\\u003e{\\\"bytes\\\"=\\u003e136847360}, \\\"active_file\\\"=\\u003e{\\\"bytes\\\"=\\u003e14061568}, \\\"active_anon\\\"=\\u003e{\\\"bytes\\\"=\\u003e136912896}, \\\"pages_out\\\"=\\u003e1829846, \\\"unevictable\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"cache\\\"=\\u003e{\\\"bytes\\\"=\\u003e16076800}, \\\"inactive_file\\\"=\\u003e{\\\"bytes\\\"=\\u003e1753088}, \\\"rss_huge\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"mapped_file\\\"=\\u003e{\\\"bytes\\\"=\\u003e4501504}, \\\"swap\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"major_page_faults\\\"=\\u003e93, \\\"page_faults\\\"=\\u003e4291992, \\\"pages_in\\\"=\\u003e1867181}, \\\"id\\\"=\\u003e\\\"apache2.service\\\", \\\"path\\\"=\\u003e\\\"/system.slice/apache2.service\\\", \\\"mem\\\"=\\u003e{\\\"failures\\\"=\\u003e0, \\\"limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e9223372036854771712}, \\\"usage\\\"=\\u003e{\\\"max\\\"=\\u003e{\\\"bytes\\\"=\\u003e273903616}, \\\"bytes\\\"=\\u003e162803712}}, \\\"memsw\\\"=\\u003e{\\\"failures\\\"=\\u003e0, \\\"limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}, \\\"usage\\\"=\\u003e{\\\"bytes\\\"=\\u003e0, \\\"max\\\"=\\u003e{\\\"bytes\\\"=\\u003e0}}}, \\\"kmem\\\"=\\u003e{\\\"failures\\\"=\\u003e0, \\\"limit\\\"=\\u003e{\\\"bytes\\\"=\\u003e9223372036854771712}, \\\"usage\\\"=\\u003e{\\\"bytes\\\"=\\u003e9879552, \\\"max\\\"=\\u003e{\\\"bytes\\\"=\\u003e11296768}}}}, \\\"blkio\\\"=\\u003e{\\\"total\\\"=\\u003e{\\\"bytes\\\"=\\u003e15708160, \\\"ios\\\"=\\u003e1079}, \\\"id\\\"=\\u003e\\\"apache2.service\\\", \\\"path\\\"=\\u003e\\\"/system.slice/apache2.service\\\"}, \\\"id\\\"=\\u003e\\\"apache2.service\\\", \\\"path\\\"=\\u003e\\\"/system.slice/apache2.service\\\", \\\"cpu\\\"=\\u003e{\\\"path\\\"=\\u003e\\\"/system.slice/apache2.service\\\", \\\"cfs\\\"=\\u003e{\\\"period\\\"=\\u003e{\\\"us\\\"=\\u003e100000}, \\\"quota\\\"=\\u003e{\\\"us\\\"=\\u003e0}, \\\"shares\\\"=\\u003e1024}, \\\"rt\\\"=\\u003e{\\\"period\\\"=\\u003e{\\\"us\\\"=\\u003e0}, \\\"runtime\\\"=\\u003e{\\\"us\\\"=\\u003e0}}, \\\"stats\\\"=\\u003e{\\\"periods\\\"=\\u003e0, \\\"throttled\\\"=\\u003e{\\\"periods\\\"=\\u003e0, \\\"ns\\\"=\\u003e0}}, \\\"id\\\"=\\u003e\\\"apache2.service\\\"}, \\\"cpuacct\\\"=\\u003e{\\\"percpu\\\"=\\u003e{\\\"2\\\"=\\u003e138192136285, \\\"1\\\"=\\u003e130852173277}, \\\"stats\\\"=\\u003e{\\\"system\\\"=\\u003e{\\\"ns\\\"=\\u003e63410000000}, \\\"user\\\"=\\u003e{\\\"ns\\\"=\\u003e176630000000}}, \\\"id\\\"=\\u003e\\\"apache2.service\\\", \\\"path\\\"=\\u003e\\\"/system.slice/apache2.service\\\", \\\"total\\\"=\\u003e{\\\"ns\\\"=\\u003e269044309562}}}, \\\"state\\\"=\\u003e\\\"sleeping\\\", \\\"username\\\"=\\u003e\\\"www-data\\\", \\\"pgid\\\"=\\u003e1889}},
fields={\\\"env\\\"=\\u003e\\\"development\\\"},
beat={\\\"name\\\"=\\u003e\\\"elasticsearch\\\", \\\"hostname\\\"=\\u003e\\\"elasticsearch\\\", \\\"version\\\"=\\u003e\\\"6.3.0\\\"},
host={\\\"name\\\"=\\u003e\\\"elasticsearch\\\"}
1530769729': invalid boolean\\nunable to parse 'beats_input @timestamp=\\\"2018-07-05T05:48:49.887Z\\\",

fluend conf

<source>
  @type beats
  bind 127.0.0.1
  port 5044
  tag beats_input
</source>
<match beats_input>
  @type influxdb
  dbname metrics
  user admin
  password 12345
  flush_interval 10s # for testing.
  host localhost
  port 8086
</match>

Please help me to resolve this issue.

Thanks,
Ganeshbabu R