Struggle with parsing json array(s) from MQTT

Telegraf
,
1

A rather newbie in telegraf config, been struggling for a while to get telegraf to parse few arrays from a longer json over mqtt. Have been skimming through the forums and google and got to a (I think) quite decent results with json_v2 parser. For sure I am still doing many things wrong and I realized it could be anything - from indentation to wrong syntax or grouping of instructions. My appologies upfront for that.
(I also hope a little bit that - as it has happened in past many times - I solve the issue just by typing it out :D)

Telegraf: 1.28.2
Influxdb: 2.1.1
Mosquitto: 2.0.15

Here is part of a longer json (few more arrays and fields inside it). Basically it’s a 12 port power distribution unit.
topic: aa/client/cc/status/device-99
json:

{
"outputCurrent": [
        1.0,
        2.0,
        3.0,
        4.0,
        5.0,
        6.0,
        7.0,
        8.0,
        9.0,
        10.0,
        11.0,
        12.0
    ],
    "outputPower": [
        101.0,
        102.0,
        103.0,
        104.0,
        105.0,
        106.0,
        107.0,
        108.0,
        109.0,
        110.0,
        111.0,
        112.0
    ],
    "totalEnergy": 95
}

and the for parsing the array I have to groups of mqtt consumer (initially I was trying to do it within one group, repeating topic_parsing instructions with their owen object/field(s), but it did not work for me).


[[inputs.mqtt_consumer]]
 servers = ["tcp://mosquitto:1883"]
 username="***"
 password="***"
 topics = [
   "aa/+/cc/status/+"
 ]
 
 data_format="json_v2"
 tagexclude = ["topic"]

  [[inputs.mqtt_consumer.topic_parsing]]
      topic = "aa/+/cc/status/+"  
      tags = "_/client/_/_/device"

    [[inputs.mqtt_consumer.json_v2]]
      [[inputs.mqtt_consumer.json_v2.object]]
      path = "outputPower"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "0"
      rename = "C1power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "1"
      rename = "C2power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "2"
      rename = "C3power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "3"
      rename = "C4power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "4"
      rename = "C5power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "5"
      rename = "C6power"
      type = "float"            
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "6"
      rename = "C7power"
      type = "float"      
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "7"
      rename = "C8power"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "8"
      rename = "C9power"
      type = "float"      
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "9"
      rename = "C10power"
      type = "float"            
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "10"
      rename = "C11power"
      type = "float" 
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "11"
      rename = "C12power"
      type = "float"

[[inputs.mqtt_consumer]]
 servers = ["tcp://mosquitto:1883"]
 username="***"
 password="***"
 topics = [
   "aa/+/cc/status/+"
 ]
 data_format="json_v2"
 tagexclude = ["topic"]
  [[inputs.mqtt_consumer.topic_parsing]]
      topic = "aa/+/cc/status/+"  
      tags = "_/client/_/_/device"

    [[inputs.mqtt_consumer.json_v2]]
      [[inputs.mqtt_consumer.json_v2.object]]
      path = "outputCurrent"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "0"
      rename = "C1current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "1"
      rename = "C2current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "2"
      rename = "C3current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "3"
      rename = "C4current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "4"
      rename = "C5current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "5"
      rename = "C6current"
      type = "float"            
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "6"
      rename = "C7current"
      type = "float"      
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "7"
      rename = "C8current"
      type = "float"
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "8"
      rename = "C9current"
      type = "float"      
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "9"
      rename = "C10current"
      type = "float"        
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "10"
      rename = "C11current"
      type = "float" 
      [[inputs.mqtt_consumer.json_v2.object.field]]
      path = "11"
      rename = "C12current"
      type = "float"

This is generally working most of the time. Untill I spotted from time to time some values are missing the updates. E.g. this would be a good looking output in telegraf log

mqtt_consumer,client=some,device=device-99 C1current=1 1698133034467903446
mqtt_consumer,client=some,device=device-99 C2current=2 1698133034467903446
mqtt_consumer,client=some,device=device-99 C3current=3 1698133034467903446
mqtt_consumer,client=some,device=device-99 C4current=4 1698133034467903446
mqtt_consumer,client=some,device=device-99 C5current=5 1698133034467903446
mqtt_consumer,client=some,device=device-99 C6current=6 1698133034467903446
mqtt_consumer,client=some,device=device-99 C7current=7 1698133034467903446
mqtt_consumer,client=some,device=device-99 C8current=8 1698133034467903446
mqtt_consumer,client=some,device=device-99 C9current=9 1698133034467903446
mqtt_consumer,client=some,device=device-99 C10current=10 1698133034467903446
mqtt_consumer,client=some,device=device-99 C11current=11 1698133034467903446
mqtt_consumer,client=some,device=device-99 C12current=12 1698133034467903446
mqtt_consumer,client=some,device=device-99 C1power=101 1698133034468199389
mqtt_consumer,client=some,device=device-99 C2power=102 1698133034468199389
mqtt_consumer,client=some,device=device-99 C3power=103 1698133034468199389
mqtt_consumer,client=some,device=device-99 C4power=104 1698133034468199389
mqtt_consumer,client=some,device=device-99 C5power=105 1698133034468199389
mqtt_consumer,client=some,device=device-99 C6power=106 1698133034468199389
mqtt_consumer,client=some,device=device-99 C7power=107 1698133034468199389
mqtt_consumer,client=some,device=device-99 C8power=108 1698133034468199389
mqtt_consumer,client=some,device=device-99 C9power=109 1698133034468199389
mqtt_consumer,client=some,device=device-99 C10power=110 1698133034468199389
mqtt_consumer,client=some,device=device-99 C11power=111 1698133034468199389
mqtt_consumer,client=some,device=device-99 C12power=112 1698133034468199389

On a bad case I could see something like this (some C# beeing missed/the other beeing “overwritten” two times):

...
mqtt_consumer,client=some,device=device-99 C7current=7 1698133034467903446
mqtt_consumer,client=some,device=device-99 C10current=8 1698133034467903446
mqtt_consumer,client=some,device=device-99 C9current=9 1698133034467903446
mqtt_consumer,client=some,device=device-99 C10current=10 1698133034467903446
mqtt_consumer,client=some,device=device-99 C11current=11 1698133034467903446
mqtt_consumer,client=some,device=device-99 C12current=12 1698133034467903446

Long story short, two questions, I guess - does the configuration script part generally looks ok?
Any idea/troubleshooting hints how to chase & debug a behaviour like the above?

Appreciated a lot and Thank You for reading this far :slight_smile:

2

Hello @austrisv,
It looks good to me.
You could try adding another tag (like [inputs.mqtt_consumer.tags] or field to add more information to your line protocol to avoid overwriting. But if that is worth the increase in cardinality depends on your use case.

Ex:
You can add static tags like:

[[inputs.snmp]] # other stuff 
  [inputs.snmp.tags] site=“EZE”

Make sure you put the table after any options to the plugin.

All that said it might be better to use a snmp field:

[[inputs.snmp]] # other stuff
   [[inputs.snmp.field]] 
   name = "sys_name"
   oid = "SNMPv2-MIB::sysName.0" 
    is_tag = true