Unable to tag when using http plugin with nested json

Telegraf
1

Hi,
I am trying to parse json data that I get from a Citrix ADC appliance REST API servicegroup_binding endpoint.

I am getting the data properly, but there is no way I can set a tag.
For other API endpoints tags work properly, but not with this one.
I am not sure if this is real bug, or just my wrong configuration, so I wanted to post it here first, before creating a Github issue.
As shown in example this API endpoint gives nested “list in a list” json data (as opposed to more flat json structure for other endpoints), so my assumption is that this may be part of the reason.
Can anyone hint me the right configuration ?

This is my test configuration:

[[inputs.http]]
  urls = [
        "https://citrixadc/nitro/v1/config/servicegroup_binding?bulkbindings=yes"
        ]
  method = "GET"

  cookie_auth_url = "https://citrixadc/nitro/v1/config/login"
  cookie_auth_method = "POST"
  cookie_auth_headers = { Content-Type = "application/json"}
  cookie_auth_body = '{"login":{"username": "username", "password":"password"}}'
  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
  cookie_auth_renewal = "24h"
  data_format = "json_v2"

  insecure_skip_verify = true

  tagexclude = ["url","host"]

  fieldpass = ["servicegroupname", "servername", "svrstate", "ip", "port"]

[[inputs.http.json_v2]]
        measurement_name = "servicegroup_binding"
        #measurement_name_path = "servicegroup_binding"

        [[inputs.http.json_v2.object]]
                path = "servicegroup_binding.#.servicegroup_servicegroupmember_binding"
               #fieldpass = ["servicegroupname", "servername", "svrstate"]

                [[inputs.http.json_v2.object.tag]]
                        path = "#.servicegroupname"
                        #type = "string"

                [[inputs.http.json_v2.object.field]]
                        path = "#.servername"
                        type = "string"

                [[inputs.http.json_v2.object.field]]
                        path = "#.ip"
                        type = "string"

                [[inputs.http.json_v2.object.field]]
                        path = "#.port"
                        type = "string"

                [[inputs.http.json_v2.object.field]]
                        path = "#.svrstate"
                        type = "string"

[inputs.http.tags]
device_type="netscaler"
hostname="citrixadc"

Here is some test output:

> servicegroup_binding,device_type=netscaler,hostname=citrixadc ip="172.1.1.1",port="81",servername="server1",servicegroupname="servicegroupname1",svrstate="UP" 1667904366000000000
> servicegroup_binding,device_type=netscaler,hostname=citrixadc ip="172.1.1.2",port="444",servername="server2",servicegroupname="servicegroupname2",svrstate="UP" 1667904366000000000
> servicegroup_binding,device_type=netscaler,hostname=citrixadc ip="172.1.1.1",port="444",servername="server1",servicegroupname="servicegroupname2",svrstate="UP" 1667904366000000000

Here is example output when using curl:

{ "errorcode": 0, "message": "Done", "severity": "NONE", "servicegroup_binding": 
[ 
{ "servicegroupname": "servicegroupname1", 
"servicegroup_lbmonitor_binding": [ { "servicegroupname": "servicegroupname1", "monitor_name": "monitor_name1", "monweight": "0", "monstate": "ENABLED", "weight": "1", "state": "ENABLED", "passive": false } ], 
"servicegroup_servicegroupmember_binding": [ { "servicegroupname": "servicegroupname1", "ip": "10.1.1.1", "port": 11111, "svrstate": "DOWN", "statechangetimesec": "Thu Oct 27 20:28:33 2022", "tickssincelaststatechange": "99450698", "weight": "1", "servername": "servername1", "customserverid": "None", "serverid": "0", "state": "ENABLED", "hashid": "0", "graceful": "NO", "delay": 0, "delay1": 0, "nameserver": "0.0.0.0", "dbsttl": 0 } ], 
"servicegroup_servicegroupentitymonbindings_binding": [ { "servicegroupname": "servicegroupname1", "servicegroupentname2": "servicegroupname1?servername1?11111", "monitor_name": "monitor_name1", "monitor_state": "UNKNOWN", "passive": false, "monitortotalprobes": "0", "monitortotalfailedprobes": "0", "monitorcurrentfailedprobes": "0", "lastresponse": "Probe skipped - node secondary.", "responsetime": "0" } ] 
},
{ "servicegroupname": "servicegroupname2",
"servicegroup_lbmonitor_binding": [ { "servicegroupname": "servicegroupname2", "monitor_name": "monitor_name2", "monweight": "0", "monstate": "ENABLED", "weight": "1", "state": "ENABLED", "passive": false } ],
"servicegroup_servicegroupmember_binding": [ { "servicegroupname": "servicegroupname2", "ip": "10.1.1.2", "port": 22222, "svrstate": "DOWN", "statechangetimesec": "Thu Oct 27 20:28:33 2022", "tickssincelaststatechange": "99450699", "weight": "1", "servername": "servername2", "customserverid": "None", "serverid": "0", "state": "ENABLED", "hashid": "0", "graceful": "NO", "delay": 0, "delay1": 0, "nameserver": "0.0.0.0", "dbsttl": 0 } ],
"servicegroup_servicegroupentitymonbindings_binding": [ { "servicegroupname": "servicegroupname2", "servicegroupentname2": "servicegroupname2?servername2?22222", "monitor_name": "monitor_name2", "monitor_state": "UNKNOWN", "passive": false, "monitortotalprobes": "0", "monitortotalfailedprobes": "0", "monitorcurrentfailedprobes": "0", "lastresponse": "Probe skipped - node secondary.", "responsetime": "0" } ]
},
{ "servicegroupname": "servicegroupname3",
"servicegroup_servicegroupmember_binding": [ { "servicegroupname": "servicegroupname3", "ip": "10.1.1.3", "port": 21, "svrstate": "UP", "statechangetimesec": "Thu Oct 27 20:28:36 2022", "tickssincelaststatechange": "99450460", "weight": "1", "servername": "servername3", "customserverid": "None", "serverid": "0", "state": "ENABLED", "hashid": "0", "graceful": "NO", "delay": 0, "delay1": 0, "nameserver": "0.0.0.0", "dbsttl": 0 }, { "servicegroupname": "servicegroupname3", "ip": "10.1.1.3", "port": 21, "svrstate": "UP", "statechangetimesec": "Thu Oct 27 20:28:36 2022", "tickssincelaststatechange": "99450460", "weight": "1", "servername": "servername4", "customserverid": "None", "serverid": "0", "state": "ENABLED", "hashid": "0", "graceful": "NO", "delay": 0, "delay1": 0, "nameserver": "0.0.0.0", "dbsttl": 0 } ],
"servicegroup_servicegroupentitymonbindings_binding": [ { "servicegroupname": "servicegroupname3", "servicegroupentname2": "servicegroupname3?servername3?21", "monitor_name": "tcp-default", "monitor_state": "UNKNOWN", "passive": false, "monitortotalprobes": "0", "monitortotalfailedprobes": "0", "monitorcurrentfailedprobes": "0", "lastresponse": "Probe skipped - node secondary.", "responsetime": "0" }, { "servicegroupname": "servicegroupname3", "servicegroupentname2": "servicegroupname3?servername4?21", "monitor_name": "tcp-default", "monitor_state": "UNKNOWN", "passive": false, "monitortotalprobes": "0", "monitortotalfailedprobes": "0", "monitorcurrentfailedprobes": "0", "lastresponse": "Probe skipped - node secondary.", "responsetime": "0" } ] }
2

Hi @SpiderD555! Welcome to the community forum.

I’ve parsed nested JSON that was nested one-deep. It was tricky and the JSON Playground helped to get the correct path = . . .. Here’s a snippet from that project. I’ve changed it somewhat to appear more generic. Each parsing block drills down to the item(s) I’m looking for using a #pattern match# terminated with @this to pick off the data at the proper nesting-depth.

Hope it helps.

    [[inputs.http.json_v2.object]]
      path = "devices.#(DEVICE_TYPE==\"Device1\")#.@this"
      included_keys = ["key1", "key2"]
      tags = ["subtype"]
      [inputs.http.json_v2.object.fields]
        key1 = "float"

    [[inputs.http.json_v2.object]]
      path = "devices.#(DEVICE_TYPE==\"Device2\")#.@this"
      included_keys = ["key3"]
      tags = ["SERIAL"]
      [inputs.http.json_v2.object.renames]
        SERIAL = "serial_num"
        key3 = "meaningful name"
      [inputs.http.json_v2.object.fields]
        key3 = "float"
3

Hi @phill
Thank you very much for your tips. Thanks to you I was able to create a properly working configuration.
Here is the solution if anyone needs it:

[[inputs.http]]
  urls = [
        "https://citrixadc/nitro/v1/config/servicegroup_binding?bulkbindings=yes"
        ]
  method = "GET"

  cookie_auth_url = "https://citrixadc/nitro/v1/config/login"
  cookie_auth_method = "POST"
  cookie_auth_headers = { Content-Type = "application/json"}
  cookie_auth_body = '{"login":{"username": "login", "password":"password"}}'
  ## cookie_auth_renewal not set or set to "0" will auth once and never renew the cookie
  cookie_auth_renewal = "24h"
  data_format = "json_v2"

  insecure_skip_verify = true

  tagexclude = ["url","host"]

[[inputs.http.json_v2]]
        measurement_name = "servicegroup_binding"

        [[inputs.http.json_v2.object]]
                path = "servicegroup_binding.#.servicegroup_servicegroupmember_binding"
                included_keys = ["servername", "ip", "port", "svrstate"]
                tags = ["servicegroupname"]

                [inputs.http.json_v2.object.fields]
                        servername = "string"
                        ip = "string"
                        port = "string"
                        svrstate = "string"