Telegraf - how get JSON data (JSON with big and difficult structure) from api url correctrly

vladimir.n.kostyanik
Fri, 01 Nov 2024 11:50:10 +0000
Telegraf

Hello!
I want to get some metrics from device with telegraf. Device has API (http://10.1.0.1/status) and i can to get JSON data from it. APi returns JSON with difficult structure (several nested arrays and some of these arrays do not have an explicit key/value pair, they can be accessed by index.)
My JSON:

{
    "instance-id": "DEV-MLFF-ANPR",
    "api-version": "2.1",
    "devices:cameras:status": {
        "framerates": [
            "23.8095",
            "23.8095",
            "25",
            "25",
            "25"
        ],
        "names": [
            "front_cam_1",
            "front_cam_2",
            "rear_cam_1",
            "rear_cam_2",
            "side_cam"
        ],
        "statuses": [
            "live",
            "live",
            "live",
            "live",
            "live"
        ],
        "timestamps": [
            "1730371112186",
            "1730371112186",
            "1730371112186",
            "1730371112186",
            "1730371112186"
        ],
        "module-id": "devices:cameras:status",
        "timestamp": "1730371131742",
        "duration": "60054"
    },
    "lpr:app:cpu-load": {
        "load": "162.01815002913995",
        "module-id": "lpr:app:cpu-load",
        "timestamp": "1730371131737",
        "duration": "60055"
    },
    "lpr:app:memory-usage": {
        "vm-rss": "7561860",
        "vm-size": "13503580",
        "module-id": "lpr:app:memory-usage",
        "timestamp": "1730371131724",
        "duration": "60058"
    },
    "lpr:app:ram-disk:usage": {
        "available": "6011834368",
        "used": "430616576",
        "module-id": "lpr:app:ram-disk:usage",
        "timestamp": "1730371131740",
        "duration": "60055"
    },
    "lpr:app:run": {
        "state": "online",
        "module-id": "lpr:app:run",
        "timestamp": "1730371131740",
        "duration": "60055"
    },
    "lpr:transits:statistic": {
        "scores": "",
        "sides": "",
        "statuses": "",
        "module-id": "lpr:transits:statistic",
        "timestamp": "1730371131742",
        "duration": "255990617"
    },
    "system:cpu:load": {
        "by-cores": [
            "3.3579583613163195",
            "0.51675279213202197",
            "12.247622225930252",
            "7.4061012167166282",
            "17.817749872730356",
            "0.68247739293635901",
            "0.71538068472151251",
            "27.511281965569111",
            "0.93442349407642245",
            "8.7094617184888001",
            "17.397933254277486",
            "0.39993334444259288",
            "12.196861626248216",
            "3.5655210439257488",
            "19.102435769102435",
            "12.559081701553005",
            "9.7630173564753004",
            "12.318596017549782",
            "8.3138953682105967",
            "13.744315310763012"
        ],
        "cores": "20",
        "total": "9.4849661675174257",
        "module-id": "system:cpu:load",
        "timestamp": "1730371131723",
        "duration": "60058"
    },
    "system:cpu:thermal": {
        "cpu-package-0": "33",
        "module-id": "system:cpu:thermal",
        "timestamp": "1730371131723",
        "duration": "60058"
    },
    "system:disk:usage": {
        "available": "184195661824",
        "used": "58732937216",
        "module-id": "system:disk:usage",
        "timestamp": "1730371131740",
        "duration": "60055"
    },
    "system:disks:smart": {
        "names": [
            "sdc",
            "storage-2",
            "main"
        ],
        "sysnames": [
            "sdc",
            "sdb",
            "sda"
        ],
        "temperatures": [
            "41",
            "46",
            "46"
        ],
        "timestamps": [
            "1730371095000",
            "1730371094000",
            "1730371094000"
        ],
        "module-id": "system:disks:smart",
        "timestamp": "1730371131742",
        "duration": "60054"
    },
    "system:memory:usage": {
        "hardware-corrupted": "0",
        "mem-active": "1033092",
        "mem-available": "54069480",
        "mem-free": "867168",
        "mem-inactive": "60021148",
        "mem-shared": "586012",
        "mem-total": "65569080",
        "swap-free": "16711932",
        "swap-total": "16777212",
        "module-id": "system:memory:usage",
        "timestamp": "1730371131723",
        "duration": "60058"
    }
}

I create config and can get some metrics, my config:

[inputs]
[[inputs.http]]
# this section working and return data
  urls = [ "http://10.1.0.1/status" ]
  method = "GET"
  name_override = "device_info"
  json_string_fields = ["instance-id", "api-version"]
  data_format = "json"
  success_status_codes = [200]
  [inputs.http.tags]
    devid = "64"
[[inputs.http]]
# this section IS NOT working and return data
  urls = [ "http://10.1.0.1/status" ]
  method = "GET"
  name_override = "device_data_status"
  data_format = "json"
  json_string_fields = ["devices:cameras:status.framerates"]
  json_query = "devices:cameras:status.framerates.0"
  success_status_codes = [200]
  [inputs.http.tags]
    devid = "64"

[[inputs.http]]
# this section working and return data
  urls = [ "http://10.1.0.1/status" ]
  method = "GET"
  name_override = "device_data_mem"
  data_format = "json"
  json_string_fields = ["mem-active", "mem-available", "mem-free", "mem-total", "swap-free", "swap-total"]
  json_query = "system:memory:usage"
  success_status_codes = [200]
  [inputs.http.tags]
    devid = "64"

When i tested config file i get next data:

device_data_mem,devid=64,host=smit-dev1,url=http://10.1.0.1/status mem-active="5728700",mem-available="52924888",mem-free="455340",mem-total="65569080",swap-free="16440572",swap-total="16777212" 1730461094000000000
> device_info,devid=64,host=smit-dev1,url=http://10.1.0.1/status api-version="2.1",instance-id="DEV-MLFF-ANPR" 1730461094000000000

and i recieve error: Error running agent: input plugins recorded 1 errors because second block my config file (it is there has string devices:cameras:status.framerates.0) try to collect data and see array and can not process it, because my array has no keys, my array has indexs and values.
I use gjson.dev - to get right string for json_query [1] but it is not working
Help me please, I am at a dead end.
If you can suggest another solution (any option to get all data from JSON) i will thanks you very much

4 posts - 2 participants