Use script Nicehash api for import data in influx

Hi to all, i have installed Telegraf, influx and grafana on docker for import data and create a dashboard.

now i need run a script for import json data to influx, but on telegraf i recived this error:

E! error loading config file /etc/telegraf/telegraf.conf: error parsing imput, undefined but requested input: imput

this the telegraf config

[global_tags]
[agent]
  interval = "60s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "10s"
  flush_jitter = "0s"
  precision = ""
  hostname = "telegraf"
  omit_hostname = false
[[outputs.influxdb]]
  urls = ["http://192.168.1.11:8086"]
  database = "influx"
  timeout = "5s"
  username = "telegraf"
  password = "pswwww"
[[inputs.cpu]]
  percpu = true
  totalcpu = true
  collect_cpu_time = false
  report_active = false
[[inputs.disk]]
  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.mem]]
[[inputs.processes]]
[[imput.exec]]
  commands = ["/home/manuel/run.sh"]
  data_format = "influx"

this the run.sh

#!/bin/bash
# Configuration
ORG="organi"
KEY="key"
SEC="secuekey"
API="https://api2.nicehash.com" #prod env
#API="https://api-test.nicehash.com" # test env

# Command
NHCLIENT="python nicehash.py -b $API -o $ORG -k $KEY -s $SEC"

# Run method
eval "$NHCLIENT -m GET -p '/main/api/v2/mining/rigs/activeWorkers'"; # -b '{json}'

and this the py script

from datetime import datetime
from time import mktime
import uuid
import hmac
import requests
import json
from hashlib import sha256
import optparse
import sys


class public_api:

    def __init__(self, host, verbose=False):
        self.host = host
        self.verbose = verbose

    def request(self, method, path, query, body):
        url = self.host + path
        if query:
            url += '?' + query

        if self.verbose:
            print(method, url)

        s = requests.Session()
        if body:
            body_json = json.dumps(body)
            response = s.request(method, url, data=body_json)
        else:
            response = s.request(method, url)

        if response.status_code == 200:
            return response.json()
        elif response.content:
            raise Exception(str(response.status_code) + ": " + response.reason + ": " + str(response.content))
        else:
            raise Exception(str(response.status_code) + ": " + response.reason)

    def get_current_global_stats(self):
        return self.request('GET', '/main/api/v2/public/stats/global/current/', '', None)

    def get_global_stats_24(self):
        return self.request('GET', '/main/api/v2/public/stats/global/24h/', '', None)

    def get_active_orders(self):
        return self.request('GET', '/main/api/v2/public/orders/active/', '', None)

    def get_active_orders2(self):
        return self.request('GET', '/main/api/v2/public/orders/active2/', '', None)

    def buy_info(self):
        return self.request('GET', '/main/api/v2/public/buy/info/', '', None)

    def get_algorithms(self):
        return self.request('GET', '/main/api/v2/mining/algorithms/', '', None)

    def get_markets(self):
        return self.request('GET', '/main/api/v2/mining/markets/', '', None)

    def get_currencies(self):
        return self.request('GET', '/main/api/v2/public/currencies/', '', None)

    def get_multialgo_info(self):
        return self.request('GET', '/main/api/v2/public/simplemultialgo/info/', '', None)

    def get_exchange_markets_info(self):
        return self.request('GET', '/exchange/api/v2/info/status', '', None)

    def get_exchange_trades(self, market):
        return self.request('GET', '/exchange/api/v2/trades', 'market=' + market, None)

    def get_candlesticks(self, market, from_s, to_s, resolution):
        return self.request('GET', '/exchange/api/v2/candlesticks', "market={}&from={}&to={}&resolution={}".format(market, from_s, to_s, resolution), None)

    def get_exchange_orderbook(self, market, limit):
        return self.request('GET', '/exchange/api/v2/orderbook', "market={}&limit={}".format(market, limit), None)

class private_api:

    def __init__(self, host, organisation_id, key, secret, verbose=False):
        self.key = key
        self.secret = secret
        self.organisation_id = organisation_id
        self.host = host
        self.verbose = verbose

    def request(self, method, path, query, body):

        xtime = self.get_epoch_ms_from_now()
        xnonce = str(uuid.uuid4())

        message = bytearray(self.key, 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(str(xtime), 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(xnonce, 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(self.organisation_id, 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(method, 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(path, 'utf-8')
        message += bytearray('\x00', 'utf-8')
        message += bytearray(query, 'utf-8')

        if body:
            body_json = json.dumps(body)
            message += bytearray('\x00', 'utf-8')
            message += bytearray(body_json, 'utf-8')

        digest = hmac.new(bytearray(self.secret, 'utf-8'), message, sha256).hexdigest()
        xauth = self.key + ":" + digest

        headers = {
            'X-Time': str(xtime),
            'X-Nonce': xnonce,
            'X-Auth': xauth,
            'Content-Type': 'application/json',
            'X-Organization-Id': self.organisation_id,
            'X-Request-Id': str(uuid.uuid4())
        }

        s = requests.Session()
        s.headers = headers

        url = self.host + path
        if query:
            url += '?' + query

        if self.verbose:
            print(method, url)

        if body:
            response = s.request(method, url, data=body_json)
        else:
            response = s.request(method, url)

        if response.status_code == 200:
            return response.json()
        elif response.content:
            raise Exception(str(response.status_code) + ": " + response.reason + ": " + str(response.content))
        else:
            raise Exception(str(response.status_code) + ": " + response.reason)

    def get_epoch_ms_from_now(self):
        now = datetime.now()
        now_ec_since_epoch = mktime(now.timetuple()) + now.microsecond / 1000000.0
        return int(now_ec_since_epoch * 1000)

    def algo_settings_from_response(self, algorithm, algo_response):
        algo_setting = None
        for item in algo_response['miningAlgorithms']:
            if item['algorithm'] == algorithm:
                algo_setting = item

        if algo_setting is None:
            raise Exception('Settings for algorithm not found in algo_response parameter')

        return algo_setting

    def get_accounts(self):
        return self.request('GET', '/main/api/v2/accounting/accounts2/', '', None)

    def get_accounts_for_currency(self, currency):
        return self.request('GET', '/main/api/v2/accounting/account2/' + currency, '', None)

    def get_withdrawal_addresses(self, currency, size, page):

        params = "currency={}&size={}&page={}".format(currency, size, page)

        return self.request('GET', '/main/api/v2/accounting/withdrawalAddresses/', params, None)

    def get_withdrawal_types(self):
        return self.request('GET', '/main/api/v2/accounting/withdrawalAddresses/types/', '', None)

    def withdraw_request(self, address_id, amount, currency):
        withdraw_data = {
            "withdrawalAddressId": address_id,
            "amount": amount,
            "currency": currency
        }
        return self.request('POST', '/main/api/v2/accounting/withdrawal/', '', withdraw_data)

    def get_my_active_orders(self, algorithm, market, limit):

        ts = self.get_epoch_ms_from_now()
        params = "algorithm={}&market={}&ts={}&limit={}&op=LT".format(algorithm, market, ts, limit)

        return self.request('GET', '/main/api/v2/hashpower/myOrders', params, None)

    def create_pool(self, name, algorithm, pool_host, pool_port, username, password):
        pool_data = {
            "name": name,
            "algorithm": algorithm,
            "stratumHostname": pool_host,
            "stratumPort": pool_port,
            "username": username,
            "password": password
        }
        return self.request('POST', '/main/api/v2/pool/', '', pool_data)

    def delete_pool(self, pool_id):
        return self.request('DELETE', '/main/api/v2/pool/' + pool_id, '', None)

    def get_my_pools(self, page, size):
        return self.request('GET', '/main/api/v2/pools/', '', None)

    def get_hashpower_orderbook(self, algorithm):
        return self.request('GET', '/main/api/v2/hashpower/orderBook/', 'algorithm=' + algorithm, None )
    
    def create_hashpower_order(self, market, type, algorithm, price, limit, amount, pool_id, algo_response):

        algo_setting = self.algo_settings_from_response(algorithm, algo_response)

        order_data = {
            "market": market,
            "algorithm": algorithm,
            "amount": amount,
            "price": price,
            "limit": limit,
            "poolId": pool_id,
            "type": type,
            "marketFactor": algo_setting['marketFactor'],
            "displayMarketFactor": algo_setting['displayMarketFactor']
        }
        return self.request('POST', '/main/api/v2/hashpower/order/', '', order_data)

    def cancel_hashpower_order(self, order_id):
        return self.request('DELETE', '/main/api/v2/hashpower/order/' + order_id, '', None)

    def refill_hashpower_order(self, order_id, amount):
        refill_data = {
            "amount": amount
        }
        return self.request('POST', '/main/api/v2/hashpower/order/' + order_id + '/refill/', '', refill_data)

    def set_price_hashpower_order(self, order_id, price, algorithm, algo_response):

        algo_setting = self.algo_settings_from_response(algorithm, algo_response)

        price_data = {
            "price": price,
            "marketFactor": algo_setting['marketFactor'],
            "displayMarketFactor": algo_setting['displayMarketFactor']
        }
        return self.request('POST', '/main/api/v2/hashpower/order/' + order_id + '/updatePriceAndLimit/', '',
                            price_data)

    def set_limit_hashpower_order(self, order_id, limit, algorithm, algo_response):
        algo_setting = self.algo_settings_from_response(algorithm, algo_response)
        limit_data = {
            "limit": limit,
            "marketFactor": algo_setting['marketFactor'],
            "displayMarketFactor": algo_setting['displayMarketFactor']
        }
        return self.request('POST', '/main/api/v2/hashpower/order/' + order_id + '/updatePriceAndLimit/', '',
                            limit_data)

    def set_price_and_limit_hashpower_order(self, order_id, price, limit, algorithm, algo_response):
        algo_setting = self.algo_settings_from_response(algorithm, algo_response)

        price_data = {
            "price": price,
            "limit": limit,
            "marketFactor": algo_setting['marketFactor'],
            "displayMarketFactor": algo_setting['displayMarketFactor']
        }
        return self.request('POST', '/main/api/v2/hashpower/order/' + order_id + '/updatePriceAndLimit/', '',
                            price_data)

    def get_my_exchange_orders(self, market):
        return self.request('GET', '/exchange/api/v2/myOrders', 'market=' + market, None)

    def get_my_exchange_trades(self, market):
        return self.request('GET','/exchange/api/v2/myTrades', 'market=' + market, None)

    def create_exchange_limit_order(self, market, side, quantity, price):
        query = "market={}&side={}&type=limit&quantity={}&price={}".format(market, side, quantity, price)
        return self.request('POST', '/exchange/api/v2/order', query, None)

    def create_exchange_buy_market_order(self, market, quantity):
        query = "market={}&side=buy&type=market&secQuantity={}".format(market, quantity)
        return self.request('POST', '/exchange/api/v2/order', query, None)

    def create_exchange_sell_market_order(self, market, quantity):
        query = "market={}&side=sell&type=market&quantity={}".format(market, quantity)
        return self.request('POST', '/exchange/api/v2/order', query, None)

    def cancel_exchange_order(self, market, order_id):
        query = "market={}&orderId={}".format(market, order_id)
        return self.request('DELETE', '/exchange/api/v2/order', query, None)


if __name__ == "__main__":
    parser = optparse.OptionParser()

    parser.add_option('-b', '--base_url', dest="base", help="Api base url", default="https://api2.nicehash.com")
    parser.add_option('-o', '--organization_id', dest="org", help="Organization id")
    parser.add_option('-k', '--key', dest="key", help="Api key")
    parser.add_option('-s', '--secret', dest="secret", help="Secret for api key")
    parser.add_option('-m', '--method', dest="method", help="Method for request", default="GET")
    parser.add_option('-p', '--path', dest="path", help="Path for request", default="/")
    parser.add_option('-q', '--params', dest="params", help="Parameters for request")
    parser.add_option('-d', '--body', dest="body", help="Body for request")

    options, args = parser.parse_args()

    private_api = private_api(options.base, options.org, options.key, options.secret)

    params = ''
    if options.params is not None:
        params = options.params

    try:
        response = private_api.request(options.method, options.path, params, options.body)
    except Exception as ex:
        print("Unexpected error:", ex)
        exit(1)

    print(response)
    exit(0)

i modify for read a local json and take some data, i need to import all data.

this my conf

[[input.file]]
  files = [file.json]
  data_format = "json"
  name_suffix = "_alldata"

this is an json example

{
  "pagination": {
    "page": 0,
    "totalPageCount": 1,
    "size": 100
  },
  "totalDevices": 1,
  "totalRigs": 1,
  "externalBalance": "0.00046461",
  "totalProfitabilityLocal": 0,
  "nextPayoutTimestamp": "2020-10-12T20:00:00Z",
  "unpaidAmount": "0.00000508",
  "totalProfitability": 9.000394182612861e-7,
  "miningRigGroups": [],
  "miningRigs": [
    {
      "stats": [
        {
          "timeConnected": 1602490067125,
          "speedRejectedR4NTime": 0,
          "xnsub": true,
          "algorithm": {
            "enumName": "SCRYPT",
            "description": "Scrypt"
          },
          "proxyId": 0,
          "speedRejectedTotal": 0,
          "speedRejectedR2Stale": 0,
          "speedAccepted": 1259.8570734933332,
          "profitability": 0,
          "speedRejectedR1Target": 0,
          "unpaidAmount": "0.00000007",
          "difficulty": 262144,
          "speedRejectedR5Other": 0,
          "statsTime": 1602527470000,
          "speedRejectedR3Duplicate": 0,
          "market": "USA"
        }
      ],
      "name": "worker1",
      "profitability": 9.000394182612861e-7,
      "unpaidAmount": "0.00000508",
      "notifications": [],
      "rigId": "worker1",
      "localProfitability": 0,
      "minerStatus": "MINING",
      "statusTime": 1602527470000,
      "type": "UNMANAGED"
    }
  ],
  "devicesStatuses": {
    "MINING": 1
  },
  "rigTypes": {
    "UNMANAGED": 1
  },
  "minerStatuses": {
    "MINING": 1
  },
  "externalAddress": true,
  "path": "",
  "rigNhmVersions": [],
  "btcAddress": "2NFVHDwJmCdwvqr3yzQLPsPXQE3pvaGxNHR"
}

with this config i see some data but not all

Hello @manustar,
It looks like your error is related to using a m instead of an n

[[imput.exec]]
  commands = ["/home/manuel/run.sh"]
  data_format = "influx"

Should be

[[input.exec]]
  commands = ["/home/manuel/run.sh"]
  data_format = "influx"

yes, the “m” error is only in the post, it’s written right in the configuration. the problem I have is in the data it retrieves, I also tried to save locally but it doesn’t get the right data, or rather not all the ones I need. I don’t know if there are any filters or tricks to adopt for data extraction

data_format = “json”

The json data format parser is very naive and simple. It works best with flat, simple JSON data.

The example data you provided is more complex. You might take a look at the json_v2 parser instead and define objects, tags, and fields, that you are interested in collecting.

I will say, that it can become rather involved with more complex configurations. At which point, I would suggest looking at the InfluxDB Client libraries. There you can pick your favorite language, write some code to parse the JSON and build line protocol however you wish and push to InfluxDB.

I had imagined it was more complex. I tried with jsonv2 directly from a file and let’s say it extrapolates data but not what I need, I’m not very practical in programming. could you give me a schematic for how to set the tags to extract the data related to totaldevice, totalrigs2 is general data and stats for single rig?

I tried with jsonv2 directly from a file and let’s say it extrapolates data but not what I need,

At a high-level you would want to use the json_v2 plugin + define the miningRigs as an object you want parsed and define any tags and fields you want under that object. Then add the totalDevices and totalRigs as tags as well.

Give that a shot, and if you still need help could you please provide your config that you are using.

thisa an exanple:

genenalinfo
totalrigs—totaldevice----devicestatuses----rigtypes—minerstatuses

rigsinformation
name----type----market—timeconnected----altorithmdetail

[[inputs.exec]]
commands: =["path/run.sh]]
name_suffix = “_mydata”
data_format = json_v2
[[inputs.exec.json_v2]]
tags = [“totalDevice”, “totalRigs”, “deviceStatuses”, ecc]
[[inputs.exec.json_v2.object]]
path = “miningRigs”
tags = [“stats”]

this explample is rigth?

tags = [“totalDevice”, “totalRigs”, “deviceStatuses”, ecc]

Looks like a good start, except not sure what that unquoted “ecc” is at the end?

If you run telegraf with the --test option, which ignored outputs, and will instead print the generated metrics to the console, what do you get?

im test it and this error on test mode

E! error loading config file /etc/telegraf/telegraf.conf: error parsing exec, adding parser failed: line 4776: cannot unmarshal TOML table into json_v2.Config (need struct or map)

this telegraf config

[[inputs.exec]]
#   ## Commands array
 commands = ["/home/user/niceapiexport/run.sh"]
#     "/usr/bin/mycollector --foo=bar",
#     "/tmp/collect_*.sh"
#  ]
 name_suffix = "_apinice"
 data_format ="json_v2"
[[input.exec.json_v2]]
 measurement_name = "Infoall"
 tags = ["minerStatuses", "rigTypes", "totalRigs", "deviceStatuses"]
 [[inputs.exec.json_v2.field]]
  path = "minerStatuses"
 [[inputs.exec.json_v2.field]]
  path = "mininRigs"
  tags = ["stats"]

tags = [“minerStatuses”, “rigTypes”, “totalRigs”, “deviceStatuses”]

  • minerStatuses devicesStatuses, and rigTypes are all JSON objects so you need to to provide a bit more details in the path than saying it is a field.
  • Tags are normally strings and totalRigs is an int so you probably want this as a field
  • deviceStatuses is misspelled and should be devicesStatuses

I would start with something like and see if it works for your situation:

  [[inputs.file.json_v2]]
    measurement_name = "Infoall"
    [[inputs.file.json_v2.field]]
      path = "minerStatuses.MINING"
      rename = "miner_mining"
    [[inputs.file.json_v2.field]]
      path = "devicesStatuses.MINING"
      rename = "status_mining"
    [[inputs.file.json_v2.field]]
      path = "rigTypes.UNMANAGED"
      rename = "type_unmanaged"
    [[inputs.file.json_v2.field]]
      path = "totalRigs"

    [[inputs.file.json_v2.object]]
      path = "miningRigs"

which produces the following and continue to adapt it:

Infoall_apinice,host=ryzen name="worker1",profitability=0.0000009000394182612861,unpaidAmount="0.00000508",rigId="worker1",localProfitability=0,minerStatus="MINING",statusTime=1602527470000,type="UNMANAGED",stats_timeConnected=1602490067125,stats_speedRejectedR4NTime=0,stats_xnsub=true,stats_algorithm_enumName="SCRYPT",stats_algorithm_description="Scrypt",stats_proxyId=0,stats_speedRejectedTotal=0,stats_speedRejectedR2Stale=0,stats_speedAccepted=1259.8570734933332,stats_profitability=0,stats_speedRejectedR1Target=0,stats_unpaidAmount="0.00000007",stats_difficulty=262144,stats_speedRejectedR5Other=0,stats_statsTime=1602527470000,stats_speedRejectedR3Duplicate=0,stats_market="USA",miner_mining=1,status_mining=1,type_unmanaged=1,totalRigs=1 1682001663000000000

You are most likely going to have to expand the various fields for miner status and device status. That JSON is not very friendly to parsing as I assume there are other values, but not always present?

now i see this error “E! [inputs.exec] Error in plugin: invalid JSON provided, unable to parse: {‘minerStatuses’: {‘STOPPED’: 2, ‘MINING’: 13}, ‘rigTypes’: {‘UNMANAGED’: 1, ‘MANAGED’: 14} ------ continue see all data from json”