Subscriber problems & memory growing

,
1

Hi there. I use Influxdb with Kapacitor(for alerting) on single machine in different docker containers with host net.
I have about 500k points writes(over UDP) every 10 minutes. This measurement has CQ for downsampling every 1 hour(about 3 millions points). And this new measurement are downsampled every 1d(about 12 million points).
And I have this TICK script:

dbrp "prices"."one_hour"

var prices = stream
  |from()
    .measurement('hourly_prices')
    .groupBy('product_id')
  |window()
    .period(20m)
    .every(10m)
    .align()
  |derivative('value')
    .unit(10m)
    .as('derivative')
  |alert()
    .warn(lambda: "derivative" != 0)
    .message('')
    .details('')
    .post('SOME_URL')

So I have two different problems:

  • Memory usage is growing over time(and in time where 1d CQ are running Memory usage jump to 100%)
    31
    312812×1118 282 KB
  • When Kapacitor starts “write/consistency” queries from influx to kapacitor goes very fast (about some milliseconds) and over time they take more time and as result I get an error:

net/http: request canceled (Client.Timeout exceeded while awaiting headers)" log_id=0HhYMpn0000 service=subscriber

  • Also I notice that not all alerts are triggered

My Influx conf:

reporting-disabled = true
bind-address = "127.0.0.1:8088"

[meta]
  dir = "/var/lib/influxdb/meta"
  retention-autocreate = true
  logging-enabled = true

[data]
  dir = "/var/lib/influxdb/data"
  index-version = "inmem"
  wal-dir = "/var/lib/influxdb/wal"
  wal-fsync-delay = "0s"
  validate-keys = false
  query-log-enabled = false
  cache-max-memory-size = 1073741824
  cache-snapshot-memory-size = 26214400
  cache-snapshot-write-cold-duration = "10m0s"
  compact-full-write-cold-duration = "4h0m0s"
  compact-throughput = 50331648
  compact-throughput-burst = 50331648
  max-series-per-database = 0
  max-values-per-tag = 0
  max-concurrent-compactions = 0
  max-index-log-file-size = 1048576
  series-id-set-cache-size = 100
  trace-logging-enabled = false
  tsm-use-madv-willneed = false

[coordinator]
  write-timeout = "10s"
  max-concurrent-queries = 0
  query-timeout = "0s"
  log-queries-after = "0s"
  max-select-point = 0
  max-select-series = 0
  max-select-buckets = 0

[retention]
  enabled = true
  check-interval = "30m0s"

[shard-precreation]
  enabled = true
  check-interval = "10m0s"
  advance-period = "30m0s"

[monitor]
  store-enabled = true
  store-database = "_internal"
  store-interval = "10s"

[subscriber]
  enabled = true
  http-timeout = "30s"
  insecure-skip-verify = false
  ca-certs = ""
  write-concurrency = 40
  write-buffer-size = 1000

[http]
  enabled = true
  bind-address = ":8086"
  auth-enabled = true
  log-enabled = false
  suppress-write-log = false
  write-tracing = false
  flux-enabled = true
  flux-log-enabled = false
  pprof-enabled = true
  debug-pprof-enabled = false
  https-enabled = false
  https-certificate = "/etc/ssl/influxdb.pem"
  https-private-key = ""
  max-row-limit = 0
  max-connection-limit = 0
  shared-secret = ""
  realm = "InfluxDB"
  unix-socket-enabled = false
  unix-socket-permissions = "0777"
  bind-socket = "/var/run/influxdb.sock"
  max-body-size = 25000000
  access-log-path = ""
  max-concurrent-write-limit = 0
  max-enqueued-write-limit = 0
  enqueued-write-timeout = 30000000000

[logging]
  format = "auto"
  level = "info"
  suppress-logo = false

[[udp]]
  enabled = true
  bind-address = ":8089"
  database = "prices"
  precision = "s"

[continuous_queries]
  log-enabled = true
  enabled = true
  query-stats-enabled = true
  run-interval = "5m"

[tls]
  min-version = ""
  max-version = ""

And Kapacitor conf:

hostname = "SOME_HOST"
data_dir = "/var/lib/kapacitor"

skip-config-overrides = false

default-retention-policy = ""

[http]
  bind-address = "0.0.0.0:9092"
  log-enabled = true
  write-tracing = false
  pprof-enabled = false
  https-enabled = false
  https-certificate = "/etc/ssl/kapacitor.pem"
  
[load]
  enabled = true
  dir = "/etc/kapacitor/load"


[replay]
  dir = "/var/lib/kapacitor/replay"

[task]
  dir = "/var/lib/kapacitor/tasks"
  snapshot-interval = "60s"

[storage]
  boltdb = "/var/lib/kapacitor/kapacitor.db"

[[influxdb]]
  enabled = true
  default = true
  name = "localhost"
  urls = ["http://HOST_IP:8086"]
  username = USERNAME
  password = PASSWORD
  timeout = 0
  
  insecure-skip-verify = false

  startup-timeout = "5m"

  disable-subscriptions = false

  subscription-mode = "cluster"

  subscription-protocol = "http"

  subscriptions-sync-interval = "1m0s"

  kapacitor-hostname = "SOME_HOST"
  
  http-port = 0
  udp-bind = ""
  udp-buffer = 1000
  udp-read-buffer = 0

  [influxdb.subscriptions]
    DB = [ "one_hour" ]