Issue with emqtt ouput plugin

Hi,

I’m having an issue with emqtt ouput plugin for telegraf. Telegraf starts up fine and metrics are getting pushed to emqtt. After some time i start seeing messages saying “A flush is already in progress, hence skipping flush”. This message comes for about 10-15 collection cycles. After this, the input plugins start reporting “it took too long for them to collect”

Telegraf is stuck at this state forever until a restart is done. I took a look at the code and it seems the code is stuck at

“c.obound <- &PacketAndToken{p: pub, t: token}” in file client.go in paho client.

is it possible to have this call bound by some timeout so it doesnt bock on this forever.

-Rahav

SIGQUIT: quit
PC=0x45d4b3 m=2 sigcode=0

goroutine 0 [idle]:
runtime.futex(0x1e89db8, 0x0, 0x7f094118ad68, 0x0, 0x7f0900000000, 0x45d146, 0x3c, 0x0, 0x7f094118adb0, 0x4105e0, …)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sys_linux_amd64.s:423 +0x23
runtime.futexsleep(0x1e89db8, 0x0, 0xdf8475800)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/os_linux.go:62 +0xd7
runtime.notetsleep_internal(0x1e89db8, 0xdf8475800, 0x0)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/lock_futex.go:174 +0xd0
runtime.notetsleep(0x1e89db8, 0xdf8475800, 0x3c939216c4301)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/lock_futex.go:194 +0x56
runtime.sysmon()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/proc.go:3805 +0x135
runtime.mstart1()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/proc.go:1179 +0x11e
runtime.mstart()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/proc.go:1149 +0x64

goroutine 1 [semacquire, 354 minutes]:
sync.runtime_Semacquire(0xc4201f089c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc4201f0890)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/waitgroup.go:131 +0x7a
github.com/influxdata/telegraf/agent.(*Agent).Run(0xc4201dc018, 0xc420064540, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:430 +0x4ed
main.reloadLoop(0xc420064ae0, 0x1eacbb0, 0x0, 0x0, 0x1eacbb0, 0x0, 0x0, 0x1eacbb0, 0x0, 0x0, …)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:231 +0xa46
main.main()
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:408 +0x65b

goroutine 17 [syscall, 354 minutes, locked to thread]:
runtime.goexit()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/asm_amd64.s:2197 +0x1

goroutine 6 [syscall, 354 minutes]:
os/signal.signal_recv(0x0)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/os/signal/signal_unix.go:28 +0x41

goroutine 6223 [chan send, 172 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddCounter(0xc4200b7060, 0x1534ec8, 0x4, 0xc4204162d0, 0x0, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:75 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*SwapStats).Gather(0xc4201c48b0, 0x1da4ec0, 0xc4200b7060, 0xc42005d620, 0x762d5a)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/memory.go:71 +0x5a1
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc420641c20, 0xc42015c380, 0xc4200b7060)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 26 [select, 354 minutes, locked to thread]:
runtime.gopark(0x15bf330, 0x0, 0x1537f6a, 0x6, 0x18, 0x2)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/proc.go:271 +0x13a
runtime.selectgoImpl(0xc420030f50, 0x0, 0x18)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/select.go:423 +0x1364
runtime.selectgo(0xc420030f50)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/signal_unix.go:434 +0x2dd
runtime.goexit()
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/asm_amd64.s:2197 +0x1

goroutine 4486 [semacquire, 232 minutes]:
sync.runtime_notifyListWait(0xc4201c6090, 0xc4000009ea)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:297 +0x10b
sync.(*Cond).Wait(0xc4201c6080)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/cond.go:57 +0x89
github.com/eclipse/paho.mqtt.golang.keepalive.func4(0xc4206ddb70, 0xc420193c00, 0xc4206e2a20, 0xc4206fe1e0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:86 +0x7d
created by github.com/eclipse/paho.mqtt.golang.keepalive
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:96 +0x41e

goroutine 21 [select, 354 minutes]:
github.com/eclipse/paho.mqtt.golang.(*router).matchAndDispatch.func1(0xc420064480, 0xc420016940, 0xc420111b01, 0xc420193c00)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/router.go:134 +0x545
created by github.com/eclipse/paho.mqtt.golang.(*router).matchAndDispatch
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/router.go:160 +0x5c

goroutine 4617 [chan send, 214 minutes]:
github.com/eclipse/paho.mqtt.golang.(*client).Publish(0xc420193c00, 0xc420286380, 0x6b, 0xc4202a0000, 0x12b3dc0, 0xc4202a6800, 0x12ca6e0, 0xc42027c3c0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/client.go:520 +0x614
github.com/influxdata/telegraf/plugins/outputs/mqtt.(*MQTT).publish(0xc4200f6000, 0xc420286380, 0x6b, 0xc4201ff200, 0x532, 0x600, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/outputs/mqtt/mqtt.go:154 +0x239
github.com/influxdata/telegraf/plugins/outputs/mqtt.(*MQTT).Write(0xc4200f6000, 0xc4202eab00, 0x58, 0x58, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/outputs/mqtt/mqtt.go:143 +0x329
github.com/influxdata/telegraf/internal/models.(*RunningOutput).write(0xc4201ca000, 0xc4202eab00, 0x58, 0x58, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/internal/models/running_output.go:179 +0x10c
github.com/influxdata/telegraf/internal/models.(*RunningOutput).Write(0xc4201ca000, 0x15bf460, 0xc4202a2f00)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/internal/models/running_output.go:161 +0x497
github.com/influxdata/telegraf/agent.(*Agent).flush.func1(0xc4202a2f00, 0xc4201ca000)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:238 +0x59
created by github.com/influxdata/telegraf/agent.(*Agent).flush
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:243 +0xa5

goroutine 4616 [semacquire, 226 minutes]:
sync.runtime_Semacquire(0xc4202a2f0c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc4202a2f00)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/waitgroup.go:131 +0x7a
github.com/influxdata/telegraf/agent.(*Agent).flush(0xc4201dc018)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:246 +0xd9
github.com/influxdata/telegraf/agent.(*Agent).flusher.func3(0xc4202f61e0, 0xc4201dc018, 0xc420064540)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:338 +0x7f
created by github.com/influxdata/telegraf/agent.(*Agent).flusher
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:345 +0x2a6

goroutine 4485 [semacquire, 232 minutes]:
sync.runtime_notifyListWait(0xc4201c6050, 0xc4000000da)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:297 +0x10b
sync.(*Cond).Wait(0xc4201c6040)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/cond.go:57 +0x89
github.com/eclipse/paho.mqtt.golang.keepalive.func3(0xc4206ddb70, 0xc420193c00, 0xc4206e2a20, 0xc4206fe220, 0x737be7600)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:70 +0x7d
created by github.com/eclipse/paho.mqtt.golang.keepalive
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:80 +0x3d5

goroutine 36 [select, 354 minutes]:
main.reloadLoop.func1(0xc4200645a0, 0xc420064540, 0xc420265180, 0xc420064ae0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:192 +0x24a
created by main.reloadLoop
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:206 +0x6ec

goroutine 37 [chan send, 176 minutes]:
github.com/influxdata/telegraf/agent.(*Agent).flusher(0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42005d260, 0x458abc, 0x180001)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:354 +0x408
github.com/influxdata/telegraf/agent.(*Agent).Run.func1(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42005d260)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:400 +0x7a
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:404 +0x316

goroutine 38 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c380, 0xc4200b7060, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c380, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c380, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 39 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c400, 0xc420353a40, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c400, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c400, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 40 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c480, 0xc4200b6620, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c480, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c480, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 41 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c500, 0xc4202ac160, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c500, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c500, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 42 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c580, 0xc4202ac3e0, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c580, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c580, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 43 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c640, 0xc4202ac640, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c640, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c640, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 44 [select]:
github.com/influxdata/telegraf/agent.gatherWithTimeout(0xc420064540, 0xc42015c700, 0xc420360ee0, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:157 +0x28f
github.com/influxdata/telegraf/agent.(*Agent).gatherer(0xc4201dc018, 0xc420064540, 0xc42015c700, 0xdf8475800, 0xc42005d200)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:124 +0x2ad
github.com/influxdata/telegraf/agent.(*Agent).Run.func3(0xc4201f0890, 0xc4201dc018, 0xc420064540, 0xc42005d200, 0xc42015c700, 0xdf8475800)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:426 +0x7d
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:427 +0x4a4

goroutine 65 [semacquire, 180 minutes]:
sync.runtime_SemacquireMutex(0xc4201ca09c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:62 +0x34
sync.(*Mutex).Lock(0xc4201ca098)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/mutex.go:87 +0x9d
github.com/influxdata/telegraf/internal/models.(*RunningOutput).write(0xc4201ca000, 0xc420614000, 0x3e8, 0x3e8, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/internal/models/running_output.go:176 +0x77
github.com/influxdata/telegraf/internal/models.(*RunningOutput).AddMetric(0xc4201ca000, 0x1da86a0, 0xc4206f5b00)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/internal/models/running_output.go:119 +0x14c
github.com/influxdata/telegraf/agent.(*Agent).flusher.func1(0xc4201f19c0, 0xc420064540, 0xc4202f6120, 0xc4201dc018)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:284 +0x1e5
created by github.com/influxdata/telegraf/agent.(*Agent).flusher
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:292 +0xe5

goroutine 47 [select]:
gopkg.in/mgo%2ev2.(*mongoCluster).syncServersLoop(0xc42000ad00)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/cluster.go:394 +0x3a3
created by gopkg.in/mgo%2ev2.newCluster
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/cluster.go:78 +0x188

goroutine 6237 [chan send, 170 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddGauge(0xc420353a40, 0x15349e8, 0x4, 0xc4204163f0, 0xc4204163c0, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:64 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*DiskStats).Gather(0xc42005c240, 0x1da4ec0, 0xc420353a40, 0x12bdbe0, 0xc42005db60)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/disk.go:79 +0xa4c
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc4205f24e0, 0xc42015c400, 0xc420353a40)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 31 [sleep]:
time.Sleep(0x37e11d600)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/time.go:59 +0xf9
gopkg.in/mgo%2ev2.(*mongoServer).pinger(0xc420250380, 0x1)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/server.go:301 +0x293
created by gopkg.in/mgo%2ev2.newServer
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/server.go:89 +0x166

goroutine 33 [IO wait]:
net.runtime_pollWait(0x7f09418f30b8, 0x72, 0xb)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc420286298, 0x72, 0x1d911a0, 0x1d878d0)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc420286298, 0xc420263800, 0x24)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc420286230, 0xc420263800, 0x24, 0x24, 0x0, 0x1d911a0, 0x1d878d0)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc42000e0a0, 0xc420263800, 0x24, 0x24, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/net/net.go:181 +0x70
gopkg.in/mgo%2ev2.fill(0x1da5a00, 0xc42000e0a0, 0xc420263800, 0x24, 0x24, 0x0, 0x11)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/socket.go:535 +0x53
gopkg.in/mgo%2ev2.(*mongoSocket).readLoop(0xc420250460)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/socket.go:551 +0x107
created by gopkg.in/mgo%2ev2.newSocket
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/gopkg.in/mgo.v2/socket.go:194 +0x259

goroutine 66 [select, 354 minutes]:
github.com/influxdata/telegraf/agent.(*Agent).flusher.func2(0xc4201f19c0, 0xc420064540, 0xc42005d260, 0xc4201dc018)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:298 +0x358
created by github.com/influxdata/telegraf/agent.(*Agent).flusher
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:321 +0x14b

goroutine 4463 [semacquire, 232 minutes]:
sync.runtime_Semacquire(0xc4206ddb7c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc4206ddb70)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/waitgroup.go:131 +0x7a
github.com/eclipse/paho.mqtt.golang.keepalive.func1(0xc4206e2a20, 0xc420193c00, 0xc4206ddb70)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:36 +0x7c
github.com/eclipse/paho.mqtt.golang.keepalive(0xc420193c00)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:102 +0x902
created by github.com/eclipse/paho.mqtt.golang.(*client).reconnect
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/client.go:348 +0x8ab

goroutine 6248 [semacquire, 170 minutes]:
sync.runtime_Semacquire(0xc4201c441c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc4201c4410)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/waitgroup.go:131 +0x7a
github.com/influxdata/telegraf/plugins/inputs/mongodb.(*MongoDB).Gather(0xc420276000, 0x1da4ec0, 0xc4202ac160, 0xc42005dc80, 0x762d5a)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb.go:103 +0x250
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc4203707e0, 0xc42015c500, 0xc4202ac160)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 6250 [chan send, 170 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddGauge(0xc4202ac3e0, 0x1533d86, 0x3, 0xc4202e41b0, 0xc4202e4180, 0xc4206fe600, 0x1, 0x1)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:64 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*CPUStats).Gather(0xc420274060, 0x1da4ec0, 0xc4202ac3e0, 0xc42005dc80, 0x762d5a)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/cpu.go:122 +0xfb1
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc4203708a0, 0xc42015c580, 0xc4202ac3e0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 6224 [chan send, 172 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddCounter(0xc4202ac640, 0x1533e8e, 0x3, 0xc420416300, 0x0, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:75 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*MemStats).Gather(0xc4201c4c40, 0x1da4ec0, 0xc4202ac640, 0xc42005d0e0, 0x762d5a)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/memory.go:39 +0x856
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc420641da0, 0xc42015c640, 0xc4202ac640)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 4472 [semacquire, 232 minutes]:
sync.runtime_Semacquire(0xc420193f28)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:47 +0x34
sync.(*WaitGroup).Wait(0xc420193f1c)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/waitgroup.go:131 +0x7a
github.com/eclipse/paho.mqtt.golang.(*client).internalConnLost(0xc420193c00, 0x1d8d120, 0xc4207179a0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/client.go:436 +0x7e
created by github.com/eclipse/paho.mqtt.golang.errorWatch
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/net.go:322 +0x242

goroutine 4484 [semacquire, 232 minutes]:
sync.runtime_notifyListWait(0xc4201c6010, 0xc4000000d8)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/runtime/sema.go:297 +0x10b
sync.(*Cond).Wait(0xc4201c6000)
/build/mts/release/sb-15299446/compcache/cayman_go/ob-6134445/linux64/lin64/src/sync/cond.go:57 +0x89
github.com/eclipse/paho.mqtt.golang.keepalive.func2(0xc4206ddb70, 0xc420193c00, 0xc4206e2a20, 0xc4206fe260, 0xc4206fe1e0, 0xc4206fe220, 0x737be7600)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:52 +0x7d
created by github.com/eclipse/paho.mqtt.golang.keepalive
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/eclipse/paho.mqtt.golang/ping.go:64 +0x382

goroutine 6206 [chan send, 172 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddCounter(0xc4200b6620, 0x15374f0, 0x6, 0xc4204164b0, 0xc420416480, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:75 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*DiskIOStats).Gather(0xc420286070, 0x1da4ec0, 0xc4200b6620, 0x12bdbe0, 0xc4205f26c0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/disk.go:161 +0x922
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc42005c900, 0xc42015c480, 0xc4200b6620)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

goroutine 6249 [chan send, 170 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddFields(0xc4202ac160, 0x1539c7c, 0x7, 0xc4202e4360, 0xc4202e4330, 0xc4206feec0, 0x1, 0x1)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:53 +0x12e
github.com/influxdata/telegraf/plugins/inputs/mongodb.(*MongodbData).flush(0xc420357da8, 0x1da4ec0, 0xc4202ac160)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb_data.go:160 +0xd5
github.com/influxdata/telegraf/plugins/inputs/mongodb.(*Server).gatherData(0xc4202ac220, 0x1da4ec0, 0xc4202ac160, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb_server.go:110 +0xebd
github.com/influxdata/telegraf/plugins/inputs/mongodb.(*MongoDB).gatherServer(0xc420276000, 0xc4202ac220, 0x1da4ec0, 0xc4202ac160, 0xc42005dc80, 0x762d5a)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb.go:171 +0x89
github.com/influxdata/telegraf/plugins/inputs/mongodb.(*MongoDB).Gather.func1(0xc4201c4410, 0x1da4ec0, 0xc4202ac160, 0xc420276000, 0xc4202ac220)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb.go:99 +0x73
created by github.com/influxdata/telegraf/plugins/inputs/mongodb.(*MongoDB).Gather
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/mongodb/mongodb.go:100 +0x3ec

goroutine 6231 [chan send, 172 minutes]:
github.com/influxdata/telegraf/agent.(*accumulator).AddCounter(0xc420360ee0, 0x1533eb8, 0x3, 0xc4206cd1d0, 0xc4206cd1a0, 0x0, 0x0, 0x0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/accumulator.go:75 +0x12e
github.com/influxdata/telegraf/plugins/inputs/system.(*NetIOStats).Gather(0xc42015c680, 0x1da4ec0, 0xc420360ee0, 0x12bdbe0, 0xc42005db60)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/plugins/inputs/system/net.go:89 +0x7c2
github.com/influxdata/telegraf/agent.gatherWithTimeout.func1(0xc42005dda0, 0xc42015c700, 0xc420360ee0)
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:153 +0x49
created by github.com/influxdata/telegraf/agent.gatherWithTimeout
/build/mts/release/sb-15299446/cayman_telegraf/telegraf/src/github.com/influxdata/telegraf/agent/agent.go:154 +0xcc

rax 0xfffffffffffffffc
rbx 0x0
rcx 0xffffffffffffffff
rdx 0x0
rdi 0x1e89db8
rsi 0x0
rbp 0x7f094118ad78
rsp 0x7f094118ad30
r8 0x0
r9 0x0
r10 0x7f094118ad68
r11 0x246
r12 0x0
r13 0x34
r14 0x0
r15 0xf3
rip 0x45d4b3
rflags 0x246
cs 0x33
fs 0x0
gs 0x0

It looks like you might have built from source, make sure you have this patch #3697 which was included in Telegraf 1.5.3.

If you have this fix already, it would probably be best to open a new bug report