I"m running influxdb in docker, trying to replace another docker container which has been running for years withour problems…
but influx on the new docker keeps crashing and i have no idea why. the nw container ( [influxdb:latest]) is running on a rpi4 with 8 GB RAM from a 500 GB SSD.
I’ve used influxd backup to back and restore to the new container.
this is what is shown in the dockerlog for that container. any hints on getting more relavant logging to find out what the problem is?:
ts=2020-09-29T21:25:34.232207Z lvl=info msg="Opened shard" log_id=0PYwv93G000 service=store trace_id=0PYwwOj0000 op_name=tsdb_open index_version=tsi1 path=/var/lib/influxdb/data/telegraf/autogen/257 duration=93.013ms,
ts=2020-09-29T21:25:34.234298Z lvl=info msg="index opened with 8 partitions" log_id=0PYwv93G000 index=tsi,
ts=2020-09-29T21:25:34.252396Z lvl=info msg="Opened shard" log_id=0PYwv93G000 service=store trace_id=0PYwwOj0000 op_name=tsdb_open index_version=tsi1 path=/var/lib/influxdb/data/telegraf/autogen/260 duration=91.368ms,
ts=2020-09-29T21:25:34.279335Z lvl=info msg="index opened with 8 partitions" log_id=0PYwv93G000 index=tsi,
ts=2020-09-29T21:25:34.288667Z lvl=info msg="Opened shard" log_id=0PYwv93G000 service=store trace_id=0PYwwOj0000 op_name=tsdb_open index_version=tsi1 path=/var/lib/influxdb/data/telegraf/autogen/259 duration=98.357ms,
ts=2020-09-29T21:25:34.303196Z lvl=info msg="index opened with 8 partitions" log_id=0PYwv93G000 index=tsi,
ts=2020-09-29T21:25:34.309112Z lvl=info msg="Opened shard" log_id=0PYwv93G000 service=store trace_id=0PYwwOj0000 op_name=tsdb_open index_version=tsi1 path=/var/lib/influxdb/data/telegraf/autogen/261 duration=76.710ms,
ts=2020-09-29T21:25:34.797863Z lvl=info msg="Reading file" log_id=0PYwv93G000 engine=tsm1 service=cacheloader path=/var/lib/influxdb/wal/_internal/monitor/1/_00146.wal size=10946202,
runtime: out of memory: cannot allocate 5578752-byte block (602865664 in use),
tal error: out of memory,
runtime stack:,
runtime.throw(0xfa5ba4, 0xd),
/usr/local/go/src/runtime/panic.go:774 +0x5c,
runtime.largeAlloc(0x55092b, 0x2ceb0101, 0x2f43370),
/usr/local/go/src/runtime/malloc.go:1140 +0x124,
runtime.mallocgc.func1(),
/usr/local/go/src/runtime/malloc.go:1033 +0x38,
runtime.systemstack(0x310c5a0),
/usr/local/go/src/runtime/asm_arm.s:354 +0x84,
runtime.mstart(),
/usr/local/go/src/runtime/proc.go:1146,
goroutine 23 [running]:,
runtime.systemstack_switch(),
/usr/local/go/src/runtime/asm_arm.s:298 +0x4 fp=0x3bff180 sp=0x3bff17c pc=0x718d8,
runtime.mallocgc(0x55092b, 0xdde1f0, 0x30e0f01, 0xb040400),
/usr/local/go/src/runtime/malloc.go:1032 +0x8d0 fp=0x3bff1e8 sp=0x3bff180 pc=0x1bb70,
runtime.makeslice(0xdde1f0, 0x55092b, 0x55092b, 0xd97f60),
/usr/local/go/src/runtime/slice.go:49 +0x6c fp=0x3bff1fc sp=0x3bff1e8 pc=0x5928c,
github.com/influxdata/influxdb/tsdb/engine/tsm1.getBuf(0x55092b, 0xb0403f0),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/pools.go:17 +0x94 fp=0x3bff218 sp=0x3bff1fc pc=0xcb9ff0,
github.com/influxdata/influxdb/tsdb/engine/tsm1.(*WALSegmentReader).Next(0x301ac00, 0x200e8000),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/wal.go:1167 +0x2bc fp=0x3bff2d8 sp=0x3bff218 pc=0xcd0778,
github.com/influxdata/influxdb/tsdb/engine/tsm1.(*CacheLoader).Load.func1(0x3bff594, 0x47de4e0, 0x3bff58c, 0x31c6980, 0x0, 0x0),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/cache.go:734 +0x34c fp=0x3bff568 sp=0x3bff2d8 pc=0xcd5b2c,
github.com/influxdata/influxdb/tsdb/engine/tsm1.(*CacheLoader).Load(0x47de4e0, 0x31c6980, 0x1, 0x1),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/cache.go:758 +0x88 fp=0x3bff59c sp=0x3bff568 pc=0xc62ec4,
github.com/influxdata/influxdb/tsdb/engine/tsm1.(*Engine).reloadCache(0x3458180, 0x0, 0x0),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/engine.go:2341 +0x1d4 fp=0x3bff6ec sp=0x3bff59c pc=0xc92434,
github.com/influxdata/influxdb/tsdb/engine/tsm1.(*Engine).Open(0x3458180, 0x3318200, 0x1e880c0),
/go/src/github.com/influxdata/influxdb/tsdb/engine/tsm1/engine.go:751 +0x278 fp=0x3bff734 sp=0x3bff6ec pc=0xc89674,
github.com/influxdata/influxdb/tsdb.(*Shard).Open.func1(0x318a160, 0x0, 0x0),
/go/src/github.com/influxdata/influxdb/tsdb/shard.go:340 +0x298 fp=0x3bffa58 sp=0x3bff734 pc=0x59d5bc,
github.com/influxdata/influxdb/tsdb.(*Shard).Open(0x318a160, 0x33182a0, 0x303e030),
/go/src/github.com/influxdata/influxdb/tsdb/shard.go:351 +0x1c fp=0x3bffa84 sp=0x3bffa58 pc=0x5875a0,
github.com/influxdata/influxdb/tsdb.(*Store).loadShards.func1(0x33fa880, 0x3194140, 0x31af230, 0x33fa8c0, 0xf8f780, 0x3842000, 0x34d03c0, 0x33d9837, 0x9, 0x33c6051, ...),
/go/src/github.com/influxdata/influxdb/tsdb/store.go:404 +0x4c4 fp=0x3bfffb4 sp=0x3bffa84 pc=0x59e534,
runtime.goexit(),
/usr/local/go/src/runtime/asm_arm.s:868 +0x4 fp=0x3bfffb4 sp=0x3bfffb4 pc=0x73610,
created by github.com/influxdata/influxdb/tsdb.(*Store).loadShards,
/go/src/github.com/influxdata/influxdb/tsdb/store.go:362 +0xb64,
goroutine 1 [chan receive]:,
github.com/influxdata/influxdb/tsdb.(*Store).loadShards(0x3194140, 0x0, 0x0),
/go/src/github.com/influxdata/influxdb/tsdb/store.go:421 +0x115c,
github.com/influxdata/influxdb/tsdb.(*Store).Open(0x3194140, 0x0, 0x0),
/go/src/github.com/influxdata/influxdb/tsdb/store.go:221 +0x1a4,
github.com/influxdata/influxdb/cmd/influxd/run.(*Server).Open(0x3190320, 0x3287cac, 0x3190320),
/go/src/github.com/influxdata/influxdb/cmd/influxd/run/server.go:442 +0x894,
github.com/influxdata/influxdb/cmd/influxd/run.(*Command).Run(0x3037860, 0x30800f0, 0x0, 0x0, 0x0, 0x30800f0),
/go/src/github.com/influxdata/influxdb/cmd/influxd/run/command.go:149 +0x7e4,
main.(*Main).Run(0x3287f8c, 0x30800f0, 0x0, 0x0, 0x2afa2f8, 0x307e030),
/go/src/github.com/influxdata/influxdb/cmd/influxd/main.go:81 +0x104,
main.main(),
/go/src/github.com/influxdata/influxdb/cmd/influxd/main.go:45 +0x140,
goroutine 18 [syscall]:,
os/signal.signal_recv(0x0),
/usr/local/go/src/runtime/sigqueue.go:147 +0x130,
os/signal.loop(),
/usr/local/go/src/os/signal/signal_unix.go:23 +0x14,
created by os/signal.init.0,
/usr/local/go/src/os/signal/signal_unix.go:29 +0x30,
goroutine 6 [select]:,
github.com/influxdata/influxdb/vendor/go.opencensus.io/stats/view.(*worker).start(0x32b0680),
/go/src/github.com/influxdata/influxdb/vendor/go.opencensus.io/stats/view/worker.go:154 +0xb0,
created by github.com/influxdata/influxdb/vendor/go.opencensus.io/stats/view.init.0,
/go/src/github.com/influxdata/influxdb/vendor/go.opencensus.io/stats/view/worker.go:32 +0x48,
goroutine 7 [IO wait]:,
internal/poll.runtime_pollWait(0xa6bb7f80, 0x72, 0x0),
/usr/local/go/src/runtime/netpoll.go:184 +0x44,
internal/poll.(*pollDesc).wait(0x318eab4, 0x72, 0x0, 0x0, 0xf9c433),
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x30,
internal/poll.(*pollDesc).waitRead(...),
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92,
internal/poll.(*FD).Accept(0x318eaa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0),
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a8,
net.(*netFD).accept(0x318eaa0, 0x0, 0xa7801, 0x0),
/usr/local/go/src/net/fd_unix.go:238 +0x20,
net.(*TCPListener).accept(0x33092b0, 0x331abc0, 0x40000000, 0x0),
/usr/local/go/src/net/tcpsock_posix.go:139 +0x20,
net.(*TCPListener).Accept(0x33092b0, 0x0, 0x0, 0x0, 0x0),
/usr/local/go/src/net/tcpsock.go:261 +0x3c,
github.com/influxdata/influxdb/tcp.(*Mux).Serve(0x331abc0, 0x1e73a50, 0x33092b0, 0x33092b0, 0x0),
/go/src/github.com/influxdata/influxdb/tcp/mux.go:75 +0x60,
created by github.com/influxdata/influxdb/cmd/influxd/run.(*Server).Open,
/go/src/github.com/influxdata/influxdb/cmd/influxd/run/server.go:393 +0x1f0,
