Internal error when using statsmodels.linearRegression()

Hi,

I’m using InfluxDB 2.2.0 and would like to draw a trend line.
This query works:

import "contrib/anaisdg/statsmodels"

from(bucket: "longterm")
  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
  |> filter(fn: (r) => r._measurement == "shift")
  |> filter(fn: (r) => r._field == "total" and not exists r.condition)
  |> yield(name: "context")

from(bucket: "longterm")
  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
  |> filter(fn: (r) => r._measurement == "shift")
  |> filter(fn: (r) => r._field == "total" and not exists r.condition)
  |> toFloat()
  |> statsmodels.linearRegression()
  |> map(fn: (r) => ({ r with _value: r.y_hat }))
  |> yield(name: "trend")

But this query leads to an internal server error:

import "contrib/anaisdg/statsmodels"

from(bucket: "longterm")
  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)
  |> filter(fn: (r) => r._measurement == "shift")
  |> filter(fn: (r) => r._field == "total" and not exists r.condition)
  |> yield(name: "context")
  |> toFloat()
  |> statsmodels.linearRegression()
  |> map(fn: (r) => ({ r with _value: r.y_hat }))
  |> yield(name: "trend")

The problem is the interaction between yield() and the statsmodels call. If I yield before it, this error is found in the logs:

influx-1  | ts=2022-05-16T12:31:51.348745Z lvl=warn msg="internal error not returned to client" log_id=0aUo7qcl000 handler=error_logger error="runtime error: invalid memory address or nil pointer dereference"
influx-1  | ts=2022-05-16T12:31:51.348478Z lvl=info msg="panic during program start" log_id=0aUo7qcl000 service=storage-reads error="runtime error: invalid memory address or nil pointer dereference" stacktrace="goroutine 2139 [running]:\nruntime/debug.Stack()\n\t/go/src/runtime/debug/stack.go:24 +0x88\ngithub.com/influxdata/influxdb/v2/query/control.(*Controller).executeQuery.func1(0x4007226900, 0x400a97d440)\n\t/root/project/query/control/controller.go:465 +0x140\npanic({0xffffa6be38e0, 0xffffa8169a70})\n\t/go/src/runtime/panic.go:1038 +0x21c\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doExpression(0x400959f180, {0xffffa6fbb4e0, 0x4017768630}, {0xffffa6fcc0d8, 0x4000fd3dd0}, {0xffffa6ff3148, 0x4017768a50})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:374 +0x1550\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doExpression(0x400959f180, {0xffffa6fbb4e0, 0x4017768630}, {0xffffa6fcbce8, 0x4000fd3d40}, {0xffffa6ff3148, 0x4017768a50})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:428 +0x1114\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doVariableAssignment(0x400959f180, {0xffffa6fbb4e0, 0x4017768630}, 0x40010dacc0, {0xffffa6ff3148, 0x4017768a50})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:319 +0x5c\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doStatement(0x400959f180, {0xffffa6fbb4e0, 0x4017768630}, {0xffffa6fc67b8, 0x40010dacc0}, {0xffffa6ff3148, 0x4017768a50})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:199 +0x3cc\ngithub.com/influxdata/flux/interpreter.function.doCall({0x400104ed00, {0xffffa6ff3148, 0x400fe4d470}, 0x400959f180}, {0xffffa6fbb4e0, 0x4017768630}, {0xffffa7004c18, 0x400eb6a3f0})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:938 +0x1e0\ngithub.com/influxdata/flux/interpreter.function.Call({0x400104ed00, {0xffffa6ff3148, 0x400fe4d470}, 0x400959f180}, {0xffffa6fbb4e0, 0x4017768630}, {0xffffa7004178, 0x400bd7c460})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:886 +0x108\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doCall(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, 0x400b3bf050, {0xffffa6ff3148, 0x400fe4c990})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:724 +0x474\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doExpression(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6fcbd78, 0x400b3bf050}, {0xffffa6ff3148, 0x400fe4c990})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:368 +0x13f4\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doArguments.func1(0x400eb6a258)\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:788 +0x31c\ngithub.com/influxdata/flux/values.BuildObjectWithSize(0x0, 0x4003a271b0)\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/values/object.go:113 +0x164\ngithub.com/influxdata/flux/values.BuildObject(...)\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/values/object.go:100\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doArguments(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, 0x400935a680, {0xffffa6ff3148, 0x400fe4c990}, {0x5, {0xffffa6f923e8, 0x400086a7a0}, 0xc}, ...)\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:775 +0x340\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doCall(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, 0x400b3befc0, {0xffffa6ff3148, 0x400fe4c990})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:702 +0x288\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doExpression(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6fcbd78, 0x400b3befc0}, {0xffffa6ff3148, 0x400fe4c990})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:368 +0x13f4\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doStatement(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6fc66b8, 0x40088e2870}, {0xffffa6ff3148, 0x400fe4c990})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:203 +0x26c\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doFile(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, 0x4009437980, {0xffffa6ff3148, 0x400fe4c990}, {0xffffa6f797e0, 0x400915dee0})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:111 +0x184\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doPackage(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, 0x400419f180, {0xffffa6ff3148, 0x400fe4c990}, {0xffffa6f797e0, 0x400915dee0})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:94 +0x98\ngithub.com/influxdata/flux/interpreter.(*Interpreter).doRoot(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6fbc9a8, 0x400419f180}, {0xffffa6ff3148, 0x400fe4c990}, {0xffffa6f797e0, 0x400915dee0})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:84 +0x114\ngithub.com/influxdata/flux/interpreter.(*Interpreter).Eval(0x400959f180, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6fbc9a8, 0x400419f180}, {0xffffa6ff3148, 0x400fe4c990}, {0xffffa6f797e0, 0x400915dee0})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/interpreter/interpreter.go:75 +0x6c\ngithub.com/influxdata/flux/runtime.(*runtime).Eval(0xffffadb07fb0, {0xffffa6fbb4e0, 0x400a6d08d0}, {0xffffa6f9b6e8, 0x4004f910a0}, {0xffffa6f92578, 0xffffadb415a0}, {0x400915de00, 0x2, 0x2})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/runtime/runtime.go:119 +0x21c\ngithub.com/influxdata/flux/lang.(*AstProgram).getSpec(0x40088e2280, {0xffffa6fbb4e0, 0x400a6d08a0}, {0xffffa6fbc350, 0x40088e2370})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/lang/compiler.go:449 +0x224\ngithub.com/influxdata/flux/lang.(*AstProgram).Start(0x40088e2280, {0xffffa6fbb4e0, 0x400a6d06f0}, {0xffffa6fbc350, 0x40088e2370})\n\t/go/pkg/mod/github.com/influxdata/flux@v0.161.0/lang/compiler.go:492 +0x200\ngithub.com/influxdata/influxdb/v2/query/control.(*Controller).executeQuery(0x400a97d440, 0x4007226900)\n\t/root/project/query/control/controller.go:488 +0x210\ngithub.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0x400a97d440)\n\t/root/project/query/control/controller.go:446 +0x34\ngithub.com/influxdata/influxdb/v2/query/control.New.func1(0x400a97d440)\n\t/root/project/query/control/controller.go:231 +0x54\ncreated by github.com/influxdata/influxdb/v2/query/control.New\n\t/root/project/query/control/controller.go:229 +0x844\n"

Are you not allowed to yield before doing such operations?

Hello @ypnos,
Thank you for sharing.
I’m not sure what’s causing that.
I’ll share with the Flux team.

@ypnos Your query looks great, this is definitely a bug in our code. I’ll file and issue with the details you have here and we will take a look at it.

Created this issue here Panic with multiple yields · Issue #4780 · influxdata/flux · GitHub