I am debugging a Kapacitor/Morgoth script (please see further below) derived from the following examples:
and
http://docs.morgoth.io/docs/getting-started/
which produces alerts like this one - which looks fine, but each measurement point is listed multiple times in the values array:
{
"id": "mqtt_consumer:host=rpi-opench-data,name=moisture,topic=OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState,units=%",
"message": "MOISTURE ALERT!\n2019-06-10 11:15:09.389656667 +0000 UTC\nINFO TeMoLiFe c67c8dfffe65d020\n\u003cno value\u003e 33\u003cno value\u003e",
"details": "{\u0026#34;Name\u0026#34;:\u0026#34;mqtt_consumer\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;miflora_alert_moisture_morgoth\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=rpi-opench-data,name=moisture,topic=OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState,units=%\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;host\u0026#34;:\u0026#34;rpi-opench-data\u0026#34;,\u0026#34;name\u0026#34;:\u0026#34;moisture\u0026#34;,\u0026#34;topic\u0026#34;:\u0026#34;OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState\u0026#34;,\u0026#34;units\u0026#34;:\u0026#34;%\u0026#34;},\u0026#34;ServerInfo\u0026#34;:{\u0026#34;Hostname\u0026#34;:\u0026#34;localhost\u0026#34;,\u0026#34;ClusterID\u0026#34;:\u0026#34;56f71fa5-5272-4305-8e00-7fd8955e4faf\u0026#34;,\u0026#34;ServerID\u0026#34;:\u0026#34;54052fbb-a451-4323-91e2-4159b4f31294\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;mqtt_consumer:host=rpi-opench-data,name=moisture,topic=OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState,units=%\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;anomalous\u0026#34;:false,\u0026#34;device_id\u0026#34;:\u0026#34;c67c8dfffe65d020\u0026#34;,\u0026#34;device_type\u0026#34;:\u0026#34;TeMoLiFe\u0026#34;,\u0026#34;moisture_anomaly\u0026#34;:0,\u0026#34;value\u0026#34;:33},\u0026#34;Level\u0026#34;:\u0026#34;INFO\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2019-06-10T11:15:09.389656667Z\u0026#34;,\u0026#34;Duration\u0026#34;:1799774377693,\u0026#34;Message\u0026#34;:\u0026#34;MOISTURE ALERT!\\n2019-06-10 11:15:09.389656667 \u0026#43;0000 UTC\\nINFO TeMoLiFe c67c8dfffe65d020\\n\\u003cno value\\u003e 33\\u003cno value\\u003e\u0026#34;}\n",
"time": "2019-06-10T11:15:09.389656667Z",
"duration": 1799774377693,
"level": "INFO",
"data": {
"series": [
{
"name": "mqtt_consumer",
"tags": {
"host": "rpi-opench-data",
"name": "moisture",
"topic": "OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState",
"units": "%"
},
"columns": [
"time",
"anomalous",
"device_id",
"device_type",
"moisture_anomaly",
"value"
],
"values": [
[
"2019-06-10T11:15:09.389656667Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:15:09.389656667Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:15:09.389656667Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:15:09.389656667Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:30:07.327880813Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:30:07.327880813Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
33
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
],
[
"2019-06-10T11:45:10.312474863Z",
false,
"c67c8dfffe65d020",
"TeMoLiFe",
0,
31
]
]
}
]
},
"previousLevel": "INFO",
"recoverable": true
}
The script
dbrp "telegraf"."autogen"
// Measurements topic: OpenCH/DEVICE_TYPE/DEVICE_ID/PubState
var regex_topic = /^OpenCH\/(.*)\/([0-9A-Fa-f]{16})\/PubState$/
var get_device_type = lambda: regexReplace(regex_topic, "topic", '$1')
var get_device_id = lambda: regexReplace(regex_topic, "topic", '$2')
// The amount of data to window at once
var window = 1h
// The field to process
var field = 'value'
// The name for the anomaly score field
var score_field = 'moisture_anomaly'
// The minimum support
var min_support = 0.05
// The error tolerance
var error_tolerance = 0.01
// The consensus
var consensus = 0.5
// Number of sigmas allowed for normal window deviation
var sigmas = 3.0
stream
|from()
.measurement('mqtt_consumer')
.where(lambda: "name" == 'moisture')
.groupBy([*])
|window()
.period(window)
.every(window)
.align()
@morgoth()
.field(field)
.scoreField(score_field)
.minSupport(min_support)
.errorTolerance(error_tolerance)
.consensus(consensus)
.sigma(sigmas)
|eval(get_device_type, get_device_id)
.as('device_type', 'device_id')
.keep()
|log()
.level('DEBUG')
|alert()
.message('MOISTURE ALERT!
{{ .Time }}
{{ .Level }} {{ index .Fields "device_type" }} {{ index .Fields "device_id" }}
{{ index .Fields "name" }} {{ index .Fields "value" }}{{ index .Fields "units" }}')
.info(lambda: "moisture_anomaly" == 0.0)
.warn(lambda: "moisture_anomaly" > 0.5)
.crit(lambda: "moisture_anomaly" > 0.9)
.log('/tmp/alerts.log')
.slack()
.channel('#debug')
Here is the relevant data set - тo duplicates there:
select * from mqtt_consumer where "name" = 'moisture' and "time" > now() - 1d group by topic order by time desc limit 10
name: mqtt_consumer
tags: topic=OpenCH/TeMoLiFe/c67c8dfffe65d020/PubState
time host name units value
---- ---- ---- ----- -----
2019-06-10T12:00:08.319394215Z rpi-opench-data moisture % 31
2019-06-10T11:45:10.312474863Z rpi-opench-data moisture % 31
2019-06-10T11:30:07.327880813Z rpi-opench-data moisture % 33
2019-06-10T11:15:09.389656667Z rpi-opench-data moisture % 33
2019-06-10T11:00:06.500298843Z rpi-opench-data moisture % 33
2019-06-10T10:45:09.615278974Z rpi-opench-data moisture % 33
2019-06-10T10:30:08.65227499Z rpi-opench-data moisture % 33
2019-06-10T10:00:07.822190269Z rpi-opench-data moisture % 33
2019-06-10T09:45:08.888091913Z rpi-opench-data moisture % 34
2019-06-10T09:30:05.942961993Z rpi-opench-data moisture % 34
name: mqtt_consumer
tags: topic=OpenCH/TeMoLiFe/c67c8dfffe65cb63/PubState
time host name units value
---- ---- ---- ----- -----
2019-06-10T12:00:04.523171166Z rpi-opench-data moisture % 19
2019-06-10T11:45:07.622013488Z rpi-opench-data moisture % 18
2019-06-10T11:30:04.679756171Z rpi-opench-data moisture % 19
2019-06-10T11:00:04.886687374Z rpi-opench-data moisture % 19
2019-06-10T10:45:06.035856004Z rpi-opench-data moisture % 19
2019-06-10T10:30:06.172188789Z rpi-opench-data moisture % 19
2019-06-10T10:00:06.300671153Z rpi-opench-data moisture % 20
2019-06-10T09:45:06.299563056Z rpi-opench-data moisture % 20
2019-06-10T09:30:04.377701818Z rpi-opench-data moisture % 20
2019-06-10T09:00:07.628597476Z rpi-opench-data moisture % 20
The script functions as expected, but the measurement point duplicates feel a bit confusing. The points are all within the specified window, but why are they listed multiple times?
Environment:
Kapacitor OSS 1.5.2 (git: HEAD 3086452d00830e01d932838d8c6d1df818648ad3)
go version go1.7.4 linux/arm
Morgoth commit c9874e8c4225235b67c91907efa55cd9d351fca6
Running on Raspberry Pi 3B
Thank you!
