prometheus: Add detailed metrics

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

test/shell/prometheus-3.sh

@@ -1,7 +1,5 @@
 #!/bin/bash -e
 
-exit 0	# FIXME: disabled until intermittent failures can be resolved
-
 # run a graph, with prometheus support
 timeout --kill-after=30s 25s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-3.yaml &
 pid=$!

test/shell/prometheus-4.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -xe
+
+# run a graph, with prometheus support
+timeout --kill-after=30s 25s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-4.yaml &
+pid=$!
+sleep 10s	# let it converge
+
+# For test debugging purpose
+curl 127.0.0.1:9233/metrics
+
+# Check for mgmt_resources
+curl 127.0.0.1:9233/metrics | grep '^mgmt_resources{kind="file"} 4$'
+
+# One CheckApply for a File ; in noop mode.
+curl 127.0.0.1:9233/metrics | grep 'mgmt_checkapply_total{apply="false",errorful="false",eventful="true",kind="file"} 1$'
+
+# Two CheckApply for a File ; without errors, with events
+curl 127.0.0.1:9233/metrics | grep 'mgmt_checkapply_total{apply="true",errorful="false",eventful="true",kind="file"} 2$'
+
+# Multiple CheckApplies with errors
+curl 127.0.0.1:9233/metrics | grep 'mgmt_checkapply_total{apply="true",errorful="true",eventful="true",kind="file"} [0-9]\+'
+
+# One soft failure ATM
+curl 127.0.0.1:9233/metrics | grep 'mgmt_failures{failure="soft",kind="file"} 1$'
+
+# Multiple soft failures since startup
+if curl 127.0.0.1:9233/metrics | grep 'mgmt_failures_total{failure="soft",kind="file"} 1$'
+then
+	false
+fi
+curl 127.0.0.1:9233/metrics | grep 'mgmt_failures_total{failure="soft",kind="file"} [0-9]\+'
+
+killall -SIGINT mgmt	# send ^C to exit mgmt
+wait $pid	# get exit status
+exit $?

test/shell/prometheus-4.yaml

@@ -0,0 +1,29 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    path: "/tmp/mgmt/NONEXIST/f1"
+    content: |
+      i am f1
+    state: exists
+    meta:
+      retry: -1
+      delay: 1000
+  - name: file2
+    path: "/tmp/mgmt/f2"
+    content: |
+      i am f2
+    state: exists
+  - name: file3
+    path: "/tmp/mgmt/f3"
+    content: |
+      i am f3
+    state: exists
+  - name: file4
+    path: "/tmp/mgmt/f4"
+    content: |
+      i am f4
+    state: exists
+    meta:
+      noop: true