Add navigation templates

So far we have only extapolated as far into the future as we could look
into the past. Everything beyond that was "infinity". Now we use the
first and last observation to extrapolate beyond that.

app.py

@@ -5,7 +5,9 @@ import logging
 import logging.config
 import os
 
-from flask import (Flask, request, jsonify, abort, render_template)
+from collections import defaultdict
+
+from flask import (Flask, request, jsonify, abort, render_template, url_for)
 
 from ltsdb_json import LTS
 from dashboard import Dashboard
@@ -97,8 +99,9 @@ def verify_node(d):
     if "/" in node:
         raise ValueError("invalid node name %s", node)
     try:
-        log.info("getting client config from %s", "config/" + node)
-        with open("config/" + node) as fh:
+        fn = "config/" + node
+        log.info("getting client config from %s", fn)
+        with open(fn) as fh:
             node_conf = json.load(fh)
     except Exception as e:
         log.warning("got %s opening %s", e, "config/" + node)
@@ -112,8 +115,19 @@ def verify_node(d):
             if timestamp > node_conf["last"]:
                 node_conf["last"] = timestamp
                 os.replace("config/" + node, "config/" + node + ".old")
-                with open("config/" + node, "w") as fh:
+                tmpfn = fn + "." + str(os.getpid())
+                oldfn = fn + ".old"
+                with open(tmpfn, "w") as fh:
                     json.dump(node_conf, fh) # XXX
+                try:
+                    os.unlink(oldfn)
+                except FileNotFoundError:
+                    pass
+                try:
+                    os.link(fn, oldfn)
+                except FileNotFoundError:
+                    pass
+                os.rename(tmpfn, fn)
                 return node
             else:
                 abort(409, "timestamp out of sync")
@@ -140,3 +154,71 @@ def dashboard_index():
 def dashboard_file(dashboard):
     d = Dashboard("dashboards/" + dashboard + ".json")
     return d.as_html()
+
+@app.get("/nav")
+def nav():
+    # Start with a list of all dimensions, the number of matching time series
+    # and a truncated list of series.
+    # If a dimension is chosen, display a choice of members
+    # choosing one or more members goes back to the list of
+    # (remaining) dimensions
+    with open("data/.index") as fh:
+        fcntl.flock(fh, fcntl.LOCK_SH)
+        index = json.load(fh)
+    timeseries = None
+    for k, v in request.args.lists():
+        if k[0] == ".":
+            continue
+        log.debug("search: %s -> %s", k, v)
+        if timeseries is None:
+            timeseries = set()
+            log.debug("search: %s: %s", k, index[k])
+            for m in v:
+                timeseries |= set(index[k][m])
+        else:
+            filter = set()
+            for m in v:
+                filter |= set(index[k][m])
+            timeseries &= filter
+    if timeseries is None:
+        timeseries = set()
+        for mc in index.values():
+            for tsl in mc.values():
+                timeseries |= set(tsl)
+    if d := request.args.get(".m"):
+        members = []
+        for m, tsl in index[d].items():
+            if set(tsl) & timeseries:
+                members.append(m)
+        return render_template("nav_member_select.html", dimension=d, members=members)
+    else:
+        params = request.args.to_dict(flat=False)
+        matching_dimensions = defaultdict(int)
+        for d, mc in index.items():
+            if d in params:
+                continue
+            for m, tsl in mc.items():
+                mtsl = set(tsl) & timeseries
+                if mtsl:
+                    matching_dimensions[d] += len(mtsl)
+        matching_dimensions_list = []
+        for d in matching_dimensions:
+            params[".m"] = d
+            url = url_for("nav", **params)
+            app.logger.debug(f"{d=} {url=}")
+            matching_dimensions_list.append(
+                {"name": d, "count": matching_dimensions[d], "url": url}
+            )
+        total_timeseries = len(timeseries)
+        timeseries = [LTS(id=ts) for ts in list(timeseries)[:100]]
+        return render_template(
+                    "nav_dimension_list.html",
+                    matching_dimensions=matching_dimensions_list,
+                    timeseries=timeseries, total_timeseries=total_timeseries)
+
+
+
+
+
+
+    # 

clients/record_postgresql_version

@@ -0,0 +1,36 @@
+#!/usr/bin/python3
+import re
+import subprocess
+import time
+
+import ltsdb_record
+
+p = subprocess.run(["psql", "-c", "select version()", "-A", "-t", "-X", "-q",],
+                   stdout=subprocess.PIPE, universal_newlines=True)
+
+# This works only for PostgreSQL 10.x and above. I don't expect to encounter
+# older versions any more.
+m = re.match(r"^PostgreSQL (\d+).(\d+) ", p.stdout)
+if m:
+    version = int(m.group(1)) + int(m.group(2)) / 100
+
+report0 = []
+report0.append({ "measure": "postgresql_version", "unit": "version", "value":
+version})
+now = time.time()
+report = [
+    {
+        "description": {
+            "hostname": ltsdb_record.node,
+            "measure": r["measure"],
+            "unit": r["unit"]
+        },
+        "data": [
+            [now, r["value"]]
+        ]
+    }
+    for r in report0
+]
+
+success = ltsdb_record.record_observations(report)
+exit(1 - success)

clients/record_smart

@@ -0,0 +1,78 @@
+#!/usr/bin/python3
+import argparse
+import re
+import subprocess
+import time
+
+import ltsdb_record
+
+ap = argparse.ArgumentParser()
+ap.add_argument("device")
+args = ap.parse_args()
+
+p = subprocess.run(["/usr/sbin/smartctl", "-A", args.device],
+                   stdout=subprocess.PIPE, universal_newlines=True)
+report0 = []
+state = 0
+for ln in p.stdout.splitlines():
+    if state == 0 and ln.startswith("ID# ATTRIBUTE_NAME"):
+        state = 1
+    elif state == 1 and ln == "":
+        state = 2
+    elif state == 1:
+        (id, attribute_name, flag, value, worst, thresh, type, updated, when_failed, raw_value) = ln.split(None, 9)
+
+        if attribute_name == "Command_Timeout":
+            # This is a tuple of three values and I don't know what they mean
+            # so I just skip them.
+            # I guess I could just record them as smart_command_timeout_1,
+            # smart_command_timeout_2 and smart_command_timeout_3 ...
+            continue
+        if "_Ct" in attribute_name or "_Count" in attribute_name or "_Cnt" in attribute_name:
+            unit = "count"
+        elif "_Hours" in attribute_name:
+            unit = "hours"
+        elif "Total_LBAs_Written" in attribute_name:
+            unit = "blocks"
+        elif "Temperature_Cel" in attribute_name:
+            unit = "°C"
+        else:
+            unit = "unknown"
+
+        if unit == "°C":
+            # Sometimes there is extra information included - just ignore that.
+            value = int(raw_value.split()[0])
+        elif unit == "hours":
+            if m := re.match(r"([0-9]+)h\+([0-9]+)m\+([0-9.]+)s", raw_value):
+                # e.g. 60633h+54m+11.557s
+                value = (int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(2))) / 3600
+            else:
+                value = int(raw_value)
+        else:
+            value = int(raw_value)
+
+        report0.append(
+            {
+                "measure": "smart_" + attribute_name.lower(),
+                "unit": unit,
+                "value": value,
+            })
+now = time.time()
+report = [
+    {
+        "description": {
+            "hostname": ltsdb_record.node,
+            "device": args.device,
+            "measure": r["measure"],
+            "unit": r["unit"],
+        },
+        "data": [
+            [now, r["value"]]
+        ]
+    }
+    for r in report0
+]
+
+success = ltsdb_record.record_observations(report)
+exit(1 - success)
+

clients/record_tlscert

@@ -1,14 +1,11 @@
 #!/usr/bin/python3
 
 import argparse
-import hmac
-import json
-import os
 import socket
 import ssl
 import time
 
-import requests
+import ltsdb_record
 
 ap = argparse.ArgumentParser()
 ap.add_argument("--verbose", action="store_true")
@@ -16,23 +13,20 @@ ap.add_argument("hostname")
 ap.add_argument("port", type=int, default=443, nargs="?")
 args = ap.parse_args()
 
-# It's a bit weird that this works.
-myhostname = socket.gethostbyaddr(socket.gethostname())[0]
-
 now = time.time()
 report0 = []
 
-with socket.create_connection((args.hostname, args.port)) as sock:
-    context = ssl.create_default_context()
-    try:
+try:
+    with socket.create_connection((args.hostname, args.port)) as sock:
+        context = ssl.create_default_context()
         with context.wrap_socket(sock, server_hostname=args.hostname) as ssock:
             cert = ssock.getpeercert()
             not_after = ssl.cert_time_to_seconds(cert["notAfter"])
             delta = not_after - now
-    except ssl.SSLCertVerificationError as e:
-        print("got error %s; setting delta to 0", e)
-        delta = 0
-    report0.append({ "measure": "tls_cert_ttl", "unit": "s", "value": delta })
+except (ssl.SSLCertVerificationError, ConnectionRefusedError) as e:
+    print("got error %s; setting delta to 0" % e)
+    delta = 0
+report0.append({ "measure": "tls_cert_ttl", "unit": "s", "value": delta })
 
 report = [
     {
@@ -49,32 +43,5 @@ report = [
     for r in report0
 ]
 
-for dir in (".", os.environ["HOME"] + "/.config/ltsdb", "/etc/ltsdb"):
-    try:
-        with open(dir + "/config.json") as fh:
-            client_config = json.load(fh)
-        baseurl = client_config["server"]
-        break
-    except FileNotFoundError:
-        pass
-while True:
-    for r in report:
-        node = myhostname
-        timestamp = time.time()
-        msg = (node + " " + str(timestamp)).encode("UTF-8")
-        digest = hmac.new(client_config["key"].encode("UTF-8"), msg, "SHA256").hexdigest()
-        r["auth"] = {
-            "node": node,
-            "timestamp": timestamp,
-            "hmac": digest,
-        }
-    #pprint.pp(report)
-    r = requests.post(baseurl + "report", json=report)
-    print(r)
-    if r.status_code == 200:
-        exit(0)
-    elif r.status_code == 409:
-        time.sleep(0.5 + random.random())
-        continue
-    else:
-        exit(1)
+success = ltsdb_record.record_observations(report)
+exit(1 - success)

dashboard.py

@@ -27,7 +27,11 @@ class Dashboard:
                 if w.get("multi"):
                     ts_list = LTS.find(w["data"][0])
                     for ts in ts_list:
-                        tso = LTS(id=ts)
+                        try:
+                            tso = LTS(id=ts)
+                        except json.decoder.JSONDecodeError as e:
+                            log.error("%s contains bad data: %s: Skipping", ts, e)
+                            continue
                         if not tso.data:
                             log.warning("%s has no data: Skipping", tso.id)
                             continue
@@ -139,9 +143,13 @@ class TimeSeries(Widget):
 
         def v2y(v):
             if self.yscale == "log":
-                return (1 -   math.log(v / min_value)
-                            / math.log(max_value / min_value)
-                       ) * 200
+                try:
+                    return (1 -   math.log(max(v, min_value) / min_value)
+                                / math.log(max_value / min_value)
+                           ) * 200
+                except ValueError:
+                    log.error(f"ValueError: v = {v}, min_value = {min_value}, max_value = {max_value}")
+                    return 0
             elif self.yscale == "linear":
                 return (1 - v/max_value) * 200
             else:
@@ -384,10 +392,14 @@ class TimeSeries(Widget):
 
         max_value = max([d[3] if len(d) >= 4 else d[1] for d in self.lts.data])
         max_value = max(max_value, 0.001) # ensure positive
+        unit = self.lts.description["unit"]
         if self.yscale == "log":
             try:
                 min_value = min(d[1] for d in self.lts.data if d[1] > 0)
-                self.extra["min"] = "%g" % min_value
+                if unit == "s":
+                    self.extra["min"] = "%g" % min_value  + " (" + self.format_time(min_value) + ")"
+                else:
+                    self.extra["min"] = "%g" % min_value
             except ValueError:
                 # no non-negative values
                 min_value = max_value / 2
@@ -401,8 +413,12 @@ class TimeSeries(Widget):
                 # Make sure min_value is less than max_value
                 min_value /= 2
             log.debug("min_value = %s, max_value = %s", min_value, max_value)
-        self.extra["max"] = "%g" % max_value
-        self.extra["last"] = "%g" % data[-1][1]
+        if unit == "s":
+            self.extra["max"] = "%g" % max_value + " (" + self.format_time(max_value) + ")"
+            self.extra["last"] = "%g" % data[-1][1] + " (" + self.format_time(data[-1][1]) + ")"
+        else:
+            self.extra["max"] = "%g" % max_value
+            self.extra["last"] = "%g" % data[-1][1]
         log.debug("collecting data")
         v_data = []
         for i in range(n):
@@ -456,6 +472,29 @@ class TimeSeries(Widget):
         log.debug("in as_html")
         return Markup(render_template("timeseries.html", widget=self))
 
+    def format_time(self, seconds):
+        value = seconds
+        unit = "s"
+        if value >= 365.25 * 86400:
+            value /= 365.25 * 86400
+            unit = "years"
+        elif value >= 86400:
+            value /= 86400
+            unit = "days"
+        elif value >= 3600:
+            value /= 3600
+            unit = "h"
+        elif value >= 60:
+            value /= 60
+            unit = "m"
+        elif value >= 1:
+            pass
+        elif value >= 0.001:
+            value *= 1000
+            unit = "ms"
+        return f"{value:.2f} {unit}"
+
+
 class Gauge(Widget):
     def __init__(self, d):
         super().__init__(d)
@@ -497,3 +536,4 @@ class Gauge(Widget):
         self.lastvalue_formatted = Markup(f"<span class='value'>{value:.2f}</span><span class='unit'>{unit}</unit>")
         return Markup(render_template("gauge.html", widget=self))
 
+# vim: sw=4

ltsdb_json.py

@@ -41,7 +41,11 @@ class LTS:
             with open(self.filename, "x+") as fh:
                 fcntl.flock(fh, fcntl.LOCK_EX)
                 json.dump({"description": self.description, "data": self.data}, fh)
+                log.info(f"Created {self.filename}")
             self.rebuild_index()
+        except json.decoder.JSONDecodeError as e:
+            log.exception(f"Cannot decode JSON in {self.filename}: {e}")
+            raise
 
     def pop(self, i):
         # Pop the element at index i and adjust the min/max values of the
@@ -127,7 +131,11 @@ class LTS:
             (_, _, hash) = fn.rpartition("/")
             with open(fn, "r") as fh:
                 fcntl.flock(fh, fcntl.LOCK_SH)
-                d = json.load(fh)
+                try:
+                    d = json.load(fh)
+                except json.decoder.JSONDecodeError as e:
+                    log.exception(f"Cannot decode JSON in {fn}: {e}")
+                    raise
             for k, v in d["description"].items():
                 d1 = index.setdefault(k, {})
                 d2 = d1.setdefault(v, [])

ltsdb_test

@@ -1,20 +1,22 @@
 #!/usr/bin/python3
+import time
+import random
 
 from ltsdb_json import LTS
 
 ts1 = LTS({"hostname": "rorschach.hjp.at", "measure": "uptime"})
-ts1.add(1661026122, 4)
+ts1.add(time.time(), 4 + random.random())
 ts1.save()
 
 ts1 = LTS({"hostname": "rorschach.hjp.at", "website": "i12e.hjp.at", "measure": "rtt"})
-ts1.add(1661026122, 0.06)
+ts1.add(time.time(), 0.06)
 ts1.save()
 
 ts1 = LTS({"hostname": "rorschach.hjp.at", "measure": "uptime"})
-ts1.add(1661026361, 5)
+ts1.add(time.time() + 240, 5 + random.random())
 ts1.save()
 
 ts1 = LTS({"hostname": "charly.wsr.ac.at", "website": "www.wifo.ac.at", "measure": "rtt"})
-ts1.add(1661026122, 0.347)
+ts1.add(time.time(), 0.347)
 ts1.save()
 

process_queue

@@ -2,8 +2,10 @@
 
 import logging
 import logging.config
+import math
 import os
 import socket
+import statistics
 import time
 
 from ltsdb_json import LTS
@@ -44,13 +46,37 @@ class DiskFullPredictor:
         current_used_bytes = lts.data[-1][1]
         current_usable_bytes = usable_lts.data[-1][1]
         tuf = 1E9
-        for d in reversed(lts.data):
-            if d[1] < current_usable_bytes * 0.1:
+        for i in reversed(range(len(lts.data))):
+            m = statistics.mean(x[1] for x in lts.data[max(0, i - 2) : min(len(lts.data), i + 3)])
+            if m < current_usable_bytes * 0.1:
                 continue # for sanity
-            if current_used_bytes ** 2 / d[1] > current_usable_bytes:
-                log.info("d = %s, current_used_bytes = %s, current_usable_bytes = %s", d, current_used_bytes, current_usable_bytes)
-                tuf = now - d[0]
+            if current_used_bytes ** 2 / m > current_usable_bytes:
+                log.info("d = %s, current_used_bytes = %s, current_usable_bytes = %s", m, current_used_bytes, current_usable_bytes)
+                tuf = now - lts.data[i][0]
                 break
+        else:
+            # Try always use the minimum of a range.
+            # We prefer the first datapoint 
+            first_used_bytes = lts.data[0][2] if len(lts.data[0]) >= 4 else lts.data[0][1]
+            # But if that's not useable we search the whole timeseries for the
+            # minimum
+            if first_used_bytes >= current_used_bytes:
+                first_used_bytes = current_used_bytes
+                first_i = None
+                for i in range(len(lts.data)):
+                    used_bytes = lts.data[i][2] if len(lts.data[i]) >= 4 else lts.data[i][1]
+                    if used_bytes < first_used_bytes:
+                        first_used_bytes = used_bytes
+                        first_i = i
+            else:
+                first_i = 0
+
+            if first_i is not None:
+                historic_growth = current_used_bytes / first_used_bytes
+                future_growth = current_usable_bytes  / current_used_bytes
+                tuf = math.log(future_growth) / math.log(historic_growth) * (now - lts.data[first_i][0])
+                tuf = max(tuf, now - lts.data[first_i][0])
+                tuf = min(tuf, 1E9)
         desc = {**lts.description,
              "measure": "time_until_disk_full",
              "node": node,

templates/dashboard.html

@@ -8,8 +8,9 @@
       body {
         font-family: sans-serif; 
       }
-      th {
+      th, td {
         text-align: left;
+        vertical-align: baseline;
       }
       main {
         display: flex;

templates/nav_dimension_list.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width; initial-scale=1">
+    <meta charset="utf-8">
+
+    <style>
+    </style>
+  </head>
+  <body>
+    <ul>
+      {% for d in matching_dimensions %}
+        <li><a href="{{d['url']}}">{{d.name}}</a> ({{d.count}})</li>
+      {% endfor %}
+    </ul>
+    {{timeseries|length}}/{{total_timeseries}} timeseries:
+    <ul>
+      {% for ts in timeseries %}
+        <li>
+          <a href="/v?ts={{ts.id}}">{{ts.description}}</a>
+        </li>
+      {% endfor %}
+    </ul>
+  </body>
+</html>
+

templates/nav_member_select.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width; initial-scale=1">
+    <meta charset="utf-8">
+
+    <style>
+    </style>
+  </head>
+  <body>
+    <form>
+      {% for dimension, members in request.args.lists() %}
+        {% for member in members %}
+          {% if dimension[0] != "." %}
+            <input name="{{dimension}}" value="{{member}}" type="hidden">
+          {% endif %}
+        {% endfor %}
+      {% endfor %}
+      <select name="{{dimension}}" multiple size={{members|length}}>
+        {% for member in members %}
+          <option>{{member}}</option>
+        {% endfor %}
+      </select>
+      <input type="submit">
+    </form>
+  </body>
+</html>