Add navigation templates

So far we have only extapolated as far into the future as we could look
into the past. Everything beyond that was "infinity". Now we use the
first and last observation to extrapolate beyond that.

app.py

@@ -5,7 +5,9 @@ import logging
 import logging.config
 import os
 
-from flask import (Flask, request, jsonify, abort, render_template)
+from collections import defaultdict
+
+from flask import (Flask, request, jsonify, abort, render_template, url_for)
 
 from ltsdb_json import LTS
 from dashboard import Dashboard
@@ -152,3 +154,71 @@ def dashboard_index():
 def dashboard_file(dashboard):
     d = Dashboard("dashboards/" + dashboard + ".json")
     return d.as_html()
+
+@app.get("/nav")
+def nav():
+    # Start with a list of all dimensions, the number of matching time series
+    # and a truncated list of series.
+    # If a dimension is chosen, display a choice of members
+    # choosing one or more members goes back to the list of
+    # (remaining) dimensions
+    with open("data/.index") as fh:
+        fcntl.flock(fh, fcntl.LOCK_SH)
+        index = json.load(fh)
+    timeseries = None
+    for k, v in request.args.lists():
+        if k[0] == ".":
+            continue
+        log.debug("search: %s -> %s", k, v)
+        if timeseries is None:
+            timeseries = set()
+            log.debug("search: %s: %s", k, index[k])
+            for m in v:
+                timeseries |= set(index[k][m])
+        else:
+            filter = set()
+            for m in v:
+                filter |= set(index[k][m])
+            timeseries &= filter
+    if timeseries is None:
+        timeseries = set()
+        for mc in index.values():
+            for tsl in mc.values():
+                timeseries |= set(tsl)
+    if d := request.args.get(".m"):
+        members = []
+        for m, tsl in index[d].items():
+            if set(tsl) & timeseries:
+                members.append(m)
+        return render_template("nav_member_select.html", dimension=d, members=members)
+    else:
+        params = request.args.to_dict(flat=False)
+        matching_dimensions = defaultdict(int)
+        for d, mc in index.items():
+            if d in params:
+                continue
+            for m, tsl in mc.items():
+                mtsl = set(tsl) & timeseries
+                if mtsl:
+                    matching_dimensions[d] += len(mtsl)
+        matching_dimensions_list = []
+        for d in matching_dimensions:
+            params[".m"] = d
+            url = url_for("nav", **params)
+            app.logger.debug(f"{d=} {url=}")
+            matching_dimensions_list.append(
+                {"name": d, "count": matching_dimensions[d], "url": url}
+            )
+        total_timeseries = len(timeseries)
+        timeseries = [LTS(id=ts) for ts in list(timeseries)[:100]]
+        return render_template(
+                    "nav_dimension_list.html",
+                    matching_dimensions=matching_dimensions_list,
+                    timeseries=timeseries, total_timeseries=total_timeseries)
+
+
+
+
+
+
+    # 

dashboard.py

@@ -27,7 +27,11 @@ class Dashboard:
                 if w.get("multi"):
                     ts_list = LTS.find(w["data"][0])
                     for ts in ts_list:
-                        tso = LTS(id=ts)
+                        try:
+                            tso = LTS(id=ts)
+                        except json.decoder.JSONDecodeError as e:
+                            log.error("%s contains bad data: %s: Skipping", ts, e)
+                            continue
                         if not tso.data:
                             log.warning("%s has no data: Skipping", tso.id)
                             continue
@@ -388,10 +392,14 @@ class TimeSeries(Widget):
 
         max_value = max([d[3] if len(d) >= 4 else d[1] for d in self.lts.data])
         max_value = max(max_value, 0.001) # ensure positive
+        unit = self.lts.description["unit"]
         if self.yscale == "log":
             try:
                 min_value = min(d[1] for d in self.lts.data if d[1] > 0)
-                self.extra["min"] = "%g" % min_value
+                if unit == "s":
+                    self.extra["min"] = "%g" % min_value  + " (" + self.format_time(min_value) + ")"
+                else:
+                    self.extra["min"] = "%g" % min_value
             except ValueError:
                 # no non-negative values
                 min_value = max_value / 2
@@ -405,8 +413,12 @@ class TimeSeries(Widget):
                 # Make sure min_value is less than max_value
                 min_value /= 2
             log.debug("min_value = %s, max_value = %s", min_value, max_value)
-        self.extra["max"] = "%g" % max_value
-        self.extra["last"] = "%g" % data[-1][1]
+        if unit == "s":
+            self.extra["max"] = "%g" % max_value + " (" + self.format_time(max_value) + ")"
+            self.extra["last"] = "%g" % data[-1][1] + " (" + self.format_time(data[-1][1]) + ")"
+        else:
+            self.extra["max"] = "%g" % max_value
+            self.extra["last"] = "%g" % data[-1][1]
         log.debug("collecting data")
         v_data = []
         for i in range(n):
@@ -460,6 +472,29 @@ class TimeSeries(Widget):
         log.debug("in as_html")
         return Markup(render_template("timeseries.html", widget=self))
 
+    def format_time(self, seconds):
+        value = seconds
+        unit = "s"
+        if value >= 365.25 * 86400:
+            value /= 365.25 * 86400
+            unit = "years"
+        elif value >= 86400:
+            value /= 86400
+            unit = "days"
+        elif value >= 3600:
+            value /= 3600
+            unit = "h"
+        elif value >= 60:
+            value /= 60
+            unit = "m"
+        elif value >= 1:
+            pass
+        elif value >= 0.001:
+            value *= 1000
+            unit = "ms"
+        return f"{value:.2f} {unit}"
+
+
 class Gauge(Widget):
     def __init__(self, d):
         super().__init__(d)
@@ -501,3 +536,4 @@ class Gauge(Widget):
         self.lastvalue_formatted = Markup(f"<span class='value'>{value:.2f}</span><span class='unit'>{unit}</unit>")
         return Markup(render_template("gauge.html", widget=self))
 
+# vim: sw=4

ltsdb_json.py

@@ -41,7 +41,11 @@ class LTS:
             with open(self.filename, "x+") as fh:
                 fcntl.flock(fh, fcntl.LOCK_EX)
                 json.dump({"description": self.description, "data": self.data}, fh)
+                log.info(f"Created {self.filename}")
             self.rebuild_index()
+        except json.decoder.JSONDecodeError as e:
+            log.exception(f"Cannot decode JSON in {self.filename}: {e}")
+            raise
 
     def pop(self, i):
         # Pop the element at index i and adjust the min/max values of the
@@ -127,7 +131,11 @@ class LTS:
             (_, _, hash) = fn.rpartition("/")
             with open(fn, "r") as fh:
                 fcntl.flock(fh, fcntl.LOCK_SH)
-                d = json.load(fh)
+                try:
+                    d = json.load(fh)
+                except json.decoder.JSONDecodeError as e:
+                    log.exception(f"Cannot decode JSON in {fn}: {e}")
+                    raise
             for k, v in d["description"].items():
                 d1 = index.setdefault(k, {})
                 d2 = d1.setdefault(v, [])

process_queue

@@ -2,6 +2,7 @@
 
 import logging
 import logging.config
+import math
 import os
 import socket
 import statistics
@@ -53,6 +54,29 @@ class DiskFullPredictor:
                 log.info("d = %s, current_used_bytes = %s, current_usable_bytes = %s", m, current_used_bytes, current_usable_bytes)
                 tuf = now - lts.data[i][0]
                 break
+        else:
+            # Try always use the minimum of a range.
+            # We prefer the first datapoint 
+            first_used_bytes = lts.data[0][2] if len(lts.data[0]) >= 4 else lts.data[0][1]
+            # But if that's not useable we search the whole timeseries for the
+            # minimum
+            if first_used_bytes >= current_used_bytes:
+                first_used_bytes = current_used_bytes
+                first_i = None
+                for i in range(len(lts.data)):
+                    used_bytes = lts.data[i][2] if len(lts.data[i]) >= 4 else lts.data[i][1]
+                    if used_bytes < first_used_bytes:
+                        first_used_bytes = used_bytes
+                        first_i = i
+            else:
+                first_i = 0
+
+            if first_i is not None:
+                historic_growth = current_used_bytes / first_used_bytes
+                future_growth = current_usable_bytes  / current_used_bytes
+                tuf = math.log(future_growth) / math.log(historic_growth) * (now - lts.data[first_i][0])
+                tuf = max(tuf, now - lts.data[first_i][0])
+                tuf = min(tuf, 1E9)
         desc = {**lts.description,
              "measure": "time_until_disk_full",
              "node": node,

templates/dashboard.html

@@ -8,8 +8,9 @@
       body {
         font-family: sans-serif; 
       }
-      th {
+      th, td {
         text-align: left;
+        vertical-align: baseline;
       }
       main {
         display: flex;

templates/nav_dimension_list.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width; initial-scale=1">
+    <meta charset="utf-8">
+
+    <style>
+    </style>
+  </head>
+  <body>
+    <ul>
+      {% for d in matching_dimensions %}
+        <li><a href="{{d['url']}}">{{d.name}}</a> ({{d.count}})</li>
+      {% endfor %}
+    </ul>
+    {{timeseries|length}}/{{total_timeseries}} timeseries:
+    <ul>
+      {% for ts in timeseries %}
+        <li>
+          <a href="/v?ts={{ts.id}}">{{ts.description}}</a>
+        </li>
+      {% endfor %}
+    </ul>
+  </body>
+</html>
+

templates/nav_member_select.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width; initial-scale=1">
+    <meta charset="utf-8">
+
+    <style>
+    </style>
+  </head>
+  <body>
+    <form>
+      {% for dimension, members in request.args.lists() %}
+        {% for member in members %}
+          {% if dimension[0] != "." %}
+            <input name="{{dimension}}" value="{{member}}" type="hidden">
+          {% endif %}
+        {% endfor %}
+      {% endfor %}
+      <select name="{{dimension}}" multiple size={{members|length}}>
+        {% for member in members %}
+          <option>{{member}}</option>
+        {% endfor %}
+      </select>
+      <input type="submit">
+    </form>
+  </body>
+</html>