import datetime import json import logging import math import time from flask import (Flask, request, jsonify, abort, render_template_string, render_template) from markupsafe import Markup from ltsdb_json import LTS log = logging.getLogger(__name__) class Dashboard: def __init__(self, filename="dashboard.json"): with open (filename) as fh: d = json.load(fh) self.title = d["title"] log.debug("title = ā€œ%sā€", self.title) self.description = d["description"] self.widgets = [] for w in d["widgets"]: if w["type"] == "timeseries": # XXX - currently just a copy of the gauge code, # but I want to add support for timeseries graphs with # multiple series later so this will change if w.get("multi"): ts_list = LTS.find(w["data"][0]) for ts in ts_list: tso = LTS(id=ts) if not tso.data: log.warning("%s has no data: Skipping", tso.id) continue if tso.data[-1][0] < time.time() - 7 * 86400: log.info("%s too old; Skipping", tso.id) continue w1 = {**w, "data": [ts]} self.widgets.append(TimeSeries(w1)) else: self.widgets.append(TimeSeries(w)) elif w["type"] == "gauge": if w.get("multi"): ts_list = LTS.find(w["data"][0]) for ts in ts_list: tso = LTS(id=ts) if not tso.data: log.warning("%s has no data: Skipping", tso.id) continue if tso.data[-1][0] < time.time() - 86400: log.info("%s too old; Skipping", tso.id) continue w1 = {**w, "data": [ts]} self.widgets.append(Gauge(w1)) else: self.widgets.append(Gauge(w)) else: self.widgets.append(Widget(w)) # Sort widgets by ascending healthscore to get the most critical at the # top. self.widgets.sort(key=lambda w: w.healthscore()) def as_html(self): return render_template("dashboard.html", dashboard=self) class Widget: def __init__(self, d): log.debug("") self.type = d["type"] self.stops = d["stops"] self.yscale = d.get("yscale", "linear") self.extra = {} log.debug("data = %s", d["data"]) self.lts = LTS(id=d["data"][0]) # by default we handle only one data source self.lastvalue = self.lts.data[-1][1] def as_html(self): log.debug("") return Markup(render_template("widget.html", widget=self)) def healthscore(self, value=None): """ Return a score between 0 (unhealthy) and 100 (healthy) """ if value == None: value = self.lastvalue log.debug("stops = %s", self.stops) stops = self.stops if stops[-1] < stops[0]: value = -value stops = [-v for v in stops] if value <= stops[0]: log.debug("ok") return 100 if value >= stops[-1]: log.debug("fail") return 0 for i in range(0, len(stops) - 1): if stops[i] <= value < stops[i+1]: log.debug("at stop %d", i) return 100 - ((value - stops[i]) / (stops[i+1] - stops[i]) + i) * 100 / (len(stops) - 1) def criticalcolor(self, value=None): healthscore = self.healthscore(value) hue = round(healthscore * 120 / 100) brightness = 30 return f"hsl({hue}, 100%, {brightness}%)" @property def description_formatted(self): s = "" for d, v in self.lts.description.items(): if v: s += render_template_string( "", d=d, v=v) for d, v in self.extra.items(): if v: s += render_template_string( "", d=d, v=v) s += "
{{d}}:{{v}}
{{d}}:{{v}}
" return Markup(s) class TimeSeries(Widget): def __init__(self, d): log.debug("") super().__init__(d) pass @property def graph(self): def t2x(t): x = n - math.log((t_last - t) / (n*dt) + 1) * n / k return x def v2y(v): if self.yscale == "log": return (1 - math.log(v / min_value) / math.log(max_value / min_value) ) * 200 elif self.yscale == "linear": return (1 - v/max_value) * 200 else: raise ValueError(f"Unknown yscale {self.yscale}") def set_x_tickmarks(): log.debug("") tickmarks = [] t = v_data[-1]["t"] x = v_data[-1]["x"] d = datetime.datetime.fromtimestamp(t) tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) min_step = 25 steps = ("s", "m", "h", "D", "10D", "M", "Y") step_i = 0 while step_i < len(steps): t0 = tickmarks[-1]["t"] x0 = tickmarks[-1]["x"] d0 = datetime.datetime.fromtimestamp(t0) log.debug("step_i = %s", step_i) if steps[step_i] == "s": d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute, d0.second) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: d1 -= datetime.timedelta(seconds=1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "m": d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: d1 -= datetime.timedelta(minutes=1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "h": d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: d1 -= datetime.timedelta(hours=1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "D": d1 = datetime.datetime(d0.year, d0.month, d0.day) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: d1 -= datetime.timedelta(days=1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "10D": year = d0.year month = d0.month day = d0.day if day > 21: day = 21 elif day > 11: day = 11 elif day > 1: day = 1 else: day = 21 month -= 1 if month < 1: month += 12 year -= 1 d1 = datetime.datetime(year, month, day) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: if day > 21: day = 21 elif day > 11: day = 11 elif day > 1: day = 1 else: day = 21 month -= 1 if month < 1: month += 12 year -= 1 d1 = datetime.datetime(year, month, day) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "M": d1 = datetime.datetime(d0.year, d0.month, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: if d1.month > 1: d1 = datetime.datetime(d1.year, d1.month-1, 1) else: d1 = datetime.datetime(d1.year-1, 12, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: step_i += 1 continue if steps[step_i] == "Y": d1 = datetime.datetime(d0.year, 1, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: d1 = datetime.datetime(d1.year-1, 1, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: log.debug("t0 = %s, x0 = %s, t1 = %s, x1 = %s", t0, x0, t1, x1) step_i += 1 continue if x1 < 0: break tickmarks.append({"t": t1, "t_h": d1.strftime("%Y-%m-%d %H:%M:%S"), "x": x1}) if tickmarks[-1]["x"] > min_step: t = v_data[0]["t"] x = v_data[0]["x"] d = datetime.datetime.fromtimestamp(t) tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) self.x_tickmarks = tickmarks log.debug("") def set_y_tickmarks(): log.debug("") unit = self.lts.description["unit"] self.y_tickmarks = [] if self.yscale == "linear": log.debug("") if unit == "s" and max_value > 3600: if max_value >= 16 * 7 * 86400: step = 4 * 7 * 86400 step_d = 4 unit = "w" elif max_value >= 4 * 7 * 86400: step = 7 * 86400 step_d = 1 unit = "w" elif max_value >= 10 * 86400: step = 3 * 86400 step_d = 3 unit = "d" elif max_value >= 4 * 86400: step = 86400 step_d = 1 unit = "d" elif max_value >= 4 * 3600: step = 3600 step_d = 1 unit = "h" else: step = 10 * 60 step_d = 10 unit = "m" v = 0 v_d = 0 while v < max_value: y = v2y(v) log.debug("v = %s, y = %s", v, y) self.y_tickmarks.append({"y": y, "v_h": f"{v_d} {unit}"}) v += step v_d += step_d else: step = 10 ** math.floor(math.log10(max_value)) v = 0 while v < max_value: y = v2y(v) log.debug("v = %s, y = %s", v, y) self.y_tickmarks.append({"y": y, "v_h": str(v)}) v += step log.debug("") elif self.yscale == "log": if unit == "s" and max_value > 3600: steps = ( (3600, "1 h"), (86400, "1 d"), (7*86400, "1 w"), (28*86400, "4 w"), (365*86400, "1 y"), (3652.5 * 86400, "10 y"), ) for s in steps: if min_value <= s[0] <= max_value: self.y_tickmarks.append({"y": v2y(s[0]), "v_h": s[1]}) else: log.debug("") v = 10 ** math.ceil(math.log10(min_value)) log.debug("v = %s", v) if v > max_value: # Implement that when it happens log.warning("No tickmark between %s and %s", min_value, max_value) return while v <= max_value: y = v2y(v) log.debug("v = %s, y = %s", v, y) self.y_tickmarks.append({"y": y, "v_h": str(v)}) v *= 10 log.debug("") else: log.debug("") raise ValueError(f"Unknown yscale {self.yscale}") log.debug("") log.debug("in graph for %s", self.lts.id) data = self.lts.data n = len(data) t_last = data[-1][0] if len(data) < 5: return "(not enough data)" dt = (t_last - data[-5][0]) / 4 k = math.log((t_last - data[0][0]) / dt / n + 1) log.debug("times = [%s ... %s ... %s]", data[0][0], data[-5][0], data[-1][0]) max_value = max([d[3] if len(d) >= 4 else d[1] for d in self.lts.data]) max_value = max(max_value, 0.001) # ensure positive if self.yscale == "log": try: min_value = min(d[1] for d in self.lts.data if d[1] > 0) self.extra["min"] = "%g" % min_value except ValueError: # no non-negative values min_value = max_value / 2 if any(True for d in self.lts.data if d[1] < min_value): # if there are values smaller than the minimum (i.e. # 0 or negative unless we have a configured minimum) # reduce the minimum again so that we have a distinct # value to clamp to. min_value /= 2 if min_value == max_value: # Make sure min_value is less than max_value min_value /= 2 log.debug("min_value = %s, max_value = %s", min_value, max_value) self.extra["max"] = "%g" % max_value self.extra["last"] = "%g" % data[-1][1] log.debug("collecting data") v_data = [] for i in range(n): t = data[i][0] v = data[i][1] if len(data[i]) >= 4: v_min = data[i][2] v_max = data[i][3] else: v_min = data[i][1] v_max = data[i][1] x = t2x(t) t_h = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t)) y = v2y(v) y_min = v2y(v_min) y_max = v2y(v_max) #print(t, t_h, x) v_data.append( { "t": t, "v": v, "x": x, "y": y, "y_min": y_min, "y_max": y_max, "color": self.criticalcolor(v), }) log.debug("setting tickmarks") set_x_tickmarks() set_y_tickmarks() log.debug("assembling svg") html = "" html += "" for tm in self.x_tickmarks: html += f"" html += f"{tm['t_h']}" for tm in self.y_tickmarks: log.debug("tm = %s", tm) html += f"" html += f"{tm['v_h']}" for v in v_data: html += f"" html += f"" html += "" log.debug("len(html) = %s", len(html)) return Markup(html) def as_html(self): log.debug("in as_html") return Markup(render_template("timeseries.html", widget=self)) class Gauge(Widget): def __init__(self, d): super().__init__(d) pass gaugesize = 100 @property def gaugepos(self): max_value = max([d[1] for d in self.lts.data]) max_value = max(max_value, 1) # ensure positive log.debug("max_value = %s", max_value) return self.lastvalue / max_value * self.gaugesize def as_html(self): log.debug("") self.lastvalue = self.lts.data[-1][1] value = self.lastvalue unit = self.lts.description["unit"] log.debug("unit = %s", unit) if unit == "s": if value >= 365.25 * 86400: value /= 365.25 * 86400 unit = "years" elif value >= 86400: value /= 86400 unit = "days" elif value >= 3600: value /= 3600 unit = "h" elif value >= 60: value /= 60 unit = "m" elif value >= 1: pass elif value >= 0.001: value *= 1000 unit = "ms" self.lastvalue_formatted = Markup(f"{value:.2f}{unit}") return Markup(render_template("gauge.html", widget=self))