From fedc4c66ac08ceb7a839a3796cdedb9efb0e36fe Mon Sep 17 00:00:00 2001 From: "Peter J. Holzer" Date: Sat, 31 Dec 2022 16:58:38 +0100 Subject: [PATCH] Implement log scale for y axis --- dashboard.py | 315 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 191 insertions(+), 124 deletions(-) diff --git a/dashboard.py b/dashboard.py index fd389c9..6a225c8 100644 --- a/dashboard.py +++ b/dashboard.py @@ -63,8 +63,10 @@ class Dashboard: class Widget: def __init__(self, d): + log.debug("") self.type = d["type"] self.stops = d["stops"] + self.yscale = d.get("yscale", "linear") log.debug("data = %s", d["data"]) self.lts = LTS(id=d["data"][0]) # by default we handle only one data source pass @@ -140,6 +142,7 @@ class Widget: class TimeSeries(Widget): def __init__(self, d): + log.debug("") super().__init__(d) pass @@ -149,113 +152,84 @@ class TimeSeries(Widget): x = n - math.log((t_last - t) / (n*dt) + 1) * n / k return x - data = self.lts.data - n = len(data) - t_last = data[-1][0] - if len(data) < 5: - return "(not enough data)" - dt = (t_last - data[-5][0]) / 4 - k = math.log((t_last - data[0][0]) / dt / n + 1) + def v2y(v): + if self.yscale == "log": + return (1 - math.log(v / min_value) + / math.log(max_value / min_value) + ) * 200 + elif self.yscale == "linear": + return (1 - v/max_value) * 200, + else: + raise ValueError(f"Unknown yscale {self.yscale}") - max_value = max([d[1] for d in self.lts.data]) - max_value = max(max_value, 0.001) # ensure positive - v_data = [] - for i in range(n): - t = data[i][0] - x = t2x(t) - t_h = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t)) - #print(t, t_h, x) - v_data.append( - { - "t": t, - "v": data[i][1], - "x": x, - "y": (1 - data[i][1]/max_value) * 200, - "color": self.criticalcolor(data[i][1]), - }) + def set_x_tickmarks(): + log.debug("") + tickmarks = [] + t = v_data[-1]["t"] + x = v_data[-1]["x"] + d = datetime.datetime.fromtimestamp(t) + tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) - tickmarks = [] - t = v_data[-1]["t"] - x = v_data[-1]["x"] - d = datetime.datetime.fromtimestamp(t) - tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) + min_step = 25 + steps = ("s", "m", "h", "D", "10D", "M", "Y") + step_i = 0 + while True: + t0 = tickmarks[-1]["t"] + x0 = tickmarks[-1]["x"] + d0 = datetime.datetime.fromtimestamp(t0) - min_step = 25 - steps = ("s", "m", "h", "D", "10D", "M", "Y") - step_i = 0 - while True: - t0 = tickmarks[-1]["t"] - x0 = tickmarks[-1]["x"] - d0 = datetime.datetime.fromtimestamp(t0) - - if steps[step_i] == "s": - d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute, d0.second) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - d1 -= datetime.timedelta(seconds=1) + if steps[step_i] == "s": + d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute, d0.second) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + d1 -= datetime.timedelta(seconds=1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "m": - d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - d1 -= datetime.timedelta(minutes=1) + if steps[step_i] == "m": + d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour, d0.minute) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + d1 -= datetime.timedelta(minutes=1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "h": - d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - d1 -= datetime.timedelta(hours=1) + if steps[step_i] == "h": + d1 = datetime.datetime(d0.year, d0.month, d0.day, d0.hour) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + d1 -= datetime.timedelta(hours=1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "D": - d1 = datetime.datetime(d0.year, d0.month, d0.day) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - d1 -= datetime.timedelta(days=1) + if steps[step_i] == "D": + d1 = datetime.datetime(d0.year, d0.month, d0.day) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + d1 -= datetime.timedelta(days=1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "10D": - year = d0.year - month = d0.month - day = d0.day - if day > 21: - day = 21 - elif day > 11: - day = 11 - elif day > 1: - day = 1 - else: - day = 21 - month -= 1 - if month < 1: - month += 12 - year -= 1 - d1 = datetime.datetime(year, month, day) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: + if steps[step_i] == "10D": + year = d0.year + month = d0.month + day = d0.day if day > 21: day = 21 elif day > 11: @@ -272,52 +246,143 @@ class TimeSeries(Widget): t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + if day > 21: + day = 21 + elif day > 11: + day = 11 + elif day > 1: + day = 1 + else: + day = 21 + month -= 1 + if month < 1: + month += 12 + year -= 1 + d1 = datetime.datetime(year, month, day) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "M": - d1 = datetime.datetime(d0.year, d0.month, 1) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - if d1.month > 1: - d1 = datetime.datetime(d1.year, d1.month-1, 1) - else: - d1 = datetime.datetime(d1.year-1, 12, 1) + if steps[step_i] == "M": + d1 = datetime.datetime(d0.year, d0.month, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue + if d1.month > 1: + d1 = datetime.datetime(d1.year, d1.month-1, 1) + else: + d1 = datetime.datetime(d1.year-1, 12, 1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue - if steps[step_i] == "Y": - d1 = datetime.datetime(d0.year, 1, 1) - t1 = d1.timestamp() - x1 = t2x(t1) - if x0 - x1 < min_step: - d1 = datetime.datetime(d1.year-1, 1, 1) + if steps[step_i] == "Y": + d1 = datetime.datetime(d0.year, 1, 1) t1 = d1.timestamp() x1 = t2x(t1) if x0 - x1 < min_step: - step_i += 1 - continue - if x1 < 0: - break - tickmarks.append({"t": t1, "t_h": d1.strftime("%Y-%m-%d %H:%M:%S"), "x": x1}) - if tickmarks[-1]["x"] > min_step: - t = v_data[0]["t"] - x = v_data[0]["x"] - d = datetime.datetime.fromtimestamp(t) - tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) - self.x_tickmarks = tickmarks + d1 = datetime.datetime(d1.year-1, 1, 1) + t1 = d1.timestamp() + x1 = t2x(t1) + if x0 - x1 < min_step: + step_i += 1 + continue + if x1 < 0: + break + tickmarks.append({"t": t1, "t_h": d1.strftime("%Y-%m-%d %H:%M:%S"), "x": x1}) + if tickmarks[-1]["x"] > min_step: + t = v_data[0]["t"] + x = v_data[0]["x"] + d = datetime.datetime.fromtimestamp(t) + tickmarks.append({"t": t, "t_h": d.strftime("%Y-%m-%d %H:%M:%S"), "x": x}) + self.x_tickmarks = tickmarks + log.debug("") - self.y_tickmarks = [] - step = 10 ** math.floor(math.log10(max_value)) - v = 0 - while v < max_value: - self.y_tickmarks.append({"y": (1 - v/max_value) * 200, "v_h": str(v)}) - v += step + def set_y_tickmarks(): + log.debug("") + self.y_tickmarks = [] + if self.yscale == "linear": + log.debug("") + step = 10 ** math.floor(math.log10(max_value)) + v = 0 + while v < max_value: + self.y_tickmarks.append({"y": v2y(v), "v_h": str(v)}) + v += step + log.debug("") + elif self.yscale == "log": + log.debug("") + v = 10 ** math.ceil(math.log10(min_value)) + log.debug("v = %s", v) + if v > max_value: + # Implement that when it happens + log.warning("No tickmark between %s and %s", min_value, max_value) + return + while v <= max_value: + y = v2y(v) + log.debug("v = %s, y = %s", v, y) + self.y_tickmarks.append({"y": y, "v_h": str(v)}) + v *= 10 + log.debug("") + else: + log.debug("") + raise ValueError(f"Unknown yscale {self.yscale}") + log.debug("") + + log.debug("in graph") + data = self.lts.data + n = len(data) + t_last = data[-1][0] + if len(data) < 5: + return "(not enough data)" + dt = (t_last - data[-5][0]) / 4 + k = math.log((t_last - data[0][0]) / dt / n + 1) + + max_value = max([d[1] for d in self.lts.data]) + max_value = max(max_value, 0.001) # ensure positive + if self.yscale == "log": + try: + min_value = min(d[1] for d in self.lts.data if d[1] > 0) + except ValueError: + # no non-negative values + min_value = max_value / 2 + if any(True for d in self.lts.data if d[1] < min_value): + # if there are values smaller than the minimum (i.e. + # 0 or negative unless we have a configured minimum) + # reduce the minimum again so that we have a distinct + # value to clamp to. + min_value /= 2 + if min_value == max_value: + # Make sure min_value is less than max_value + min_value /= 2 + log.debug("min_value = %s, max_value = %s", min_value, max_value) + log.debug("collecting data") + v_data = [] + for i in range(n): + t = data[i][0] + v = data[i][1] + x = t2x(t) + t_h = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t)) + y = v2y(v) + #print(t, t_h, x) + v_data.append( + { + "t": t, + "v": v, + "x": x, + "y": y, + "color": self.criticalcolor(v), + }) + + log.debug("setting tickmarks") + set_x_tickmarks() + set_y_tickmarks() + + log.debug("assembling svg") html = "" html += "" for tm in self.x_tickmarks: @@ -329,9 +394,11 @@ class TimeSeries(Widget): for v in v_data: html += f"" html += "" + log.debug("len(html) = %s", len(html)) return Markup(html) def as_html(self): + log.debug("in as_html") return Markup(render_template("timeseries.html", widget=self)) class Gauge(Widget):