ltsdb/app.py

import fcntl
import hmac
import json
import logging
import logging.config
import os

from collections import defaultdict

from flask import (Flask, request, jsonify, abort, render_template, url_for)

from ltsdb_json import LTS
from dashboard import Dashboard

import config

logging.config.dictConfig(config.logging)

app = Flask(__name__)

log = logging.getLogger()

@app.route("/")
def home():
    return jsonify({ "success": None })


@app.route("/report", methods=["POST"])
def report():
    return record()

@app.route("/record", methods=["POST"])
def record():
    data = request.get_json()
    n_ts = 0
    n_dp = 0
    for d in data:
        d["description"]["remote_addr"] = request.remote_addr
        d["description"]["node"] = verify_node(d)

        log.info("received %s", json.dumps(d))
        ts = LTS(d["description"])
        for dp in d["data"]:
            ts.add(*dp)
            ts.save()
            n_dp += 1
        n_ts += 1
    return jsonify({ "success": True, "timeseries": n_ts, "datapoints": n_dp })

@app.route("/ts/<id>")
def get_timeseries(id):
    try:
        ts = LTS(id=id)
    except FileNotFoundError:
        abort(404)
    return jsonify({"description": ts.description, "data": ts.data})

@app.route("/dimensions")
def list_dimensions():
    with open("data/.index") as fh:
        fcntl.flock(fh, fcntl.LOCK_SH)
        index = json.load(fh)
    # Just return the number of timeseries for each dimension/member, not
    # the timeseries themselves
    for d in index.keys():
        for m in index[d].keys():
            index[d][m] = len(index[d][m])
    return jsonify(index)

@app.route("/search")
def search():
    log.debug("search: %s", request.args)
    return jsonify(_search())

def _search():
    timeseries = None
    with open("data/.index") as fh:
        fcntl.flock(fh, fcntl.LOCK_SH)
        index = json.load(fh)
    for k, v in request.args.lists():
        log.debug("search: %s -> %s", k, v)
        if timeseries is None:
            timeseries = set()
            log.debug("search: %s: %s", k, index[k])
            for m in v:
                timeseries |= set(index[k][m])
        else:
            filter = set()
            for m in v:
                filter |= set(index[k][m])
            timeseries &= filter
    results = list(timeseries)
    return results

def verify_node(d):
    node = d["auth"]["node"]
    timestamp = d["auth"]["timestamp"]
    digest1 = d["auth"]["hmac"]
    if "/" in node:
        raise ValueError("invalid node name %s", node)
    try:
        fn = "config/" + node
        log.info("getting client config from %s", fn)
        with open(fn) as fh:
            node_conf = json.load(fh)
    except Exception as e:
        log.warning("got %s opening %s", e, "config/" + node)
        abort(401, "unknown client")
    last = node_conf["last"]
    for key in node_conf["keys"]:
        msg = (node + " " + str(timestamp)).encode("UTF-8")
        hmac2 = hmac.new(key.encode("UTF-8"), msg, "SHA256")
        digest2 = hmac2.hexdigest()
        if hmac.compare_digest(digest1, digest2):
            if timestamp > node_conf["last"]:
                node_conf["last"] = timestamp
                os.replace("config/" + node, "config/" + node + ".old")
                tmpfn = fn + "." + str(os.getpid())
                oldfn = fn + ".old"
                with open(tmpfn, "w") as fh:
                    json.dump(node_conf, fh) # XXX
                try:
                    os.unlink(oldfn)
                except FileNotFoundError:
                    pass
                try:
                    os.link(fn, oldfn)
                except FileNotFoundError:
                    pass
                os.rename(tmpfn, fn)
                return node
            else:
                abort(409, "timestamp out of sync")
    abort(401, "auth failed")

@app.get("/v")
def visualize():
    timeseries_ids = request.args.getlist("ts")
    if not timeseries_ids:
        timeseries_ids = _search()
    log.debug("timeseries_ids = %s", timeseries_ids)
    timeseries_data = []
    for id in timeseries_ids:
        ts = LTS(id=id)
        timeseries_data.append(ts)
    return render_template("visualize.html", ts=timeseries_data)

@app.get("/dashboard/")
def dashboard_index():
    d = Dashboard("dashboards/" + "index" + ".json")
    return d.as_html()

@app.get("/dashboard/<dashboard>")
def dashboard_file(dashboard):
    d = Dashboard("dashboards/" + dashboard + ".json")
    return d.as_html()

@app.get("/nav")
def nav():
    # Start with a list of all dimensions, the number of matching time series
    # and a truncated list of series.
    # If a dimension is chosen, display a choice of members
    # choosing one or more members goes back to the list of
    # (remaining) dimensions
    with open("data/.index") as fh:
        fcntl.flock(fh, fcntl.LOCK_SH)
        index = json.load(fh)
    timeseries = None
    for k, v in request.args.lists():
        if k[0] == ".":
            continue
        log.debug("search: %s -> %s", k, v)
        if timeseries is None:
            timeseries = set()
            log.debug("search: %s: %s", k, index[k])
            for m in v:
                timeseries |= set(index[k][m])
        else:
            filter = set()
            for m in v:
                filter |= set(index[k][m])
            timeseries &= filter
    if timeseries is None:
        timeseries = set()
        for mc in index.values():
            for tsl in mc.values():
                timeseries |= set(tsl)
    if d := request.args.get(".m"):
        members = []
        for m, tsl in index[d].items():
            if set(tsl) & timeseries:
                members.append(m)
        return render_template("nav_member_select.html", dimension=d, members=members)
    else:
        params = request.args.to_dict(flat=False)
        matching_dimensions = defaultdict(int)
        for d, mc in index.items():
            if d in params:
                continue
            for m, tsl in mc.items():
                mtsl = set(tsl) & timeseries
                if mtsl:
                    matching_dimensions[d] += len(mtsl)
        matching_dimensions_list = []
        for d in matching_dimensions:
            params[".m"] = d
            url = url_for("nav", **params)
            app.logger.debug(f"{d=} {url=}")
            matching_dimensions_list.append(
                {"name": d, "count": matching_dimensions[d], "url": url}
            )
        total_timeseries = len(timeseries)
        timeseries = [LTS(id=ts) for ts in list(timeseries)[:100]]
        return render_template(
                    "nav_dimension_list.html",
                    matching_dimensions=matching_dimensions_list,
                    timeseries=timeseries, total_timeseries=total_timeseries)






    #