hjp
/
df2db
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: hjp:6c67de3bc96de2875ed85cc2b06aa5381905ee64
Branches Tags
hjp:master
...
compare: hjp:422e86f0a7c87bc670a1f668d948b482057d9593
Branches Tags
hjp:master

2 Commits
6c67de3bc9 ... 422e86f0a7

Author SHA1 Message Date
Peter J. Holzer 422e86f0a7 Use lateral joins to improve performance 
Milliseconds instead of seconds \o/.
 2021-09-05 09:27:13 +02:00
Peter J. Holzer 2f4096c5af Silently ignore permission errors 
Those usually occur on special filesystems which we don't care about
anyway.
 2021-09-05 00:56:21 +02:00
3 changed files with 120 additions and 80 deletions
Show Stats Expand all files Collapse all files

74
check_df_files
View File

@ -12,34 +12,52 @@ db = psycopg2.connect('')
csr = db.cursor(cursor_factory=psycopg2.extras.DictCursor) csr = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
csr.execute( csr.execute(
""" """
with recursive t as ( with recursive
select '2 minutes'::interval as d t as (
union select '2 minutes'::interval as d
select d * 2 from t where d < '1 year'::interval union
), select d * 2 from t where d < '1 year'::interval
a as ( ),
select distinct hostname, filesystem, min(ts), max(ts) fs as (
from df, t select distinct hostname, filesystem
where ts >= now() - t.d from df
group by hostname, filesystem, t.d where ts >= now() - '2 minutes'::interval
), group by hostname, filesystem
forecast as ( ),
select forecast as (
a.hostname, a.filesystem, select
c.ts - b.ts as d, fs.hostname, fs.filesystem,
c.ts + (c.ts - b.ts) as when, new.ts + (new.ts - old.ts) as when,
c.f_used + (c.f_used - b.f_used) as used, new.ts + (new.ts - old.ts) - now() as d,
(c.f_used + (c.f_used - b.f_used)) / b.f_usable * 100 as percent, new.f_used + (new.f_used - old.f_used) as used,
(c.f_used + (c.f_used - b.f_used)) > b.f_usable as capacity_exceeded (new.f_used + (new.f_used - old.f_used)) / old.f_usable * 100 as percent,
from a (new.f_used + (new.f_used - old.f_used)) > old.f_usable as capacity_exceeded
join df b on (a.hostname = b.hostname and a.filesystem = b.filesystem and a.min = b.ts) from fs, t,
join df c on (a.hostname = c.hostname and a.filesystem = c.filesystem and a.max = c.ts) lateral (
) select *
select * from forecast from df
where capacity_exceeded where
order by 1, 2, 3 df.hostname = fs.hostname
""" and df.filesystem = fs.filesystem
and df.ts >= now() - t.d
order by ts
limit 1
) old,
lateral (
select *
from df
where
df.hostname = fs.hostname
and df.filesystem = fs.filesystem
order by ts desc
limit 1
) new
)
select * from forecast
where capacity_exceeded
order by "when"
"""
) )
status = 0 status = 0

74
check_df_space
View File

@ -12,34 +12,52 @@ db = psycopg2.connect('')
csr = db.cursor(cursor_factory=psycopg2.extras.DictCursor) csr = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
csr.execute( csr.execute(
""" """
with recursive t as ( with recursive
select '2 minutes'::interval as d t as (
union select '2 minutes'::interval as d
select d * 2 from t where d < '1 year'::interval union
), select d * 2 from t where d < '1 year'::interval
a as ( ),
select distinct hostname, filesystem, min(ts), max(ts) fs as (
from df, t select distinct hostname, filesystem
where ts >= now() - t.d from df
group by hostname, filesystem, t.d where ts >= now() - '2 minutes'::interval
), group by hostname, filesystem
forecast as ( ),
select forecast as (
a.hostname, a.filesystem, select
c.ts - b.ts as d, fs.hostname, fs.filesystem,
c.ts + (c.ts - b.ts) as when, new.ts + (new.ts - old.ts) as when,
c.s_used + (c.s_used - b.s_used) as used, new.ts + (new.ts - old.ts) - now() as d,
(c.s_used + (c.s_used - b.s_used)) / b.s_usable * 100 as percent, new.s_used + (new.s_used - old.s_used) as used,
(c.s_used + (c.s_used - b.s_used)) > b.s_usable as capacity_exceeded (new.s_used + (new.s_used - old.s_used)) / old.s_usable * 100 as percent,
from a (new.s_used + (new.s_used - old.s_used)) > old.s_usable as capacity_exceeded
join df b on (a.hostname = b.hostname and a.filesystem = b.filesystem and a.min = b.ts) from fs, t,
join df c on (a.hostname = c.hostname and a.filesystem = c.filesystem and a.max = c.ts) lateral (
) select *
select * from forecast from df
where capacity_exceeded where
order by 1, 2, 3 df.hostname = fs.hostname
""" and df.filesystem = fs.filesystem
and df.ts >= now() - t.d
order by ts
limit 1
) old,
lateral (
select *
from df
where
df.hostname = fs.hostname
and df.filesystem = fs.filesystem
order by ts desc
limit 1
) new
)
select * from forecast
where capacity_exceeded
order by "when"
"""
) )
status = 0 status = 0

52
df2db
View File

@ -12,34 +12,38 @@ csr = db.cursor()
with open("/proc/mounts") as mounts: with open("/proc/mounts") as mounts:
for mount in mounts: for mount in mounts:
(dev, mountpoint, fstype, rest) = mount.split(' ', 3) (dev, mountpoint, fstype, rest) = mount.split(' ', 3)
s = os.statvfs(mountpoint) try:
if s.f_blocks == 0: s = os.statvfs(mountpoint)
continue # not a real file system if s.f_blocks == 0:
continue # not a real file system
s_total = s.f_blocks * s.f_bsize s_total = s.f_blocks * s.f_bsize
s_usable = (s.f_blocks - s.f_bfree + s.f_bavail) * s.f_bsize s_usable = (s.f_blocks - s.f_bfree + s.f_bavail) * s.f_bsize
s_used = (s.f_blocks - s.f_bfree) * s.f_bsize s_used = (s.f_blocks - s.f_bfree) * s.f_bsize
f_total = s.f_files f_total = s.f_files
f_usable = s.f_files - s.f_ffree + s.f_favail f_usable = s.f_files - s.f_ffree + s.f_favail
f_used = s.f_files - s.f_ffree f_used = s.f_files - s.f_ffree
csr.execute( csr.execute(
""" """
insert into df( insert into df(
hostname, filesystem, hostname, filesystem,
s_total, s_usable, s_used,
f_total, f_usable, f_used
)
values (
%s, %s,
%s, %s, %s,
%s, %s, %s
)
""",
(hostname, mountpoint,
s_total, s_usable, s_used, s_total, s_usable, s_used,
f_total, f_usable, f_used f_total, f_usable, f_used))
) except PermissionError:
values ( # silently ignore some errors
%s, %s, pass
%s, %s, %s,
%s, %s, %s
)
""",
(hostname, mountpoint,
s_total, s_usable, s_used,
f_total, f_usable, f_used))
db.commit() db.commit()