hjp
/
rss2html
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Port to postgresql, perl 5.24

This commit is contained in:
Peter J. Holzer 2017-10-02 15:26:48 +02:00
parent 7d8e075143
commit c9dc0a56f6
4 changed files with 39 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
index.cgi
View File

@ -29,10 +29,10 @@
use strict; use strict;
use warnings; use warnings;
use v5.14; use v5.24;
use utf8; use utf8;
use experimental 'smartmatch'; # for given/when use experimental 'smartmatch'; # for given/when
use experimental 'autoderef'; use experimental 'postderef';
use CGI; use CGI;
use Cache::Memcached; use Cache::Memcached;
@ -43,6 +43,7 @@ use POSIX qw(strftime);
use Rss2Html::Scrubber; use Rss2Html::Scrubber;
use Time::HiRes qw(time); use Time::HiRes qw(time);
my $db_conn = "dbi:Pg:dbname=rss2html";
$| = 1; $| = 1;
my $start = time(); my $start = time();
@ -60,18 +61,14 @@ if ($q->param('mark')) {
} }
sub redirect { sub redirect {
my $dbh = DBI->connect("dbi:SQLite:dbname=rss2html.sqlite", "", "", my $dbh = DBI->connect($db_conn);
{
sqlite_use_immediate_transaction => 1,
});
$dbh->{sqlite_unicode} = 1;
my $item_id = $q->param('redir'); my $item_id = $q->param('redir');
my $item = $dbh->selectrow_hashref("select * from items where id=?", {}, $item_id); my $item = $dbh->selectrow_hashref("select * from items where id=?", {}, $item_id);
if ($item) { if ($item) {
print "Status: 302\n"; print "Status: 302\n";
print "Location: $item->{link}\n"; print "Location: $item->{link}\n";
print "\n"; print "\n";
$dbh->do("insert into read(user, item_id) values(?, ?)", {}, $q->remote_user, $item_id); $dbh->do("insert into read(username, item_id) values(?, ?)", {}, $q->remote_user, $item_id);
$mcd->delete(item_info_key($q->remote_user, $item->{feed_id})); $mcd->delete(item_info_key($q->remote_user, $item->{feed_id}));
exit(0); exit(0);
} }
@ -84,7 +81,7 @@ sub redirect {
sub mark { sub mark {
print_log("mark start"); print_log("mark start");
my $dbh = DBI->connect("dbi:SQLite:dbname=rss2html.sqlite", "", ""); my $dbh = DBI->connect($db_conn, "", "");
$dbh->{sqlite_unicode} = 1; $dbh->{sqlite_unicode} = 1;
my $item_id = $q->param('mark'); my $item_id = $q->param('mark');
my $item = $dbh->selectrow_hashref("select * from items where id=?", {}, $item_id); my $item = $dbh->selectrow_hashref("select * from items where id=?", {}, $item_id);
@ -94,7 +91,7 @@ sub mark {
print "Status: 302\n"; print "Status: 302\n";
print "Location: ", $q1->self_url, "\n"; print "Location: ", $q1->self_url, "\n";
print "\n"; print "\n";
$dbh->do("insert into read(user, item_id) values(?, ?)", {}, $q->remote_user, $item_id); $dbh->do("insert into read(username, item_id) values(?, ?)", {}, $q->remote_user, $item_id);
$mcd->delete(item_info_key($q->remote_user, $item->{feed_id})); $mcd->delete(item_info_key($q->remote_user, $item->{feed_id}));
print_log("mark done"); print_log("mark done");
exit(0); exit(0);
@ -126,7 +123,7 @@ sub list {
print "<h1>RSS 2 HTML</h1>\n"; print "<h1>RSS 2 HTML</h1>\n";
print "<div class='lastupdate'>", strftime("%Y-%m-%d %H:%M:%S%z", localtime()), "</div>\n"; print "<div class='lastupdate'>", strftime("%Y-%m-%d %H:%M:%S%z", localtime()), "</div>\n";
my $dbh = DBI->connect("dbi:SQLite:dbname=rss2html.sqlite", "", ""); my $dbh = DBI->connect($db_conn, "", "");
$dbh->{sqlite_unicode} = 1; $dbh->{sqlite_unicode} = 1;
print_log("list_feeds start"); print_log("list_feeds start");
@ -135,16 +132,17 @@ sub list {
my $items; my $items;
if ($show_read) { if ($show_read) {
my $fields = "feeds.id as feed_id, feeds.title as feed_title, allow_img, link, items.title as item_title, content, items.id as item_id, issued, user, lang"; my $fields = "feeds.id as feed_id, feeds.title as feed_title, allow_img, link, items.title as item_title, content, "
. "items.id as item_id, issued, username, lang";
my $tables = "items my $tables = "items
join feeds on items.feed_id=feeds.id join feeds on items.feed_id=feeds.id
left outer join read on (items.id=read.item_id and user=?)"; left outer join read on (items.id=read.item_id and username=?)";
my @where; my @where;
if (%feed_item_show) { if (%feed_item_show) {
push @where, "feeds.id in (" . join(',', keys %feed_item_show) . ")"; push @where, "feeds.id in (" . join(',', keys %feed_item_show) . ")";
} }
if (!$show_old) { if (!$show_old) {
push @where, "(items.old is null or items.old == 0)" push @where, "(items.old is null or items.old = 0)"
} }
my $where = "where " . join(" and ", @where); my $where = "where " . join(" and ", @where);
@ -157,9 +155,9 @@ sub list {
my $item = pop @$items1; my $item = pop @$items1;
{ {
no warnings 'uninitialized'; no warnings 'uninitialized';
print_log("id=$item->{item_id}, user=$item->{user}"); print_log("id=$item->{item_id}, username=$item->{username}");
} }
if (!$item->{user}) { if (!$item->{username}) {
print_log("id=$item->{item_id}: unread"); print_log("id=$item->{item_id}: unread");
unshift @$items, $item; unshift @$items, $item;
} elsif ($nrd < $show_read) { } elsif ($nrd < $show_read) {
@ -170,10 +168,10 @@ sub list {
} }
} else { } else {
my @where = ("user is null"); my @where = ("username is null");
if (!$show_old) { if (!$show_old) {
push @where, "(items.old is null or items.old == 0)" push @where, "(items.old is null or items.old = 0)"
} }
my $where = "where " . join(" and ", @where); my $where = "where " . join(" and ", @where);
@ -183,13 +181,13 @@ sub list {
"select feeds.id as feed_id, feeds.title as feed_title, allow_img, link, items.title as item_title, content, items.id as item_id, issued, lang "select feeds.id as feed_id, feeds.title as feed_title, allow_img, link, items.title as item_title, content, items.id as item_id, issued, lang
from items from items
join feeds on items.feed_id=feeds.id join feeds on items.feed_id=feeds.id
left outer join read on (items.id=read.item_id and user=?) left outer join read on (items.id=read.item_id and username=?)
$where $where
order by issued", order by issued",
{ Slice => {} }, $q->remote_user); { Slice => {} }, $q->remote_user);
} }
my $feeds = $dbh->selectall_arrayref( my $feeds = $dbh->selectall_arrayref(
"select id, title from feeds where update_frequency is not null order by id", "select id, title from feeds where active order by id",
{ Slice => {} } { Slice => {} }
); );
for my $f (@$feeds) { for my $f (@$feeds) {
@ -226,7 +224,7 @@ sub print_itemlist {
print_log(scalar @$items . " after remix"); print_log(scalar @$items . " after remix");
for my $item (@$items) { for my $item (@$items) {
$n_items++; $n_items++;
my $is_read = defined($item->{user}); my $is_read = defined($item->{username});
my $scrubbed_content = $mcd->get(scrubbed_content_key($item->{item_id})); my $scrubbed_content = $mcd->get(scrubbed_content_key($item->{item_id}));
if ($scrubbed_content) { if ($scrubbed_content) {
$scrubbed_content = decode_utf8($scrubbed_content); $scrubbed_content = decode_utf8($scrubbed_content);
@ -273,17 +271,17 @@ sub remix {
print_log("item w/o feed_id: " . Dumper($item)); print_log("item w/o feed_id: " . Dumper($item));
} }
$queues->{$item->{feed_id}} //= []; $queues->{$item->{feed_id}} //= [];
push $queues->{$item->{feed_id}}, $item; push $queues->{$item->{feed_id}}->@*, $item;
} }
my @ordered_feeds = sort { $queues->{$a}[0]{issued} <=> $queues->{$b}[0]{issued} } keys $queues; my @ordered_feeds = sort { $queues->{$a}[0]{issued} <=> $queues->{$b}[0]{issued} } keys $queues->%*;
my $new_items = []; my $new_items = [];
my $found; my $found;
do { do {
$found = 0; $found = 0;
for my $feed (@ordered_feeds) { for my $feed (@ordered_feeds) {
my $item = shift $queues->{$feed}; my $item = shift $queues->{$feed}->@*;
if ($item) { if ($item) {
push $new_items, $item; push $new_items->@*, $item;
$found++; $found++;
print_log("found an item in feed $feed. " . scalar(@{$queues->{$feed}}) . " items left") print_log("found an item in feed $feed. " . scalar(@{$queues->{$feed}}) . " items left")
} }
@ -305,7 +303,7 @@ sub list_feeds {
print "<div class='feedlist'>\n"; print "<div class='feedlist'>\n";
$dbh->begin_work; $dbh->begin_work;
my $feeds = $dbh->selectall_arrayref("select * from feeds where update_frequency is not null", { Slice => {} }); my $feeds = $dbh->selectall_arrayref("select * from feeds where active", { Slice => {} });
my $seconds_per_week = 86400 * 7; my $seconds_per_week = 86400 * 7;
for my $feed (@$feeds) { for my $feed (@$feeds) {
$feed->{title} //= "-"; $feed->{title} //= "-";
@ -325,7 +323,7 @@ sub print_item_info {
my $item_info = $mcd->get($item_info_key); my $item_info = $mcd->get($item_info_key);
my $cmd my $cmd
= "select count(i.id) as nr_items, max(i.issued) as last_issued, max(i.seen) as last_seen, count(r.item_id) as nr_read = "select count(i.id) as nr_items, max(i.issued) as last_issued, max(i.seen) as last_seen, count(r.item_id) as nr_read
from items i left outer join (select item_id from read where user=?) r on i.id=r.item_id from items i left outer join (select item_id from read where username=?) r on i.id=r.item_id
where i.feed_id=?"; where i.feed_id=?";
unless ($show_old) { unless ($show_old) {
$cmd .= " and (old is null or old = 0)"; $cmd .= " and (old is null or old = 0)";
@ -408,6 +406,7 @@ sub delta_t {
sub item_info_key { sub item_info_key {
my ($user, $feed_id) = @_; my ($user, $feed_id) = @_;
$user //= "anonymous";
return "rss2html/item_info/$user/$feed_id"; return "rss2html/item_info/$user/$feed_id";
} }

8
lib/Rss2Html/Feed.pm
View File

@ -32,7 +32,7 @@ has 'id' => (
has 'url' => ( has 'url' => (
is => 'rw', is => 'rw',
); );
has 'update_frequency' => ( has 'active' => (
is => 'rw', is => 'rw',
); );
has 'last_update' => ( has 'last_update' => (
@ -124,7 +124,7 @@ sub update {
} }
} }
} else { } else {
print STDERR "error getting $self->{url}: $@\n"; $self->log->error("error getting $self->{url}: $@");
} }
} elsif ($self->{type} eq 'atom') { } elsif ($self->{type} eq 'atom') {
my $api = XML::Atom::Client->new(); my $api = XML::Atom::Client->new();
@ -175,7 +175,7 @@ sub update {
$dbh->do("update items set old=1 $dbh->do("update items set old=1
where id in (select id from items where id in (select id from items
where feed_id=? and (old is null or old != 1) where feed_id=? and (old is null or old != 1)
order by issued desc limit -1 offset 100)", order by issued desc offset 100)",
{}, {},
$self->{id}); $self->{id});
@ -205,7 +205,7 @@ sub invalidate_item_info {
my ($self) = @_; my ($self) = @_;
my $dbh = $self->dbh; my $dbh = $self->dbh;
my $users = $dbh->selectcol_arrayref("select distinct user from read"); my $users = $dbh->selectcol_arrayref("select distinct username from read");
for my $user (@$users) { for my $user (@$users) {
$mcd->delete($self->item_info_key($user)); $mcd->delete($self->item_info_key($user));
} }

9
lib/Rss2Html/FeedList.pm
View File

@ -17,11 +17,7 @@ has 'dbh' => (
sub BUILD { sub BUILD {
my ($self, $args) = @_; my ($self, $args) = @_;
my $dbh = DBI->connect("dbi:SQLite:dbname=rss2html.sqlite", "", "", my $dbh = DBI->connect("dbi:Pg:dbname=rss2html", "", "");
{
sqlite_use_immediate_transaction => 1,
});
$dbh->{sqlite_unicode} = 1;
$self->_set_dbh($dbh); $self->_set_dbh($dbh);
} }
@ -31,7 +27,7 @@ sub feeds {
no autovivification qw(fetch); no autovivification qw(fetch);
my $feeds_sql = $dbh->selectall_arrayref("select * from feeds where update_frequency is not null", { Slice => {} }); my $feeds_sql = $dbh->selectall_arrayref("select * from feeds where active", { Slice => {} });
my @feeds; my @feeds;
for (@$feeds_sql) { for (@$feeds_sql) {
push @feeds, Rss2Html::Feed->new(%$_, dbh => $dbh); push @feeds, Rss2Html::Feed->new(%$_, dbh => $dbh);
@ -129,3 +125,4 @@ sub cleanup {
#--- #---
__PACKAGE__->meta->make_immutable; __PACKAGE__->meta->make_immutable;
1; 1;
# vim: tw=132

12
rss2html.postgresql
View File

@ -1,7 +1,7 @@
CREATE TABLE feeds (id integer primary key autoincrement, url varchar, update_frequency int, last_update int, title varchar, type varchar, allow_img boolean, expire int, lang varchar); CREATE TABLE feeds (id serial primary key, url varchar, active boolean, last_update int, title varchar, type varchar, allow_img boolean, expire int, lang varchar);
CREATE TABLE items (id integer primary key autoincrement, title varchar, link varchar, content varchar, issued int, seen int, feed_id integer, old int); CREATE TABLE items (id serial primary key, title varchar, link varchar, content varchar, issued int, seen int, feed_id integer, old int);
CREATE TABLE read(item_id integer, user varchar, foreign key (item_id) references items(id)); CREATE TABLE read(item_id integer, username varchar, foreign key (item_id) references items(id));
CREATE INDEX items_link_idx on items(link); CREATE INDEX on items(link);
CREATE INDEX read_item_id_idx on read(item_id); CREATE INDEX read_item_id_idx on read(item_id);
CREATE TABLE later (item_id integer, user varchar, foreign key (item_id) references items(id)); CREATE TABLE later (item_id integer, username varchar, foreign key (item_id) references items(id));
CREATE INDEX later_item_id_idx on later(item_id); CREATE INDEX on later(item_id);