From f8856de2c96af0f5a471141d8284b5c9943ea385 Mon Sep 17 00:00:00 2001 From: hjp Date: Fri, 9 Apr 2010 20:28:20 +0000 Subject: [PATCH] (late commit) Improved search for last session (now only finds sessions which were completed successfully). Some speed optimizations (not very successfull, iirc). --- lib/Simba/CA.pm | 74 ++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/lib/Simba/CA.pm b/lib/Simba/CA.pm index 7748628..ae671a1 100644 --- a/lib/Simba/CA.pm +++ b/lib/Simba/CA.pm @@ -152,11 +152,7 @@ sub backup2disk { $self->{target} = $target; # get previous generation - my @dirs = glob($self->{basedir} . '/????-??-??T??.??.??/' . $target->{host} . '/' . $target->{dir}); - - $self->{last_backup} = $dirs[-1]; - $self->{last_backup} = $1 if $self->{last_backup} =~ /(.*)/; # detaint - $self->{last_backup_id} = $self->get_last_session_id(); + $self->get_last_session(); my $timestamp = $self->{timestamp} || strftime('%Y-%m-%dT%H.%M.%S', localtime); $self->{this_backup} = $self->{basedir} . "/$timestamp/" . $target->{host} . '/' . $target->{dir}; @@ -178,6 +174,9 @@ sub backup2disk { # split into fields chomp; my $f = $self->parse($_); + if ($count % 1000 == 0) { + $self->log(9, "file $count: $f->{name}"); + } my $success = 1; if ($f->{t} eq 'f') { @@ -201,6 +200,7 @@ sub backup2disk { # insert into DB. $self->db_record_version($target, $f) if ($success); } + $self->flush_insert_instances(); $self->close_session(); $self->log(3, "finished backup for target host " . $target->{host} . " dir " . $target->{dir} . ": $count files"); $self->{counts}{objects} += $count; @@ -466,20 +466,16 @@ sub db_record_version { } my $t3 = gettimeofday(); $self->{times}{db_record_version_versions2} += $t3 - $t2; - $self->{dbh}->do("insert into instances(file, - file_id, - date, online, - session, version) - values(?, - ?, - ?, ?, - ?, ?)", - {}, - $db_f->[0]{id}, - $f->{id}, - time(), 1, - $self->{session_id}, $version_id - ); + push @{ $self->{caches}{insert_instances} }, + [ + $db_f->[0]{id}, + $f->{id}, + time(), 1, + $self->{session_id}, $version_id + ]; + if (@{ $self->{caches}{insert_instances} } > 10) { + $self->flush_insert_instances(); + } my $t4 = gettimeofday(); $self->{times}{db_record_version_insert_instances} += $t4 - $t3; @@ -487,6 +483,26 @@ sub db_record_version { } +sub flush_insert_instances { + my ($self) = @_; + my $dbh = $self->{dbh}; + + if (@{ $self->{caches}{insert_instances} }) { + my $cmd = "insert into instances(file, file_id, date, online, session, version)" + . " values " + . join(", ", + map { + "(" + . join(",", map { $dbh->quote($_) } @$_) + . ")" + } @{ $self->{caches}{insert_instances} } + ); + $dbh->do($cmd); + } + $self->{caches}{insert_instances} = []; +} + + sub new_session { my ($self) = @_; $self->{dbh}->do("insert into sessions(start_date, prefix) values(?, ?)", {}, time(), $self->{this_backup}); @@ -515,14 +531,16 @@ sub close_file_connection { } } -sub get_last_session_id { +sub get_last_session { my ($self) = @_; - return unless $self->{last_backup}; - my $sessions = $self->{dbh}->selectall_arrayref("select * from sessions where prefix=?", - { Slice => {} }, - $self->{last_backup}); - die "$self->{last_backup} not a unique prefix" unless @$sessions == 1; - return $sessions->[0]{id}; + my $sessions + = $self->{dbh}->selectall_arrayref( + "select * from sessions where end_date is not null and prefix like ? order by end_date desc", + { Slice => {} }, + $self->{basedir} . '/%/' . $self->{target}->{host} . '/' . $self->{target}->{dir} + ); + $self->{last_backup} = $sessions->[0]{prefix}; + $self->{last_backup_id} = $sessions->[0]{id}; } =head2 linkdup @@ -545,9 +563,9 @@ sub linkdup { my $sth = $self->{dbh}->prepare("select * from versions2, instances, files, sessions where file_type=? and file_size=? and file_mtime=? and file_owner=? and file_group=? and file_acl=? - and file_unix_bits=? + and file_unix_bits=? and checksum=? and online=1 - and instances.file=files.id + and instances.file=files.id and versions2.id=instances.version and instances.session=sessions.id order by instances.session desc