[mb-commits] branch, beta, updated. Merge branch 'mbs-5272-rollup-replication' into beta Merge branch 'master' in...
MusicBrainz Git Server <[email protected]>
| Newsgroups | gmane.comp.audio.musicbrainz.cvs |
|---|---|
| Message-ID | <E1TdSIZ-00015G-6P@wiley> |
The branch, beta has been updated
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=1d8bce086ae910a027ce8363efe59f9319efc53f (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=79d3d0f1eb3705eea7d0ce2ae9e92e09af9b4ee1 (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=da574fd83ddeba7f7088f7bc506a14de1ec49eee (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=edd89935c0773e3f3e8bc51180efa1725af25b9e (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=0a2110972a0ec9a0506332b4b6e51f1fe73abc0d (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=8d66f90396cc8de351fe9f41a58577ed29208dd1 (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=c0590b92500c3a3fc3ba1b960a0793e167020f70 (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=3972d4c3b7d2991cb5e97d48cab8ee1450cf3732 (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=3e75177783025c8326a832e8b76cf81667491216 (commit)
via http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=8949e6d6ffdbe7d98fcba52e7bea8ca52fb3135c (commit)
from http://git.musicbrainz.org/gitweb/?p=musicbrainz-server/core.git;a=commit;h=3139b9274ebd70622821dfc0bd6c6a4961ccc1af (commit)
Summary of changes:
admin/replication/BundleReplicationPackets | 243 ++++++++++++++++++++++++++++
admin/replication/LoadReplicationChanges | 90 ++++++++---
2 files changed, 312 insertions(+), 21 deletions(-)
create mode 100755 admin/replication/BundleReplicationPackets
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 1d8bce086ae910a027ce8363efe59f9319efc53f
Merge: 3139b92 79d3d0f
Author: Ian McEwen <[email protected]>
Date: Tue Nov 27 13:34:54 2012 -0700
Merge branch 'mbs-5272-rollup-replication' into beta
commit 79d3d0f1eb3705eea7d0ce2ae9e92e09af9b4ee1
Merge: da574fd 996b98e
Author: Ian McEwen <[email protected]>
Date: Tue Nov 27 13:33:32 2012 -0700
Merge branch 'master' into mbs-5272-rollup-replication
commit da574fd83ddeba7f7088f7bc506a14de1ec49eee
Author: Ian McEwen <[email protected]>
Date: Tue Oct 30 15:42:17 2012 -0700
MBS-5272: at ruaok's request, don't prefer ftp.mb
diff --git a/admin/replication/LoadReplicationChanges b/admin/replication/LoadReplicationChanges
index 66e86fa..6205697 100755
--- a/admin/replication/LoadReplicationChanges
+++ b/admin/replication/LoadReplicationChanges
@@ -46,8 +46,7 @@ my $limit = 0;
my $count = 0;
my $period = 'hourly';
my $catchup;
-my $baseuri = "http://ftp.musicbrainz.org/pub/musicbrainz/data/replication";
-my $fallbackuri = "http://data.musicbrainz.org/pub/musicbrainz/data/replication";
+my $baseuri = "http://data.musicbrainz.org/pub/musicbrainz/data/replication";
my $lockfile = my $deflockfile = "/tmp/.mb-LoadReplicationChanges";
my @process_opts;
@@ -55,7 +54,6 @@ GetOptions(
"help" => \$fHelp,
"lockfile=s" => \$lockfile,
"base-uri=s" => \$baseuri,
- "fallback-uri=s" => \$fallbackuri,
"process-arg=s" => \@process_opts,
"limit=i" => \$limit,
"period=s" => \$period,
@@ -73,8 +71,6 @@ Usage: LoadReplicationChanges [options]
--lockfile=FILE use FILE as the lock file to prevent us running in
parallel (default: $deflockfile)
--base-uri=URI load the replication data from this location
- (default: from ftp.musicbrainz.org via HTTP)
- --fallback-uri=URI fallback to this location if base-uri doesn't have a packet
(default: from data.musicbrainz.org via HTTP)
--process-arg=ARG Add ARG to each invocation of ProcessReplicationChanges.
If required, ARG can be an option (e.g.
@@ -197,13 +193,6 @@ elsif ($catchup) {
if ($resp->code == RC_NOT_FOUND)
{
- print localtime() . " : Replication packet #$iNextReplicationSequence not available from main location, trying fallback\n";
- $url = "$fallbackuri/$uriprefix$file";
- $resp = $ua->mirror($url, $localfile);
-}
-
-if ($resp->code == RC_NOT_FOUND)
-{
# TODO check for newer replication packets, in case the server has lost one?
# die if so
commit edd89935c0773e3f3e8bc51180efa1725af25b9e
Author: Ian McEwen <[email protected]>
Date: Fri Oct 5 04:54:18 2012 -0700
MBS-5272: use ftp.mb by default, falling back to data.mb. Add catchup-to-$period feature
diff --git a/admin/replication/LoadReplicationChanges b/admin/replication/LoadReplicationChanges
index 5ce78a0..66e86fa 100755
--- a/admin/replication/LoadReplicationChanges
+++ b/admin/replication/LoadReplicationChanges
@@ -45,7 +45,9 @@ my $fKeepFiles = 0;
my $limit = 0;
my $count = 0;
my $period = 'hourly';
-my $baseuri = "http://data.musicbrainz.org/pub/musicbrainz/data/replication";
+my $catchup;
+my $baseuri = "http://ftp.musicbrainz.org/pub/musicbrainz/data/replication";
+my $fallbackuri = "http://data.musicbrainz.org/pub/musicbrainz/data/replication";
my $lockfile = my $deflockfile = "/tmp/.mb-LoadReplicationChanges";
my @process_opts;
@@ -53,9 +55,11 @@ GetOptions(
"help" => \$fHelp,
"lockfile=s" => \$lockfile,
"base-uri=s" => \$baseuri,
+ "fallback-uri=s" => \$fallbackuri,
"process-arg=s" => \@process_opts,
"limit=i" => \$limit,
"period=s" => \$period,
+ "catchup" => \$catchup,
) or exit 2;
$baseuri =~ s/\/$//;
@@ -65,19 +69,24 @@ sub usage
print <<EOF;
Usage: LoadReplicationChanges [options]
- --help show this help
- --lockfile=FILE use FILE as the lock file to prevent us running in
- parallel (default: $deflockfile)
- --base-uri=URI load the replication data from this location
- (default: from musicbrainz.org via HTTP)
- --process-arg=ARG Add ARG to each invocation of ProcessReplicationChanges.
- If required, ARG can be an option (e.g.
- --process-arg=--debug-xact). Add --process-arg=ARG
- again to specify additional arguments.
- --limit=N apply only N packets
- --period=ARG ARG can be e.g. 'daily' or 'weekly'; default 'hourly'.
- This will only work if you're currently in sync with
- these replication options.
+ --help show this help
+ --lockfile=FILE use FILE as the lock file to prevent us running in
+ parallel (default: $deflockfile)
+ --base-uri=URI load the replication data from this location
+ (default: from ftp.musicbrainz.org via HTTP)
+ --fallback-uri=URI fallback to this location if base-uri doesn't have a packet
+ (default: from data.musicbrainz.org via HTTP)
+ --process-arg=ARG Add ARG to each invocation of ProcessReplicationChanges.
+ If required, ARG can be an option (e.g.
+ --process-arg=--debug-xact). Add --process-arg=ARG
+ again to specify additional arguments.
+ --limit=N apply only N packets
+ --period=ARG ARG can be e.g. 'daily' or 'weekly'; default 'hourly'.
+ This will only work if you're currently in sync with
+ these replication options.
+ --catchup For periods other than 'hourly', apply hourly packets
+ until in sync with available packets for the provided
+ period
EOF
}
@@ -152,10 +161,12 @@ NEXT_PACKET:
my $iNextReplicationSequence = $iReplicationSequence + 1;
my $file = "replication-$iNextReplicationSequence.tar.bz2";
+my $uriprefix = '';
if ($period ne 'hourly') {
- $file = "$period/replication-$period-$iNextReplicationSequence.tar.bz2";
+ $file = "replication-$period-$iNextReplicationSequence.tar.bz2";
+ $uriprefix = "$period/";
}
-my $url = "$baseuri/$file";
+my $url = "$baseuri/$uriprefix$file";
my $localfile = "$tmpdir/$file";
use LWP::UserAgent;
@@ -171,6 +182,26 @@ my $resp = $ua->mirror($url, $localfile);
# OK, not found, and the rest.
use HTTP::Status qw( RC_OK RC_NOT_FOUND RC_NOT_MODIFIED );
+if ($catchup && ($resp->code == RC_OK or $resp->code == RC_NOT_MODIFIED)) {
+ print localtime() . " : Caught up to packets for period '$period'. Please rerun now without --catchup.\n";
+ exit 0;
+}
+elsif ($catchup) {
+ print localtime() . " : Not yet caught up for period '$period'. Applying an hourly packet...\n";
+ $file = "replication-$iNextReplicationSequence.tar.bz2";
+ $uriprefix = '';
+ $url = "$baseuri/$uriprefix$file";
+ $localfile = "$tmpdir/$file";
+ $resp = $ua->mirror($url, $localfile);
+}
+
+if ($resp->code == RC_NOT_FOUND)
+{
+ print localtime() . " : Replication packet #$iNextReplicationSequence not available from main location, trying fallback\n";
+ $url = "$fallbackuri/$uriprefix$file";
+ $resp = $ua->mirror($url, $localfile);
+}
+
if ($resp->code == RC_NOT_FOUND)
{
# TODO check for newer replication packets, in case the server has lost one?
commit 0a2110972a0ec9a0506332b4b6e51f1fe73abc0d
Author: Ian McEwen <[email protected]>
Date: Fri Oct 5 04:13:03 2012 -0700
MBS-5272: make LoadReplicationChanges use the period subdirectories too
diff --git a/admin/replication/LoadReplicationChanges b/admin/replication/LoadReplicationChanges
index e7d9193..5ce78a0 100755
--- a/admin/replication/LoadReplicationChanges
+++ b/admin/replication/LoadReplicationChanges
@@ -153,7 +153,7 @@ NEXT_PACKET:
my $iNextReplicationSequence = $iReplicationSequence + 1;
my $file = "replication-$iNextReplicationSequence.tar.bz2";
if ($period ne 'hourly') {
- $file = "replication-$period-$iNextReplicationSequence.tar.bz2";
+ $file = "$period/replication-$period-$iNextReplicationSequence.tar.bz2";
}
my $url = "$baseuri/$file";
my $localfile = "$tmpdir/$file";
commit 8d66f90396cc8de351fe9f41a58577ed29208dd1
Author: Ian McEwen <[email protected]>
Date: Fri Oct 5 03:50:16 2012 -0700
MBS-5272: default $outputdir to $rootdir/$period, and read period packets from $outputdir
diff --git a/admin/replication/BundleReplicationPackets b/admin/replication/BundleReplicationPackets
index 1649045..54bbbd9 100755
--- a/admin/replication/BundleReplicationPackets
+++ b/admin/replication/BundleReplicationPackets
@@ -72,7 +72,8 @@ Usage: BundleReplicationPackets root-directory [options]
previous packet)
--limit how many packets to include (default:
include all packets until present
- --output output directory for finished packet (default same as root)
+ --output output directory for finished packet
+ (default: root-directory/period)
EOF
}
@@ -80,7 +81,7 @@ usage(), exit if $fHelp;
usage(), exit 2 if !@ARGV;
my $rootdir = shift @ARGV;
-$outputdir ||= $rootdir;
+$outputdir ||= "$rootdir/$period";
my $num = shift @ARGV;
my @sequences = <$rootdir/replication-*.tar.bz2>;
@@ -90,15 +91,15 @@ my @sequences = <$rootdir/replication-*.tar.bz2>;
# Don't look at the directory if we've been given a start sequence
if (!$start) {
- my @period_sequences = <$rootdir/replication-$period-*.tar.bz2>;
- @period_sequences = reverse sort map { s%$rootdir/replication-$period-(\d+).tar.bz2%$1%; $_; } @period_sequences;
+ my @period_sequences = <$outputdir/replication-$period-*.tar.bz2>;
+ @period_sequences = reverse sort map { s%$outputdir/replication-$period-(\d+).tar.bz2%$1%; $_; } @period_sequences;
if (scalar @period_sequences) {
# Open up the most recent one for this period to determine where to start
my $period_mostrecent = $period_sequences[0];
print localtime() . " : Inspecting most recent '$period' packet ($period_mostrecent)\n";
- my $localfile = "$rootdir/replication-$period-$period_mostrecent.tar.bz2";
+ my $localfile = "$outputdir/replication-$period-$period_mostrecent.tar.bz2";
my $periodcheckdir = tempdir('periodcheck-XXXXXX', DIR => $tmpdir);
extract_tar($localfile, $periodcheckdir);
commit c0590b92500c3a3fc3ba1b960a0793e167020f70
Author: Ian McEwen <[email protected]>
Date: Sat Sep 29 02:39:24 2012 -0700
MBS-5272: ensure there are no jumps in replication sequence when creating bundles
diff --git a/admin/replication/BundleReplicationPackets b/admin/replication/BundleReplicationPackets
index 38466ed..1649045 100755
--- a/admin/replication/BundleReplicationPackets
+++ b/admin/replication/BundleReplicationPackets
@@ -130,13 +130,16 @@ if ($limit) {
@seq_to_use = @seq_to_use[0..$limit - 1];
}
-print join " ", @seq_to_use;
+die "No packets available for this period" unless scalar @seq_to_use;
my $newpacketdir = tempdir("mb-bundle-XXXXXX", DIR => $tmpdir);
mkdir "$newpacketdir/mbdump" or die $!;
my $schema_sequence;
+my $last_replication_sequence = $start - 1;
for my $sequence (@seq_to_use) {
+ die ("Replication sequence $sequence is too far from $last_replication_sequence")
+ if $last_replication_sequence + 1 != $sequence;
my $individual_packet_dir = tempdir("replication-bundle-XXXXXX", DIR => $tmpdir);
# untar the packet
@@ -178,6 +181,8 @@ for my $sequence (@seq_to_use) {
print localtime() . " : Removing $individual_packet_dir\n";
rmtree($individual_packet_dir);
+
+ $last_replication_sequence = $sequence;
}
# make tarfile
commit 3972d4c3b7d2991cb5e97d48cab8ee1450cf3732
Author: Ian McEwen <[email protected]>
Date: Sat Sep 29 01:44:40 2012 -0700
MBS-5272: support periods other than hourly in LoadReplicationChanges
diff --git a/admin/replication/LoadReplicationChanges b/admin/replication/LoadReplicationChanges
index e961ea7..e7d9193 100755
--- a/admin/replication/LoadReplicationChanges
+++ b/admin/replication/LoadReplicationChanges
@@ -44,16 +44,18 @@ my $tmpdir = "/tmp";
my $fKeepFiles = 0;
my $limit = 0;
my $count = 0;
+my $period = 'hourly';
my $baseuri = "http://data.musicbrainz.org/pub/musicbrainz/data/replication";
my $lockfile = my $deflockfile = "/tmp/.mb-LoadReplicationChanges";
my @process_opts;
GetOptions(
- "help" => \$fHelp,
+ "help" => \$fHelp,
"lockfile=s" => \$lockfile,
"base-uri=s" => \$baseuri,
"process-arg=s" => \@process_opts,
"limit=i" => \$limit,
+ "period=s" => \$period,
) or exit 2;
$baseuri =~ s/\/$//;
@@ -72,6 +74,10 @@ Usage: LoadReplicationChanges [options]
If required, ARG can be an option (e.g.
--process-arg=--debug-xact). Add --process-arg=ARG
again to specify additional arguments.
+ --limit=N apply only N packets
+ --period=ARG ARG can be e.g. 'daily' or 'weekly'; default 'hourly'.
+ This will only work if you're currently in sync with
+ these replication options.
EOF
}
@@ -146,6 +152,9 @@ NEXT_PACKET:
my $iNextReplicationSequence = $iReplicationSequence + 1;
my $file = "replication-$iNextReplicationSequence.tar.bz2";
+if ($period ne 'hourly') {
+ $file = "replication-$period-$iNextReplicationSequence.tar.bz2";
+}
my $url = "$baseuri/$file";
my $localfile = "$tmpdir/$file";
@@ -229,6 +238,15 @@ unless (defined($REPLICATION_SEQUENCE) and $REPLICATION_SEQUENCE =~ /\A(\d+)\n\z
}
$REPLICATION_SEQUENCE = $1;
+my $LAST_REPLICATION_SEQUENCE = read_file("LAST_REPLICATION_SEQUENCE");
+unless (defined($LAST_REPLICATION_SEQUENCE) and $LAST_REPLICATION_SEQUENCE =~ /\A(\d+)\n\z/)
+{
+ print localtime() . " : LAST_REPLICATION_SEQUENCE file missing or malformed\n";
+ $LAST_REPLICATION_SEQUENCE = $REPLICATION_SEQUENCE;
+} else {
+ $LAST_REPLICATION_SEQUENCE = $1;
+}
+
unless ($REPLICATION_SEQUENCE == $iNextReplicationSequence)
{
print localtime() . " : Oops! Downloaded packet $file"
@@ -246,7 +264,17 @@ unless (defined($TIMESTAMP) and $TIMESTAMP =~ /\A(.*)\n\z/)
print localtime() . " : TIMESTAMP file missing or malformed, ignoring\n";
} else {
chomp($TIMESTAMP = $1);
- print localtime() . " : This packet was produced at $TIMESTAMP\n";
+ print localtime() . " : This packet was produced (or begins) at $TIMESTAMP\n";
+}
+
+my $LAST_TIMESTAMP = read_file("LAST_TIMESTAMP");
+unless (defined($LAST_TIMESTAMP) and $LAST_TIMESTAMP =~ /\A(.*)\n\z/)
+{
+ print localtime() . " : LAST_TIMESTAMP file missing or malformed, setting to TIMESTAMP\n";
+ $LAST_TIMESTAMP = $TIMESTAMP;
+} else {
+ chomp($LAST_TIMESTAMP = $1);
+ print localtime() . " : This packet covers changes until $LAST_TIMESTAMP\n";
}
# Check existence of the dbmirror_pending and dbmirror_pendingdata data files
@@ -281,18 +309,18 @@ exit $? if $?;
# Check that current_replication_sequence has gone up by one
# Also last_replication_date should match TIMESTAMP
-$iNextReplicationSequence = $iReplicationSequence + 1
- if not defined $iNextReplicationSequence;
+$LAST_REPLICATION_SEQUENCE = $iReplicationSequence + 1
+ if not defined $LAST_REPLICATION_SEQUENCE;
{
my $row = $sql->select_single_row_hash("SELECT * FROM replication_control");
- $row->{current_replication_sequence} == $iNextReplicationSequence
- or die "Applied changes, but current_replication_sequence is $row->{current_replication_sequence} not $iNextReplicationSequence\n";
- not defined($TIMESTAMP)
- or $row->{last_replication_date} eq $TIMESTAMP
- or warn "Applied changes, but last_replication_date is '$row->{last_replication_date}' not '$TIMESTAMP'\n";
+ $row->{current_replication_sequence} == $LAST_REPLICATION_SEQUENCE
+ or die "Applied changes, but current_replication_sequence is $row->{current_replication_sequence} not $LAST_REPLICATION_SEQUENCE\n";
+ not defined($LAST_TIMESTAMP)
+ or $row->{last_replication_date} eq $LAST_TIMESTAMP
+ or warn "Applied changes, but last_replication_date is '$row->{last_replication_date}' not '$LAST_TIMESTAMP'\n";
$iReplicationSequence = $row->{current_replication_sequence};
}
-$dtReplicationDate = $TIMESTAMP;
+$dtReplicationDate = $LAST_TIMESTAMP;
$count += 1;
if ($limit > 0 && $count >= $limit) {
commit 3e75177783025c8326a832e8b76cf81667491216
Author: Ian McEwen <[email protected]>
Date: Sat Sep 29 01:44:17 2012 -0700
MBS-5272: add a BundleReplicationPackets script to bundle several packets together
diff --git a/admin/replication/BundleReplicationPackets b/admin/replication/BundleReplicationPackets
new file mode 100755
index 0000000..38466ed
--- /dev/null
+++ b/admin/replication/BundleReplicationPackets
@@ -0,0 +1,237 @@
+#!/usr/bin/env perl
+
+use warnings;
+# vi: set ts=4 sw=4 :
+#____________________________________________________________________________
+#
+# MusicBrainz -- the open internet music database
+#
+# Copyright (C) 2012 Ian McEwen
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# $Id$
+#____________________________________________________________________________
+
+use FindBin;
+use lib "$FindBin::Bin/../../lib";
+
+use strict;
+
+use Getopt::Long;
+use File::Temp qw( tempdir );
+use File::Path qw( rmtree );
+use File::Copy qw( copy );
+
+my $fHelp;
+my $tmpdir = "/tmp";
+my $period = my $defperiod = 'daily';
+my $outputdir;
+my $start;
+my $limit;
+
+GetOptions(
+ "help" => \$fHelp,
+ "period=s" => \$period,
+ "start=s" => \$start,
+ "limit=s" => \$limit,
+ "output=s" => \$outputdir,
+) or exit 2;
+
+sub usage
+{
+ print <<EOF;
+Usage: BundleReplicationPackets root-directory [options]
+
+ BundleReplicationPackets takes hourly packets since the
+ previous packet for a given period and bundles them
+ together into a new packet. It will not bundle across
+ schema sequence boundaries, however.
+
+ It's the caller's responsibility to ensure the period
+ names accurately describe their contents.
+
+ root-directory is the base directory where packets reside
+
+ --help show this help
+ --period name for the period,
+ default '$defperiod'
+ --start where to start (default: inspect
+ previous packet)
+ --limit how many packets to include (default:
+ include all packets until present
+ --output output directory for finished packet (default same as root)
+EOF
+}
+
+usage(), exit if $fHelp;
+usage(), exit 2 if !@ARGV;
+
+my $rootdir = shift @ARGV;
+$outputdir ||= $rootdir;
+my $num = shift @ARGV;
+
+my @sequences = <$rootdir/replication-*.tar.bz2>;
+@sequences = reverse sort
+ map { s%$rootdir/replication-(\d+)\.tar\.bz2%$1%; $_; }
+ grep { $_ =~ /replication-(\d+)\.tar\.bz2/ } @sequences;
+
+# Don't look at the directory if we've been given a start sequence
+if (!$start) {
+ my @period_sequences = <$rootdir/replication-$period-*.tar.bz2>;
+ @period_sequences = reverse sort map { s%$rootdir/replication-$period-(\d+).tar.bz2%$1%; $_; } @period_sequences;
+
+ if (scalar @period_sequences) {
+ # Open up the most recent one for this period to determine where to start
+ my $period_mostrecent = $period_sequences[0];
+
+ print localtime() . " : Inspecting most recent '$period' packet ($period_mostrecent)\n";
+ my $localfile = "$rootdir/replication-$period-$period_mostrecent.tar.bz2";
+ my $periodcheckdir = tempdir('periodcheck-XXXXXX', DIR => $tmpdir);
+
+ extract_tar($localfile, $periodcheckdir);
+
+ my $last_val = read_file("LAST_REPLICATION_SEQUENCE", $periodcheckdir);
+ if ($last_val =~ /\A(\d+)\n\z/) {
+ $last_val = $1;
+ }
+
+ $start = $last_val + 1;
+
+ print localtime() . " : Removing $periodcheckdir\n";
+ rmtree($periodcheckdir);
+
+ print localtime() . " : Previous '$period' packet ended $last_val, starting $start\n";
+ } else {
+ # Otherwise, assume they want all the packets
+ $start = $sequences[scalar @sequences - 1];
+ print localtime() . " : No previous '$period' packet, using all packets (starting $start)\n";
+ }
+}
+
+$start or die("No start value found");
+
+my @seq_to_use = reverse grep { $_ >= $start } @sequences;
+if ($limit) {
+ die("Not enough existing packets to reach limit $limit") if scalar @seq_to_use < $limit;
+ print localtime() . " : Using a maximum of $limit packets\n";
+ @seq_to_use = @seq_to_use[0..$limit - 1];
+}
+
+print join " ", @seq_to_use;
+
+my $newpacketdir = tempdir("mb-bundle-XXXXXX", DIR => $tmpdir);
+mkdir "$newpacketdir/mbdump" or die $!;
+
+my $schema_sequence;
+for my $sequence (@seq_to_use) {
+ my $individual_packet_dir = tempdir("replication-bundle-XXXXXX", DIR => $tmpdir);
+
+ # untar the packet
+ my $localfile = "$rootdir/replication-$sequence.tar.bz2";
+ extract_tar($localfile, $individual_packet_dir);
+
+ # Set schema sequence, or check it matches previous packets
+ my $this_schema_sequence = read_file('SCHEMA_SEQUENCE', $individual_packet_dir);
+ if ($schema_sequence) {
+ die("Schema sequence mismatch: $this_schema_sequence <> $schema_sequence")
+ unless $schema_sequence eq $this_schema_sequence;
+ } else {
+ $schema_sequence = $this_schema_sequence;
+ for my $file (qw(SCHEMA_SEQUENCE REPLICATION_SEQUENCE COPYING README TIMESTAMP)) {
+ copy("$individual_packet_dir/$file", "$newpacketdir/$file") or die $!;
+ }
+ }
+
+ # concatenate dbmirror_pending/dbmirror_pendingdata files
+ open(my $dbmirror_pending, '<', "$individual_packet_dir/mbdump/dbmirror_pending") or die $!;
+ open(my $new_dbmirror_pending, '>>', "$newpacketdir/mbdump/dbmirror_pending") or die $!;
+ while (<$dbmirror_pending>) {
+ print { $new_dbmirror_pending } $_;
+ }
+ close $dbmirror_pending;
+ close $new_dbmirror_pending;
+
+ open(my $dbmirror_pendingdata, '<', "$individual_packet_dir/mbdump/dbmirror_pendingdata") or die $!;
+ open(my $new_dbmirror_pendingdata, '>>', "$newpacketdir/mbdump/dbmirror_pendingdata") or die $!;
+ while (<$dbmirror_pendingdata>) {
+ print { $new_dbmirror_pendingdata } $_;
+ }
+ close $dbmirror_pendingdata;
+ close $new_dbmirror_pendingdata;
+
+ # copy REPLICATION_SEQUENCE to LAST_REPLICATION_SEQUENCE (also TIMESTAMP)
+ copy("$individual_packet_dir/REPLICATION_SEQUENCE", "$newpacketdir/LAST_REPLICATION_SEQUENCE") or die $!;
+ copy("$individual_packet_dir/TIMESTAMP", "$newpacketdir/LAST_TIMESTAMP") or die $!;
+
+ print localtime() . " : Removing $individual_packet_dir\n";
+ rmtree($individual_packet_dir);
+}
+
+# make tarfile
+
+my $tar_filename = "$outputdir/replication-$period-$start.tar.bz2";
+my @tar_files = (qw(
+ TIMESTAMP LAST_TIMESTAMP COPYING README
+ REPLICATION_SEQUENCE LAST_REPLICATION_SEQUENCE
+ SCHEMA_SEQUENCE
+ mbdump/dbmirror_pending mbdump/dbmirror_pendingdata
+));
+
+print localtime() . " : Creating packet $tar_filename\n";
+
+system { "/bin/tar" } "tar",
+ "-C", $newpacketdir,
+ "--bzip2", "--create",
+ "--verbose",
+ "--file", $tar_filename,
+ "--", @tar_files;
+
+$? == 0 or die "Tar returned $?";
+
+print localtime() . " : Removing $newpacketdir\n";
+rmtree($newpacketdir);
+
+################################################################################
+
+sub read_file
+{
+ my ($file, $base) = @_;
+ open(my $fh, "<$base/$file")
+ or return undef;
+ local $/;
+ <$fh>;
+}
+
+sub write_file
+{
+ my ($file, $contents, $base) = @_;
+ open(my $fh, ">$base/$file") or die $!;
+ print $fh $contents or die $!;
+ close $fh or die $!;
+}
+
+sub extract_tar
+{
+ my ($localfile, $dir) = @_;
+ print localtime() . " : Decompressing $localfile to $dir\n";
+ system "/bin/tar",
+ "-C", $dir,
+ "--bzip2",
+ "-xvf",
+ $localfile,
+ ;
+ exit $? if $?;
+}
commit 8949e6d6ffdbe7d98fcba52e7bea8ca52fb3135c
Author: Ian McEwen <[email protected]>
Date: Tue Sep 11 02:31:14 2012 -0700
MBS-5272: make it possible to do more than increment the replication sequence once after application of a packet
diff --git a/admin/replication/LoadReplicationChanges b/admin/replication/LoadReplicationChanges
index 7e17ca5..e961ea7 100755
--- a/admin/replication/LoadReplicationChanges
+++ b/admin/replication/LoadReplicationChanges
@@ -290,8 +290,8 @@ $iNextReplicationSequence = $iReplicationSequence + 1
not defined($TIMESTAMP)
or $row->{last_replication_date} eq $TIMESTAMP
or warn "Applied changes, but last_replication_date is '$row->{last_replication_date}' not '$TIMESTAMP'\n";
+ $iReplicationSequence = $row->{current_replication_sequence};
}
-++$iReplicationSequence;
$dtReplicationDate = $TIMESTAMP;
$count += 1;
-----------------------------------------------------------------------
hooks/post-receive
--
mb_server