#!/usr/bin/perl

# This is mk-slave-restart, a program to watch replication and try to
# restart the slave on errors.
#
# This program is copyright (c) 2007 Baron Schwartz.  Feedback and
# improvements are welcome.
#
# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
# systems, you can issue `man perlgpl' or `man perlartistic' to read these
# licenses.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA  02111-1307  USA.

use strict;
use warnings FATAL => 'all';

# ###########################################################################
# OptionParser package 1178
# ###########################################################################
use strict;
use warnings FATAL => 'all';

package OptionParser;

use Getopt::Long;
use List::Util qw(max);
use English qw(-no_match_vars);

sub new {
   my ( $class, @opts ) = @_;
   my %key_seen;
   my %long_seen;
   my %key_for;
   my %defaults;
   my @mutex;
   my @atleast1;
   my %long_for;
   my %disables;
   my %copyfrom;
   unshift @opts,
      { s => 'help',    d => 'Show this help message' },
      { s => 'version', d => 'Output version information and exit' };
   foreach my $opt ( @opts ) {
      if ( ref $opt ) {
         my ( $long, $short ) = $opt->{s} =~ m/^([\w-]+)(?:\|([^!+=]*))?/;
         $opt->{k} = $short || $long;
         $key_for{$long} = $opt->{k};
         $long_for{$opt->{k}} = $long;
         $long_for{$long} = $long;
         $opt->{l} = $long;
         die "Duplicate option $opt->{k}" if $key_seen{$opt->{k}}++;
         die "Duplicate long option $opt->{l}" if $long_seen{$opt->{l}}++;
         $opt->{t} = $short;
         $opt->{n} = $opt->{s} =~ m/!/;
         $opt->{g} ||= 'o';
         if ( (my ($y) = $opt->{s} =~ m/=([mdHhAaz])/) ) {
            $opt->{y} = $y;
            $opt->{s} =~ s/=./=s/;
         }
         $opt->{r} = $opt->{d} =~ m/required/;
         if ( (my ($def) = $opt->{d} =~ m/default(?: ([^)]+))?/) ) {
            $defaults{$opt->{k}} = defined $def ? $def : 1;
         }
         if ( (my ($dis) = $opt->{d} =~ m/(disables .*)/) ) {
            $disables{$opt->{k}} = [ $class->get_participants($dis) ];
         }
      }
      else { # It's an instruction.

         if ( $opt =~ m/at least one|mutually exclusive|one and only one/ ) {
            my @participants = map {
                  die "No such option '$_' in $opt" unless $long_for{$_};
                  $long_for{$_};
               } $class->get_participants($opt);
            if ( $opt =~ m/mutually exclusive|one and only one/ ) {
               push @mutex, \@participants;
            }
            if ( $opt =~ m/at least one|one and only one/ ) {
               push @atleast1, \@participants;
            }
         }
         elsif ( $opt =~ m/default to/ ) {
            my @participants = map {
                  die "No such option '$_' in $opt" unless $long_for{$_};
                  $key_for{$_};
               } $class->get_participants($opt);
            $copyfrom{$participants[0]} = $participants[1];
         }

      }
   }

   foreach my $dis ( keys %disables ) {
      $disables{$dis} = [ map {
            die "No such option '$_' while processing $dis" unless $long_for{$_};
            $long_for{$_};
         } @{$disables{$dis}} ];
   }

   return bless {
      specs => [ grep { ref $_ } @opts ],
      notes => [],
      instr => [ grep { !ref $_ } @opts ],
      mutex => \@mutex,
      defaults => \%defaults,
      long_for => \%long_for,
      atleast1 => \@atleast1,
      disables => \%disables,
      key_for  => \%key_for,
      copyfrom => \%copyfrom,
      strict   => 1,
      groups   => [ { k => 'o', d => 'Options' } ],
   }, $class;
}

sub get_participants {
   my ( $self, $str ) = @_;
   my @participants;
   foreach my $thing ( $str =~ m/(--?[\w-]+)/g ) {
      if ( (my ($long) = $thing =~ m/--(.+)/) ) {
         push @participants, $long;
      }
      else {
         foreach my $short ( $thing =~ m/([^-])/g ) {
            push @participants, $short;
         }
      }
   }
   return @participants;
}

sub parse {
   my ( $self, %defaults ) = @_;
   my @specs = @{$self->{specs}};
   my %factor_for = (k => 1_024, M => 1_048_576, G => 1_073_741_824);

   my %opt_seen;
   my %vals = %{$self->{defaults}};
   @vals{keys %defaults} = values %defaults;
   foreach my $spec ( @specs ) {
      $vals{$spec->{k}} = undef unless defined $vals{$spec->{k}};
      $opt_seen{$spec->{k}} = 1;
   }

   foreach my $key ( keys %defaults ) {
      die "Cannot set default for non-existent option '$key'\n"
         unless $opt_seen{$key};
   }

   Getopt::Long::Configure('no_ignore_case', 'bundling');
   GetOptions( map { $_->{s} => \$vals{$_->{k}} } @specs )
      or $self->error('Error parsing options');

   if ( $vals{version} ) {
      my $prog = $self->prog;
      printf("%s  Ver %s Distrib %s Changeset %s\n",
         $prog, $main::VERSION, $main::DISTRIB, $main::SVN_REV);
      exit(0);
   }

   if ( @ARGV && $self->{strict} ) {
      $self->error("Unrecognized command-line options @ARGV");
   }

   foreach my $dis ( grep { defined $vals{$_} } keys %{$self->{disables}} ) {
      my @disses = map { $self->{key_for}->{$_} } @{$self->{disables}->{$dis}};
      @vals{@disses} = map { undef } @disses;
   }

   foreach my $spec ( grep { $_->{r} } @specs ) {
      if ( !defined $vals{$spec->{k}} ) {
         $self->error("Required option --$spec->{l} must be specified");
      }
   }

   foreach my $mutex ( @{$self->{mutex}} ) {
      my @set = grep { defined $vals{$self->{key_for}->{$_}} } @$mutex;
      if ( @set > 1 ) {
         my $note = join(', ',
            map { "--$self->{long_for}->{$_}" }
                @{$mutex}[ 0 .. scalar(@$mutex) - 2] );
         $note .= " and --$self->{long_for}->{$mutex->[-1]}"
               . " are mutually exclusive.";
         $self->error($note);
      }
   }

   foreach my $required ( @{$self->{atleast1}} ) {
      my @set = grep { defined $vals{$self->{key_for}->{$_}} } @$required;
      if ( !@set ) {
         my $note = join(', ',
            map { "--$self->{long_for}->{$_}" }
                @{$required}[ 0 .. scalar(@$required) - 2] );
         $note .= " or --$self->{long_for}->{$required->[-1]}";
         $self->error("Specify at least one of $note");
      }
   }

   foreach my $spec ( grep { $_->{y} && defined $vals{$_->{k}} } @specs ) {
      my $val = $vals{$spec->{k}};
      if ( $spec->{y} eq 'm' ) {
         my ( $num, $suffix ) = $val =~ m/(\d+)([smhd])$/;
         if ( $suffix ) {
            $val = $suffix eq 's' ? $num            # Seconds
                 : $suffix eq 'm' ? $num * 60       # Minutes
                 : $suffix eq 'h' ? $num * 3600     # Hours
                 :                  $num * 86400;   # Days
            $vals{$spec->{k}} = $val;
         }
         else {
            $self->error("Invalid --$spec->{l} argument");
         }
      }
      elsif ( $spec->{y} eq 'd' ) {
         my $from_key = $self->{copyfrom}->{$spec->{k}};
         my $default = {};
         if ( $from_key ) {
            $default = $self->{dsn}->parse($self->{dsn}->as_string($vals{$from_key}));
         }
         $vals{$spec->{k}} = $self->{dsn}->parse($val, $default);
      }
      elsif ( $spec->{y} eq 'z' ) {
         my ($pre, $num, $factor) = $val =~ m/^([+-])?(\d+)([kMG])?$/;
         if ( defined $num ) {
            if ( $factor ) {
               $num *= $factor_for{$factor};
            }
            $vals{$spec->{k}} = ($pre || '') . $num;
         }
         else {
            $self->error("Invalid --$spec->{l} argument");
         }
      }
   }

   foreach my $spec ( grep { $_->{y} } @specs ) {
      my $val = $vals{$spec->{k}};
      if ( $spec->{y} eq 'H' || (defined $val && $spec->{y} eq 'h') ) {
         $vals{$spec->{k}} = { map { $_ => 1 } split(',', ($val || '')) };
      }
      elsif ( $spec->{y} eq 'A' || (defined $val && $spec->{y} eq 'a') ) {
         $vals{$spec->{k}} = [ split(',', ($val || '')) ];
      }
   }

   return %vals;
}

sub error {
   my ( $self, $note ) = @_;
   $self->{__error__} = 1;
   push @{$self->{notes}}, $note;
}

sub prog {
   (my $prog) = $PROGRAM_NAME =~ m/([.A-Za-z-]+)$/;
   return $prog || $PROGRAM_NAME;
}

sub prompt {
   my ( $self ) = @_;
   my $prog   = $self->prog;
   my $prompt = $self->{prompt} || '<options>';
   return "Usage: $prog $prompt\n";
}

sub descr {
   my ( $self ) = @_;
   my $prog = $self->prog;
   my $descr  = $prog . ' ' . ($self->{descr} || '')
          . "  For more details, please use the --help option, "
          . "or try 'perldoc $prog' for complete documentation.";
   $descr = join("\n", $descr =~ m/(.{0,80})(?:\s+|$)/g);
   $descr =~ s/ +$//mg;
   return $descr;
}

sub usage_or_errors {
   my ( $self, %opts ) = @_;
   if ( $opts{help} ) {
      print $self->usage(%opts);
      exit(0);
   }
   elsif ( $self->{__error__} ) {
      print $self->errors();
      exit(0);
   }
}

sub errors {
   my ( $self ) = @_;
   my $usage = $self->prompt() . "\n";
   if ( (my @notes = @{$self->{notes}}) ) {
      $usage .= join("\n  * ", 'Errors in command-line arguments:', @notes) . "\n";
   }
   return $usage . "\n" . $self->descr();
}

sub usage {
   my ( $self, %vals ) = @_;
   my @specs = @{$self->{specs}};

   my $maxl = max(map { length($_->{l}) + ($_->{n} ? 4 : 0)} @specs);

   my $maxs = max(0,
      map { length($_->{l}) + ($_->{n} ? 4 : 0)}
      grep { $_->{t} } @specs);

   my $lcol = max($maxl, ($maxs + 3));
   my $rcol = 80 - $lcol - 6;
   my $rpad = ' ' x ( 80 - $rcol );

   $maxs = max($lcol - 3, $maxs);

   my $usage = $self->descr() . "\n" . $self->prompt();
   foreach my $g ( @{$self->{groups}} ) {
      $usage .= "\n$g->{d}:\n";
      foreach my $spec ( sort { $a->{l} cmp $b->{l} } grep { $_->{g} eq $g->{k} } @specs ) {
         my $long  = $spec->{n} ? "[no]$spec->{l}" : $spec->{l};
         my $short = $spec->{t};
         my $desc  = $spec->{d};
         $desc = join("\n$rpad", grep { $_ } $desc =~ m/(.{0,$rcol})(?:\s+|$)/g);
         $desc =~ s/ +$//mg;
         if ( $short ) {
            $usage .= sprintf("  --%-${maxs}s -%s  %s\n", $long, $short, $desc);
         }
         else {
            $usage .= sprintf("  --%-${lcol}s  %s\n", $long, $desc);
         }
      }
   }

   if ( (my @instr = @{$self->{instr}}) ) {
      $usage .= join("\n", map { "  $_" } @instr) . "\n";
   }
   if ( $self->{dsn} ) {
      $usage .= "\n" . $self->{dsn}->usage();
   }
   $usage .= "\nOptions and values after processing arguments:\n";
   foreach my $spec ( sort { $a->{l} cmp $b->{l} } @specs ) {
      my $val   = $vals{$spec->{k}};
      my $type  = $spec->{y} || '';
      my $bool  = $spec->{s} =~ m/^[\w-]+(?:\|[\w-])?!?$/;
      $val      = $bool                     ? ( $val ? 'TRUE' : 'FALSE' )
                : !defined $val             ? '(No value)'
                : $type eq 'd'              ? $self->{dsn}->as_string($val)
                : $type =~ m/H|h/           ? join(',', sort keys %$val)
                : $type =~ m/A|a/           ? join(',', @$val)
                :                             $val;
      $usage .= sprintf("  --%-${lcol}s  %s\n", $spec->{l}, $val);
   }
   return $usage;
}

sub prompt_noecho {
   shift @_ if ref $_[0] eq __PACKAGE__;
   my ( $prompt ) = @_;
   local $OUTPUT_AUTOFLUSH = 1;
   print $prompt;
   my $response;
   eval {
      require Term::ReadKey;
      Term::ReadKey::ReadMode('noecho');
      chomp($response = <STDIN>);
      Term::ReadKey::ReadMode('normal');
      print "\n";
   };
   if ( $EVAL_ERROR ) {
      die "Cannot read response; is Term::ReadKey installed? $EVAL_ERROR";
   }
   return $response;
}

sub groups {
   my ( $self, @groups ) = @_;
   push @{$self->{groups}}, @groups;
}

1;

# ###########################################################################
# End OptionParser package
# ###########################################################################

# ###########################################################################
# VersionParser package 1149
# ###########################################################################
use strict;
use warnings FATAL => 'all';

package VersionParser;

sub new {
   my ( $class ) = @_;
   bless {}, $class;
}

sub parse {
   my ( $self, $str ) = @_;
   return sprintf('%03d%03d%03d', $str =~ m/(\d+)/g);
}

sub version_ge {
   my ( $self, $dbh, $target ) = @_;
   $self->{$dbh} ||= $self->parse(
      $dbh->selectrow_array('SELECT VERSION()'));
   return $self->{$dbh} ge $self->parse($target);
}

1;

# ###########################################################################
# End VersionParser package
# ###########################################################################

# ###########################################################################
# DSNParser package 1149
# ###########################################################################
use strict;
use warnings FATAL => 'all';

package DSNParser;

sub new {
   my ( $class, @opts ) = @_;
   my $self = {
      opts => {
         D => {
            desc => 'Database to use',
            dsn  => 'database',
            copy => 1,
         },
         F => {
            desc => 'Only read default options from the given file',
            dsn  => 'mysql_read_default_file',
            copy => 1,
         },
         h => {
            desc => 'Connect to host',
            dsn  => 'host',
            copy => 1,
         },
         p => {
            desc => 'Password to use when connecting',
            dsn  => 'password',
            copy => 1,
         },
         P => {
            desc => 'Port number to use for connection',
            dsn  => 'port',
            copy => 1,
         },
         S => {
            desc => 'Socket file to use for connection',
            dsn  => 'mysql_socket',
            copy => 1,
         },
         u => {
            desc => 'User for login if not current user',
            dsn  => 'user',
            copy => 1,
         },
      },
   };
   foreach my $opt ( @opts ) {
      $self->{opts}->{$opt->{key}} = { desc => $opt->{desc}, copy => $opt->{copy} };
   }
   return bless $self, $class;
}

sub prop {
   my ( $self, $prop, $value ) = @_;
   if ( @_ > 2 ) {
      $self->{$prop} = $value;
   }
   return $self->{$prop};
}

sub parse {
   my ( $self, $dsn, $prev, $defaults ) = @_;
   return unless $dsn;
   $prev     ||= {};
   $defaults ||= {};
   my %vals;
   my %opts = %{$self->{opts}};
   if ( $dsn !~ m/=/ && $self->prop('autokey') ) {
      $vals{ $self->prop('autokey') } = $dsn;
   }
   else {
      my %hash = map { m/^(.)=(.*)$/g } split(/,/, $dsn);
      foreach my $key ( keys %opts ) {
         $vals{$key} = $hash{$key};
         if ( !defined $vals{$key} && defined $prev->{$key} && $opts{$key}->{copy} ) {
            $vals{$key} = $prev->{$key};
         }
         if ( !defined $vals{$key} ) {
            $vals{$key} = $defaults->{$key};
         }
      }
      foreach my $key ( keys %hash ) {
         die "Unrecognized DSN part '$key' in '$dsn'\n"
            unless exists $opts{$key};
      }
   }
   if ( (my $required = $self->prop('required')) ) {
      foreach my $key ( keys %$required ) {
         die "Missing DSN part '$key' in '$dsn'\n" unless $vals{$key};
      }
   }
   return \%vals;
}

sub as_string {
   my ( $self, $dsn ) = @_;
   return $dsn unless ref $dsn;
   return join(',', map { "$_=$dsn->{$_}" } grep { defined $dsn->{$_} } sort keys %$dsn );
}

sub usage {
   my ( $self ) = @_;
   my $usage
      = "DSN syntax is key=value[,key=value...]  Allowable DSN keys:\n"
      . "  KEY  COPY  MEANING\n"
      . "  ===  ====  =============================================\n";
   my %opts = %{$self->{opts}};
   foreach my $key ( sort keys %opts ) {
      $usage .= "  $key    "
             .  ($opts{$key}->{copy} ? 'yes   ' : 'no    ')
             .  ($opts{$key}->{desc} || '[No description]')
             . "\n";
   }
   if ( (my $key = $self->prop('autokey')) ) {
      $usage .= "  If the DSN is a bareword, the word is treated as the '$key' key.\n";
   }
   return $usage;
}

sub get_cxn_params {
   my ( $self, $info ) = @_;
   my $dsn;
   my %opts = %{$self->{opts}};
   my $driver = $self->prop('dbidriver') || '';
   if ( $driver eq 'Pg' ) {
      $dsn = 'DBI:Pg:dbname=' . ( $info->{D} || '' ) . ';'
         . join(';', map  { "$opts{$_}->{dsn}=$info->{$_}" }
                     grep { defined $info->{$_} }
                     qw(h P));
   }
   else {
      $dsn = 'DBI:mysql:' . ( $info->{D} || '' ) . ';'
         . join(';', map  { "$opts{$_}->{dsn}=$info->{$_}" }
                     grep { defined $info->{$_} }
                     qw(F h P S))
         . ';mysql_read_default_group=mysql';
   }
   return ($dsn, $info->{u}, $info->{p});
}

1;

# ###########################################################################
# End DSNParser package
# ###########################################################################
package main;

use DBI;
use English qw(-no_match_vars);
use List::Util qw(min max);
use Time::HiRes qw(sleep);

our $VERSION = '1.0.2';
our $DISTRIB = '1316';
our $SVN_REV = sprintf("%d", q$Revision: 1308 $ =~ m/(\d+)/g || 0);

# ############################################################################
# Get configuration information.
# ############################################################################

my @opt_spec = (
   { s => 'askpass',           d => 'Prompt for a password for the connection' },
   { s => 'daemonize',         d => 'Fork to background and detach (POSIX only)' },
   { s => 'database|D=s',      d => 'Database to use' },
   { s => 'defaults-file|F=s', d => 'Only read mysql options from the given file' },
   { s => 'error-numbers|e=h', d => 'Only restart this comma-separated list of errors' },
   { s => 'error-text|E=s',    d => 'Only restart errors that match this pattern' },
   { s => 'error-length|L=i',  d => 'Max length of error message to print' },
   { s => 'host|h=s',          d => 'Connect to host' },
   { s => 'maxsleep|M=f',      d => 'Maximum sleep seconds (default 64)'},
   { s => 'minsleep|m=f',      d => 'Minimum sleep seconds (default 0.015625)'},
   { s => 'password|p=s',      d => 'Password to use when connecting' },
   { s => 'port|P=i',          d => 'Port number to use for connection' },
   { s => 'skipcount|k=i',     d => 'Number of statements to skip (default 1)' },
   { s => 'sleep|s=f',         d => 'Initial sleep seconds (default 1)' },
   { s => 'socket|S=s',        d => 'Socket file to use for connection' },
   { s => 'time|t=m',          d => 'Time to run before exiting (suffix: s/m/h/d)' },
   { s => 'untilmaster=s',     d => 'Run until this master log file and position' },
   { s => 'untilrelay=s',      d => 'Run until this relay log file and position' },
   { s => 'user|u=s',          d => 'User for login if not current user' },
   { s => 'verbose|v+',        d => 'Verbosity; can specify multiple times; default 0' },
);

my $dsn_parser = new DSNParser();
my $vp = new VersionParser();
my $opt_parser = new OptionParser(@opt_spec);
$opt_parser->{prompt} = '<options>';
$opt_parser->{descr}  = 'watches a MySQL replication slave for errors, and '
                      . 'tries to restart replication if it stops.';
my %opts = $opt_parser->parse();

if ( !$opts{help} ) {
   if ( $opts{untilmaster} ) {
      if ( $opts{untilmaster} !~ m/^[.\w-]+,\d+$/ ) {
         $opt_parser->error("Invalid --untilmaster argument, must be file,pos");
      }
   }
   if ( $opts{untilrelay} ) {
      if ( $opts{untilrelay} !~ m/^[.\w-]+,\d+$/ ) {
         $opt_parser->error("Invalid --untilrelay argument, must be file,pos");
      }
   }
}

$opt_parser->usage_or_errors(%opts);

# ############################################################################
# Connect and go to work.
# ############################################################################
if ( $opts{askpass} ) {
   $opts{p} = OptionParser::prompt_noecho("Enter password: ");
}
my $dbh = DBI->connect($dsn_parser->get_cxn_params(\%opts), { AutoCommit => 1, RaiseError => 1, PrintError => 0 } );

# VERY IMPORTANT: Lowercases all column names for fetchrow_hashref.  This is
# because different MySQL versions use different lettercase in SHOW SLAVE
# STATUS.
$dbh->{FetchHashKeyName} = 'NAME_lc';
$dbh->{InactiveDestroy}  = 1;         # Don't disconnect on fork/daemonize

# Daemonize only after connecting and doing --askpass.
if ( $opts{daemonize} ) {
   require POSIX;
   chdir '/'                 or die "Can't chdir to /: $OS_ERROR";
   open STDIN, '/dev/null'   or die "Can't read /dev/null: $OS_ERROR";
   open STDOUT, '>/dev/null' or die "Can't write to /dev/null: $OS_ERROR";
   defined( my $pid = fork ) or die "Can't fork: $OS_ERROR";
   exit if $pid;
   POSIX::setsid()           or die "Can't start a new session: $OS_ERROR";
   open STDERR, '>&STDOUT'   or die "Can't dup STDOUT: $OS_ERROR";
}

my $start_sql = $vp->version_ge($dbh, '4.0.5') ? 'START SLAVE' : 'SLAVE START';
if ( $opts{untilmaster} ) {
   my ( $file, $pos ) = split(',', $opts{untilmaster});
   $start_sql .= " UNTIL MASTER_LOG_FILE = '$file', MASTER_LOG_POS = $pos";
}
elsif ( $opts{untilrelay} ) {
   my ( $file, $pos ) = split(',', $opts{untilrelay});
   $start_sql .= " UNTIL RELAY_LOG_FILE = '$file', RELAY_LOG_POS = $pos";
}

my $fetch_stat = $dbh->prepare('SHOW SLAVE STATUS');
my $set_skip   = $dbh->prepare("SET GLOBAL SQL_SLAVE_SKIP_COUNTER = $opts{k}");
my $start      = $dbh->prepare($start_sql);

my $exit_time = time() + ($opts{t} || 0);
my $sleep = $opts{s};
my ($last_log, $last_pos);

while ( ( !$opts{t} || time() < $exit_time ) ) {
   my $was_running = 1;
   $fetch_stat->execute();
   my $stat = $fetch_stat->fetchall_arrayref({})->[0];
   die "No SLAVE STATUS output found\n" unless $stat;

   if ( !$last_log
      || $last_log ne $stat->{relay_log_file}
      || $last_pos != $stat->{relay_log_pos}
   ) {
      $stat->{slave_sql_running} ||= 'No';
      $stat->{last_error}        ||= '';
      $stat->{last_errno}        ||= 0;

      if ( $opts{untilmaster} && pos_ge($stat, 'master') ) {
         die "Slave has advanced past $opts{untilmaster} on master.\n";
      }
      elsif ( $opts{untilrelay} && pos_ge($stat, 'relay') ) {
         die "Slave has advanced past $opts{untilrelay} in relay logs.\n";
      }

      if ( $stat->{slave_sql_running} eq 'No' ) {
         # Print the time, error, etc
         if ( $opts{v} ) {
            my $err = '';
            if ( $opts{v} > 1 ) {
               ($err = $stat->{last_error} ) =~ s/\s+/ /g;
               if ( $opts{L} ) {
                  $err = substr($err, 0, $opts{L});
               }
            }
            printf("%s %s %11d %d %s\n",
               ts(time),
               $stat->{relay_log_file},
               $stat->{relay_log_pos},
               $stat->{last_errno} || 0,
               $err
            );
         }

         if ( $opts{e} && !exists($opts{e}->{$stat->{last_errno}}) ) {
            die "Error $stat->{last_errno} is not in --error-numbers.\n";
         }
         elsif ( $opts{E} && $stat->{last_error} && $stat->{last_error} !~ m/$opts{E}/ ) {
            die "Error does not match --error-text.\n";
         }
         else {
            $set_skip->execute();
            $start->execute();
            $was_running = 0;
            # Only set this on events I tried to restart.  Otherwise there
            # could be a race condition: I see it, I record it, but it hasn't
            # caused an error yet; so I won't try to restart it when it does.
            # (The point of this is to avoid trying to restart the same event
            # twice in case another race condition happens -- I restart it,
            # then check the server and it hasn't yet cleared the error
            # message and restarted the SQL thread).
            $last_log = $stat->{relay_log_file};
            $last_pos = $stat->{relay_log_pos};
         }
      }
   }

   # Adjust sleep time.
   if ( $was_running ) {
      $sleep = min($opts{M}, $sleep * 2);
   }
   else {
      $sleep = max($opts{m}, $sleep / 2);
   }

   # Errors are very likely to follow each other in quick succession.  NOTE:
   # this policy has a side effect with respect to $sleep.  Suppose $sleep is
   # 512 and mk-slave-restart finds an error; now $sleep is 256, but
   # mk-slave-restart sleeps only 1 (the initial value of --sleep).  Suppose
   # there is no error when it wakes up after 1 second, because 1 was too
   # short.  Now it doubles $sleep, back to 512.  $sleep has the same value it
   # did before the error was ever found.
   print "sleeping $sleep\n" if $opts{v} > 2;
   sleep($was_running ? $sleep : min($sleep, $opts{s}));
}

# ############################################################################
# Subroutines.
# ############################################################################

# Determines if the $stat's log coordinates are greater than or equal to the
# desired coordinates. $which is 'master' or 'relay'
sub pos_ge {
   my ( $stat, $which ) = @_;
   my $fmt  = '%s/%020d';
   my $curr = $which eq 'master'
      ? sprintf($fmt, @{$stat}{qw(relay_master_log_file exec_master_log_pos)})
      : sprintf($fmt, @{$stat}{qw(relay_log_file relay_log_pos)});
   my $stop = sprintf($fmt, split(',', $opts{"until$which"}));
   return $curr ge $stop;
}

sub ts {
   my ( $time ) = @_;
   my ( $sec, $min, $hour, $mday, $mon, $year )
      = localtime($time);
   $mon  += 1;
   $year += 1900;
   return sprintf("%d-%02d-%02dT%02d:%02d:%02d",
      $year, $mon, $mday, $hour, $min, $sec);
}

# ############################################################################
# Documentation.
# ############################################################################

=pod

=head1 NAME

mk-slave-restart - Watch and restart MySQL replication after errors.

=head1 SYNOPSIS

 mk-slave-restart --verbose

=head1 DESCRIPTION

mk-slave-restart watches a MySQL replication slave and tries to skip
statements that cause errors.  It polls the slave intelligently with an
exponentially varying sleep time.  You can specify errors to skip and run the
slave until a certain binlog position.

Note: it has come to my attention that Yahoo! had or has an internal tool
called fix_repl, described to me by a past Yahoo! employee and mentioned in
the first edition of High Performance MySQL.  Apparently this tool does the
same thing.  Make no mistake, though: this is not a way to "fix replication."
In fact I would not even encourage its use on a regular basis; I only use it
when I have an error I know I just need to skip past.

Indiscriminate use of this tool can easily screw up a server you might have
had a chance to truly fix.  You have been warned.

=head1 OPTIONS

=over

=item --askpass

Prompt for a password for the connection.

=item --daemonize

Fork to the background and detach from the shell.  This probably doesn't work
on Microsoft Windows.

=item --database

Database to use.

=item --defaults-file

Only read default options from the given file.

=item --error-numbers

Makes mk-slave-restart only try to restart if the error number is in this
comma-separated list of errors.  If it sees an error not in the list, it will
exit.

The error number is in the last_errno column of SHOW SLAVE STATUS.

=item --error-text

A Perl regular expression against which the error text, if any, is matched.  If
the error text exists and matches, mk-slave-restart will try to restart the
slave.  If it exists but doesn't match, mk-slave-restart will exit.

The error text is in the last_error column of SHOW SLAVE STATUS.

=item --error-length

When L<"--verbose"> is set high enough to print the error, this option will
truncate the error text to the specified length.  This can be useful to prevent
wrapping on the terminal.

=item --help

Show a brief help message and exit.

=item --host

Connect to host.

=item --maxsleep

The maximum time mk-slave-restart will sleep before polling the slave again.
See L<"SLEEP">.

=item --minsleep

The minimum time mk-slave-restart will sleep before polling the slave again.
See L<"SLEEP">.

=item --password

Password to use when connecting.

=item --port

Port number to use for connection.

=item --skipcount

The number of statements to skip when restarting the slave.

=item --sleep

The initial sleep time between checking the slave.  See L<"SLEEP">.

=item --socket

Socket file to use for connection.

=item --time

Causes mk-slave-restart to stop after the specified time has elapsed.  The
argument can have a suffix of s, m, h, or d, indicating seconds, minutes, hours,
or days.  The number is interpreted as seconds if there is no suffix.

=item --untilmaster

Start the slave, and retry if it fails, until it reaches the given replication
coordinates.  The coordinates are the logfile and position on the master, given
by relay_master_log_file, exec_master_log_pos.  The argument must be in the
format "file,pos".  Separate the filename and position with a single comma and
no space.

This will also cause an UNTIL clause to be given to START SLAVE.

After reaching this point, the slave should be stopped and mk-slave-restart
will exit.

=item --untilrelay

Like L<"--untilmaster">, but in the slave's relay logs instead.  The coordinates
are given by relay_log_file, relay_log_pos.

=item --user

User for login if not current user.

=item --verbose

Verbosity; specify multiple times for more verbosity.  Default is no output.
Verbosity 1 outputs a timestamp, relay_log_file, relay_log_pos, and last_errno.
Verbosity 2 adds last_error.  See also L<"--error-length">.  Verbosity 3
prints the current sleep time each time mk-slave-restart sleeps.

=item --version

Output version information and exit.

=back

=head1 SYSTEM REQUIREMENTS

You need Perl, DBI, DBD::mysql, and some core packages that ought to be
installed in any reasonably new version of Perl.

=head1 OUTPUT

If you specify --verbose, mk-slave-restart prints a line every time it sees
the slave has an error.  See L<"--verbose"> for details.

=head1 SLEEP

mk-slave-restart sleeps intelligently between polling the slave.  The current
sleep time varies.

=over

=item *

The initial sleep time is given by L<"--sleep">.

=item *

If it checks and finds an error, it halves the previous sleep time.

=item *

If it finds no error, it doubles the previous sleep time.

=item *

The sleep time is bounded below by L<"--minsleep"> and above by L<"--maxsleep">.

=item *

Immediately after finding an error, mk-slave-restart assumes another error is
very likely to happen next, so it sleeps the current sleep time or the initial
sleep time, whichever is less.

=back

=head1 COMPATIBILITY

mk-slave-restart should work on many versions of MySQL.  Lettercase of many
output columns from SHOW SLAVE STATUS has changed over time, so it treats them
all as lowercase.

=head1 BUGS

If you find bugs, need features, etc please use the bug tracker, forums, and
mailing lists at http://sourceforge.net/projects/maatkit.

=head1 SEE ALSO

See also L<mk-table-checksum>, L<mk-table-sync>, L<mk-slave-delay>.

=head1 COPYRIGHT, LICENSE AND WARRANTY

This program is copyright (c) 2007 Baron Schwartz.  Feedback and improvements
are welcome.

THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
systems, you can issue `man perlgpl' or `man perlartistic' to read these
licenses.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA  02111-1307  USA.

=head1 AUTHOR

Baron Schwartz

=head1 VERSION

This manual page documents Ver 1.0.2 Distrib 1316 $Revision: 1308 $.

=cut
