added tool to recover disk queue if .qi file is missing (recover_qi.pl)

3 files changed, 211 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index ef9d3735..c23f973e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.8.9  [V5-stable] 2012-03-??
+- added tool to recover disk queue if .qi file is missing (recover_qi.pl)
+  Thanks to Kaiwang Chen for contributing this tool
 - bugfix: stopped DA queue was never processed after a restart due to a
   regression from statistics module
 - added better doc for statsobj interface
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 96657ad4..962ae507 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -47,4 +47,5 @@ zpipe_LDADD = -lz
 msggen_SOURCES = msggen.c
 endif
 
-EXTRA_DIST = $(man_MANS)
+EXTRA_DIST = $(man_MANS) \
+	recover_qi.pl
diff --git a/tools/recover_qi.pl b/tools/recover_qi.pl
new file mode 100644
index 00000000..4e2cf9d5
--- /dev/null
+++ b/tools/recover_qi.pl
@@ -0,0 +1,207 @@
+#!/usr/bin/perl -w

+# recover rsyslog disk queue index (.qi) from queue files (.nnnnnnnn).

+#

+# See:

+#   runtime/queue.c: qqueuePersist()

+#   runtime/queue.c: qqueueTryLoadPersistedInfo()

+#

+# kaiwang.chen@gmail.com 2012-03-14

+#

+use strict;

+use Getopt::Long;

+

+my %opt = ();

+GetOptions(\%opt,"spool|w=s","basename|f=s","digits|d=i","help!");

+if ($opt{help}) {

+  print "Usage:

+\t$0 -w WorkDirectory -f QueueFileName -d 8 > QueueFileName.qi

+";

+  exit;

+}

+

+# runtime/queue.c: qConstructDisk()

+my $iMaxFiles = 10000000; # 0+"1".( "0"x($opt{digits} - 1));

+

+# get the list of queue files, spool directory excluded

+my $re = qr/^\Q$opt{basename}\E\.\d{$opt{digits}}$/;

+opendir(DIR, $opt{spool}) or die "can’t open spool: $!";

+my @qf = grep { /$re/ && -f "$opt{spool}/$_" } readdir(DIR);

+closedir DIR;

+

+# ensure order and continuity

+@qf = sort @qf;

+my ($head) = ($qf[0] =~ /(\d+)$/);

+my ($tail) = ($qf[-1] =~ /(\d+)$/);

+$head += 0;

+$tail += 0;

+if ($tail-$head+1 != @qf || $tail > $iMaxFiles) {

+  die "broken queue: missing file(s) or wrong tail\n";

+}

+

+# collect some counters about the queue, assuming all are unprocessed entries.

+my $sizeOnDisk = 0;

+my $iQueueSize = 0;

+chdir($opt{spool}) or die "can't chdir to spool: $!";

+print STDERR "traversing ". @qf ." files, please wait...\n";

+for (@qf) {

+  open FH, "<", $_ or die "can't read queue file $_\n";

+  $sizeOnDisk += (stat FH)[7];

+  while (<FH>) {

+    $iQueueSize++ if /^<Obj/;  # runtime/msg.c: MsgSerialize()

+  }

+  close FH;

+}

+# happen to reuse last stat

+my $iCurrOffs_Write = (stat(_))[7];

+

+# runtime/queue.c: qqueuePersist()

+my $qqueue = Rsyslog::OPB->new("qqueue",1);

+$qqueue->property("iQueueSize", "INT", $iQueueSize);

+$qqueue->property("tVars.disk.sizeOnDisk", "INT64", $sizeOnDisk);

+$qqueue->property("tVars.disk.bytesRead", "INT64", 0);

+

+# runtime/stream.h: strmType_t

+my $STREAMTYPE_FILE_CIRCULAR = 1;

+# runtime/stream.h: strmMode_t

+my $STREAMMODE_READ = 1;

+my $STREAMMODE_WRITE_APPEND = 4;

+

+# runtime/stream.c: strmSerialize()

+# write to end

+my $strm_Write = Rsyslog::Obj->new("strm",1);

+$strm_Write->property(      "iCurrFNum",  "INT",                     $tail);

+$strm_Write->property(       "pszFName",  "PSZ",            $opt{basename});

+$strm_Write->property(      "iMaxFiles",  "INT",                $iMaxFiles);

+$strm_Write->property( "bDeleteOnClose",  "INT",                         0);

+$strm_Write->property(          "sType",  "INT", $STREAMTYPE_FILE_CIRCULAR);

+$strm_Write->property("tOperationsMode",  "INT",  $STREAMMODE_WRITE_APPEND);

+$strm_Write->property(      "tOpenMode",  "INT",                      0600);

+$strm_Write->property(      "iCurrOffs","INT64",          $iCurrOffs_Write);

+# read from head

+my $strm_ReadDel = Rsyslog::Obj->new("strm",1);

+$strm_ReadDel->property(      "iCurrFNum",  "INT",                     $head);

+$strm_ReadDel->property(       "pszFName",  "PSZ",            $opt{basename});

+$strm_ReadDel->property(      "iMaxFiles",  "INT",                $iMaxFiles);

+$strm_ReadDel->property( "bDeleteOnClose",  "INT",                         1);

+$strm_ReadDel->property(          "sType",  "INT", $STREAMTYPE_FILE_CIRCULAR);

+$strm_ReadDel->property("tOperationsMode",  "INT",          $STREAMMODE_READ);

+$strm_ReadDel->property(      "tOpenMode",  "INT",                      0600);

+$strm_ReadDel->property(      "iCurrOffs","INT64",                         0);

+

+# .qi

+print $qqueue->serialize();

+print $strm_Write->serialize();

+print $strm_ReadDel->serialize();

+

+exit;

+#-----------------------------------------------------------------------------

+

+package Rsyslog::Serializable;

+# runtime/obj.c 

+sub COOKIE_OBJLINE   { '<' }

+sub COOKIE_PROPLINE  { '+' }

+sub COOKIE_ENDLINE   { '>' }

+sub COOKIE_BLANKLINE { '.' }

+# VARTYPE(short_ptype)

+sub VARTYPE {

+  my ($t) = @_;

+  # runtime/obj-types.h: propType_t

+  my $ptype = "PROPTYPE_".$t;

+  # runtime/var.h: varType_t

+  my %vm = ( 

+    VARTYPE_NONE       => 0,

+    VARTYPE_STR        => 1,

+    VARTYPE_NUMBER     => 2,

+    VARTYPE_SYSLOGTIME => 3,

+  );

+  # runtime/obj.c: SerializeProp()

+  my %p2v = (

+    #PROPTYPE_NONE  => "",

+    PROPTYPE_PSZ   => "VARTYPE_STR",

+    PROPTYPE_SHORT => "VARTYPE_NUMBER",

+    PROPTYPE_INT   => "VARTYPE_NUMBER",

+    PROPTYPE_LONG  => "VARTYPE_NUMBER",

+    PROPTYPE_INT64 => "VARTYPE_NUMBER",

+    PROPTYPE_CSTR  => "VARTYPE_STR",

+    #PROPTYPE_SYSLOGTIME => "VARTYPE_SYSLOGTIME",

+  );

+  my $vtype = $p2v{$ptype};

+  unless ($vtype) {

+    die "property type $t is not supported!\n";

+  }

+  return $vm{$vtype};

+}

+sub serialize {

+  my $self = shift;

+  # runtime/obj.c: objSerializeHeader()

+  my $x = COOKIE_OBJLINE();

+  $x .= join(":", $self->type(), $self->cver(), $self->id(), $self->version());

+  $x .= ":\n";

+  for ( values %{$self->{props}} ) {

+    # runtime/obj.c: SerializeProp()

+    $x .= COOKIE_PROPLINE();

+    $x .= join(":",

+            $_->{name},

+            VARTYPE($_->{type}),

+            length($_->{value}),

+            $_->{value});

+    $x .= ":\n";

+  }

+  # runtime/obj.c: EndSerialize()

+  $x .= COOKIE_ENDLINE() . "End\n";

+  $x .= COOKIE_BLANKLINE() . "\n";

+}

+# constructor: new(id,version)

+sub new {

+  my ($class, $id, $version) = @_;

+  $class = ref $class if ref $class;

+  bless {

+    id => $id,

+    version => $version,

+    props => {},

+  }, $class;

+}

+sub id {

+  my $self = shift;

+  if (@_) {

+    my $x = $self->{id};

+    $self->{id} = shift;

+    return $x;

+  }

+  return $self->{id};

+}

+sub version {

+  my $self = shift;

+  if (@_) {

+    my $x = $self->{version};

+    $self->{version} = shift;

+    return $x;

+  }

+  return $self->{version};

+}

+# property(name, type, value)

+sub property {

+  my $self = shift;

+  my $name = shift;

+  if (@_) {

+    my $x = $self->{props}{$name};

+    $self->{props}{$name}{name} = $name;

+    $self->{props}{$name}{type} = shift;

+    $self->{props}{$name}{value} = shift;

+    return $x;

+  }

+  return $self->{props}{$name};

+}

+1;

+package Rsyslog::OPB;

+use base qw(Rsyslog::Serializable);

+sub type { 'OPB' }

+sub cver { 1 }

+sub new { shift->SUPER::new(@_) }

+1;

+package Rsyslog::Obj;

+use base qw(Rsyslog::Serializable);

+sub type { 'Obj' }

+sub cver { 1 }

+sub new { shift->SUPER::new(@_) }

+1;