summarize_logs.pl 2.95 KB
Newer Older
1
#!/usr/bin/env perl
2 3 4

# Copyright 2012 Johns Hopkins University (Author: Daniel Povey).  Apache 2.0.

5
#scalar(@ARGV) >= 1 && print STDERR "Usage: summarize_warnings.pl <log-dir>\n" && exit 1;
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

sub split_hundreds { # split list of filenames into groups of 100.
  my $names = shift @_;
  my @A = split(" ", $names);
  my @ans = ();
  while (@A > 0) {
    my $group = "";
    for ($x = 0; $x < 100 && @A>0; $x++) {
      $fname = pop @A;
      $group .= "$fname ";
    }
    push @ans, $group;
  }
  return @ans;
}

22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
sub parse_accounting_entry {
  $entry= shift @_;

  @elems = split " ", $entry;
  
  $time=undef;
  $threads=undef;
  foreach $elem (@elems) {
    if ( $elem=~ m/time=(\d+)/ ) {
      $elem =~ s/time=(\d+)/$1/;
      $time = $elem;
    } elsif ( $elem=~ m/threads=(\d+)/ ) {
      $elem =~ s/threads=(\d+)/$1/g;
      $threads = $elem;
    } else {
      die "Unknown entry \"$elem\" when parsing \"$entry\" \n";
    }
  }

  if (defined($time) and defined($threads) ) {
    return ($time, $threads);
  } else {
    die "The accounting entry \"$entry\" did not contain all necessary attributes";
  }
}

foreach $dir (@ARGV) {

  #$dir = $ARGV[0];
  print $dir

  ! -d $dir && print STDERR "summarize_warnings.pl: no such directory $dir\n" ;

  $dir =~ s:/$::; # Remove trailing slash.


  # Group the files into categories where all have the same base-name.
  foreach $f (glob ("$dir/*.log")) {
    $f_category = $f;
    # do next expression twice; s///g doesn't work as they overlap.
    $f_category =~ s:\.\d+\.(?!\d+):.*.:;
    #$f_category =~ s:\.\d+\.:.*.:;
    $fmap{$f_category} .= " $f";
  }
}

foreach $c (sort (keys %fmap) ) {
69 70 71 72 73 74 75 76
  $n = 0;
  foreach $fgroup (split_hundreds($fmap{$c})) {
    $n += `grep -w WARNING $fgroup | wc -l`;
  }
  if ($n != 0) {
    print "$n warnings in $c\n"
  }
}
77
foreach $c (sort (keys %fmap)) {
78 79 80 81 82 83 84 85
  $n = 0;
  foreach $fgroup (split_hundreds($fmap{$c})) {
    $n += `grep -w ERROR $fgroup | wc -l`;
  }
  if ($n != 0) {
    print "$n errors in $c\n"
  }
}
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109

$supertotal_cpu_time=0.0;
$supertotal_clock_time=0.0;
$supertotal_threads=0.0;

foreach $c (sort (keys %fmap)) {
  $n = 0;

  $total_cpu_time=0.0;
  $total_clock_time=0.0;
  $total_threads=0.0;
  foreach $fgroup (split_hundreds($fmap{$c})) {
    $lines=`grep -a "# Accounting: " $fgroup |sed 's/.* Accounting: *//g'`;
    
    #print $lines ."\n";

    @entries = split "\n", $lines;

    foreach $line (@entries) {
      $time, $threads = parse_accounting_entry($line);

      $total_cpu_time += $time * $threads;
      $total_threads += $threads;
      if ( $time > $total_clock_time ) {
110
        $total_clock_time = $time;
111 112 113 114 115 116 117 118 119 120 121
      }
    }
  }
  print "total_cpu_time=$total_cpu_time clock_time=$total_clock_time total_threads=$total_threads group=$c\n";

  $supertotal_cpu_time += $total_cpu_time;
  $supertotal_clock_time += $total_clock_time;
  $supertotal_threads += $total_threads;
}
print "total_cpu_time=$supertotal_cpu_time clock_time=$supertotal_clock_time total_threads=$supertotal_threads group=all\n";