Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1018 insertions, 0 deletions

diff --git a/utils/rspamd_stats.pl b/utils/rspamd_stats.pl
new file mode 100755
index 0000000..9c5f2ac
--- /dev/null
+++ b/utils/rspamd_stats.pl
@@ -0,0 +1,1018 @@
+#!/usr/bin/env perl
+
+use 5.010;
+use Data::Dumper;
+use Getopt::Long;
+use Pod::Usage;
+use Time::Local;
+use IO::Handle;
+use warnings;
+use strict;
+
+my @symbols_search;
+my @symbols_exclude;
+my @symbols_bidirectional;
+my @symbols_groups;
+my @symbols_ignored;
+my %symbols_mult;
+my %groups;
+my $reject_score   = 15.0;
+my $junk_score     = 6.0;
+my $diff_alpha     = 0.1;
+my $correlations   = 0;
+my $nrelated       = 10;
+my $log_file       = "";
+my $search_pattern = "";
+my $startTime      = "";
+my $endTime;
+my $num_logs;
+my $exclude_logs = 0;
+my $man          = 0;
+my $json         = 0;
+my $help         = 0;
+
+# Associate file extensions with decompressors
+my %decompressor = (
+    'bz2' => 'bzip2 -cd',
+    'gz'  => 'gzip -cd',
+    'xz'  => 'xz -cd',
+    'zst' => 'zstd -cd',
+);
+
+GetOptions(
+    "reject-score|r=f"      => \$reject_score,
+    "junk-score|j=f"        => \$junk_score,
+    "symbol|s=s@"           => \@symbols_search,
+    "symbol-bidir|S=s@"     => \@symbols_bidirectional,
+    "exclude|X=s@"          => \@symbols_exclude,
+    "ignore=s@"             => \@symbols_ignored,
+    "group|g=s@"            => \@symbols_groups,
+    "log|l=s"               => \$log_file,
+    "mult=s"                => \%symbols_mult,
+    "alpha-score|alpha|a=f" => \$diff_alpha,
+    "correlations|c"        => \$correlations,
+    "nrelated=i"            => \$nrelated,
+    "search-pattern=s"      => \$search_pattern,
+    "start=s"               => \$startTime,
+    "end=s"                 => \$endTime,
+    "num-logs|n=i"          => \$num_logs,
+    "exclude-logs|x=i"      => \$exclude_logs,
+    "json|j"                => \$json,
+    "help|?"                => \$help,
+    "man"                   => \$man
+) or pod2usage(2);
+
+pod2usage(1) if $help;
+pod2usage( -exitval => 0, -verbose => 2 ) if $man;
+
+# Global vars
+my $total           = 0;
+my $total_spam      = 0;
+my $total_junk      = 0;
+my $junk_symbols    = 0;
+my $spam_symbols    = 0;
+my $ham_symbols     = 0;
+my $ham_spam_change = 0;
+my $ham_junk_change = 0;
+my %sym_res;
+my $rspamd_log;
+my $enabled             = 0;
+my $log_file_num        = 1;
+my $spinner_update_time = 0;
+
+my %action;
+my %timeStamp;
+my %scanTime = (
+    max   => 0,
+    total => 0,
+);
+my %bidir_match;
+
+foreach ( $startTime, $endTime ) { $_ = &normalized_time($_) }
+
+# Convert bidirectional symbols
+foreach my $s (@symbols_bidirectional) {
+    $bidir_match{$s} = {
+        spam => "${s}_SPAM",
+        ham  => "${s}_HAM",
+    };
+    push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+}
+
+# Deal with groups
+my $group_id = 0;
+foreach my $g (@symbols_groups) {
+    my @symbols    = split /,/, $g;
+    my $group_name = "group$group_id";
+
+    foreach my $s (@symbols) {
+        $groups{$s} = $group_name;
+        push @symbols_search, $s unless grep /^$s$/, @symbols_search;
+    }
+}
+
+@symbols_search = '.*'
+  unless @symbols_search;
+
+if ( $log_file eq '-' || $log_file eq '' ) {
+    $rspamd_log = \*STDIN;
+    &ProcessLog();
+}
+elsif ( -d "$log_file" ) {
+    my $log_dir = "$log_file";
+
+    my @logs = &GetLogfilesList($log_dir);
+
+    # Process logs
+    foreach (@logs) {
+        my $ext = (/[^.]+\.?([^.]*?)$/)[0];
+        my $dc  = $decompressor{$ext} || 'cat';
+
+        open( $rspamd_log, "-|", "$dc $log_dir/$_" )
+          or die "cannot execute $dc $log_dir/$_ : $!";
+
+        printf { interactive(*STDERR) } "\033[J  Parsing log files: [%d/%d] %s\033[G", $log_file_num++, scalar @logs,
+          $_;
+        $spinner_update_time = 0;    # Force spinner update
+        &spinner;
+
+        &ProcessLog;
+
+        close($rspamd_log)
+          or warn "cannot close $dc $log_dir/$_: $!";
+    }
+    print { interactive(*STDERR) } "\033[J\033[G";    # Progress indicator clean-up
+}
+else {
+    my $ext = ( $log_file =~ /[^.]+\.?([^.]*?)$/ )[0];
+    my $dc  = $decompressor{$ext} || 'cat';
+    open( $rspamd_log, "-|", "$dc $log_file" )
+      or die "cannot execute $dc $log_file : $!";
+    $spinner_update_time = 0;                         # Force spinner update
+    &spinner;
+    &ProcessLog();
+}
+
+my $total_ham = $total - ( $total_spam + $total_junk );
+
+if ($json) {
+    print "{";
+    &Summary();
+    print '"symbols":{';
+    &SymbolsStat();
+    print "}}\n";
+}
+else {
+    &SymbolsStat();
+    &Summary();
+}
+
+exit;
+
+sub IsIgnored {
+    my ($sym) = @_;
+
+    foreach my $ex (@symbols_ignored) {
+        if ( $sym =~ /^$ex$/ ) {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+sub GenRelated {
+    my ( $htb, $target_sym ) = @_;
+
+    my @result;
+    my $i = 0;
+    foreach my $sym ( sort { $htb->{$b} <=> $htb->{$a} } keys %{$htb} ) {
+        if ( $sym ne $target_sym ) {
+            my @elt = ( $sym, $htb->{$sym} );
+            push @result, \@elt;
+            $i++;
+        }
+
+        last if $i > $nrelated;
+    }
+
+    return \@result;
+}
+
+sub StringifyRelated {
+    my ( $ar, $total ) = @_;
+    return
+      join( "\n", ( map { sprintf "\t%s(%s: %.1f%%)", $_->[0], $_->[1], $_->[1] / ( $total * 1.0 ) * 100.0 } @{$ar} ) );
+}
+
+sub SymbolsStat {
+    if ( $total > 0 ) {
+        my $has_comma = 0;
+        while ( my ( $s, $r ) = each(%sym_res) ) {
+            if ( $r->{hits} > 0 ) {
+                my $th = $r->{hits};
+                my $sh = $r->{spam_hits};
+                my $jh = $r->{junk_hits};
+                my $hh = $r->{hits} - $sh - $jh;
+                my ( $htp, $stp, $jtp );
+                $htp = $hh * 100.0 / $total_ham  if $total_ham != 0;
+                $stp = $sh * 100.0 / $total_spam if $total_spam != 0;
+                $jtp = $jh * 100.0 / $total_junk if $total_junk != 0;
+
+                if ($json) {
+                    if ($has_comma) {
+                        print ",";
+                    }
+                    else {
+                        $has_comma = 1;
+                    }
+                    print "\"$s\":{";
+                    JsonObjectElt( "avg_weight", $r->{'weight'}, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "hits", $th, "%d" );
+                    print ",";
+                    JsonObjectElt( "hits_percentage", $th / $total, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "spam_hits", $sh, "%d" );
+                    print ",";
+                    JsonObjectElt( "spam_to_total", $sh / $th, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "spam_percentage", $stp / 100.0 || 0, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "ham_hits", $hh, "%d" );
+                    print ",";
+                    JsonObjectElt( "ham_to_total", $hh / $th, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "ham_percentage", $htp / 100.0 || 0, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "junk_hits", $jh, "%d" );
+                    print ",";
+                    JsonObjectElt( "junk_to_total", $jh / $th, "%.4f" );
+                    print ",";
+                    JsonObjectElt( "junk_percentage", $jtp / 100.0 || 0, "%.4f" );
+                }
+                else {
+                    printf "%s   avg. weight %.3f, hits %d(%.3f%%):
+  Ham  %7.3f%%, %6d/%-6d (%7.3f%%)
+  Spam %7.3f%%, %6d/%-6d (%7.3f%%)
+  Junk %7.3f%%, %6d/%-6d (%7.3f%%)
+", $s, $r->{weight} / $r->{hits}, $th, ( $th / $total * 100 ),
+                      ( $hh / $th * 100 ), $hh, $total_ham,  ( $htp or 0 ),
+                      ( $sh / $th * 100 ), $sh, $total_spam, ( $stp or 0 ),
+                      ( $jh / $th * 100 ), $jh, $total_junk, ( $jtp or 0 );
+                }
+                my ( $schp, $jchp );
+                $schp = $r->{spam_change} / $total_spam * 100.0 if $total_spam;
+                $jchp = $r->{junk_change} / $total_junk * 100.0 if $total_junk;
+
+                if ( $r->{weight} != 0 ) {
+                    if ( !$json ) {
+                        if ( $r->{weight} > 0 ) {
+                            printf "
+Spam changes (ham/junk -> spam): %6d/%-6d (%7.3f%%)
+Spam  changes / total spam hits: %6d/%-6d (%7.3f%%)
+Junk changes      (ham -> junk): %6d/%-6d (%7.3f%%)
+Junk  changes / total junk hits: %6d/%-6d (%7.3f%%)
+",
+                              $r->{spam_change}, $th,         ( $r->{spam_change} / $th * 100 ),
+                              $r->{spam_change}, $total_spam, ( $schp or 0 ),
+                              $r->{junk_change}, $th,         ( $r->{junk_change} / $th * 100 ),
+                              $r->{junk_change}, $total_junk, ( $jchp or 0 );
+                        }
+                        else {
+                            printf "
+Spam changes (spam -> junk/ham): %6d/%-6d (%7.3f%%)
+Spam changes / total spam hits : %6d/%-6d (%7.3f%%)
+Junk changes (junk -> ham)     : %6d/%-6d (%7.3f%%)
+Junk changes / total junk hits : %6d/%-6d (%7.3f%%)
+",
+                              $r->{spam_change}, $th,         ( $r->{spam_change} / $th * 100 ),
+                              $r->{spam_change}, $total_spam, ( $schp or 0 ),
+                              $r->{junk_change}, $th,         ( $r->{junk_change} / $th * 100 ),
+                              $r->{junk_change}, $total_junk, ( $jchp or 0 );
+                        }
+                    }
+                    else {
+                        print ",";
+                        JsonObjectElt( "spam_change", $r->{spam_change}, "%.4f" );
+                        print ",";
+                        JsonObjectElt( "junk_change", $r->{junk_change}, "%.4f" );
+                    }
+                }
+
+                if ($correlations) {
+
+                    my $spam_related = GenRelated( $r->{symbols_met_spam}, $s );
+                    my $junk_related = GenRelated( $r->{symbols_met_junk}, $s );
+                    my $ham_related  = GenRelated( $r->{symbols_met_ham},  $s );
+
+                    if ( !$json ) {
+                        print "Correlations report:\n";
+
+                        while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+                            my $corr_prob   = $r->{'hits'} / $total;
+                            my $merged_hits = 0;
+                            if ( $r->{symbols_met_spam}->{$cs} ) {
+                                $merged_hits += $r->{symbols_met_spam}->{$cs};
+                            }
+                            if ( $r->{symbols_met_junk}->{$cs} ) {
+                                $merged_hits += $r->{symbols_met_junk}->{$cs};
+                            }
+                            if ( $r->{symbols_met_ham}->{$cs} ) {
+                                $merged_hits += $r->{symbols_met_ham}->{$cs};
+                            }
+
+                            if ( $merged_hits > 0 ) {
+                                printf "Probability of %s when %s fires: %.3f\n", $cs, $s,
+                                  ( ( $merged_hits / $total ) / $corr_prob );
+                            }
+                        }
+
+                        print "Related symbols report:\n";
+                        printf "Top related in spam:\n %s\n", StringifyRelated( $spam_related, $r->{spam_hits} );
+                        printf "Top related in junk:\n %s\n", StringifyRelated( $junk_related, $r->{junk_hits} );
+                        printf "Top related in ham:\n %s\n",
+                          StringifyRelated( $ham_related, $r->{hits} - $r->{spam_hits} - $r->{junk_hits} );
+                    }
+                    else {
+                        print ",";
+                        print "\"correllations\":{";
+
+                        my $has_comma_ = 0;
+                        while ( my ( $cs, $hits ) = each %{ $r->{corr} } ) {
+                            if ($has_comma_) {
+                                print ",";
+                            }
+                            else {
+                                $has_comma_ = 1;
+                            }
+                            my $corr_prob = $hits / $total;
+                            my $sym_prob  = $r->{hits} / $total;
+                            JsonObjectElt( $cs, ( $corr_prob / $sym_prob ), "%.4f" );
+                        }
+
+                        print "}";
+                    }
+                }
+
+                print "}" if $json;
+            }
+            else {
+                print "Symbol $s has not been met\n" if !$json;
+            }
+
+            print '-' x 80 . "\n" if !$json;
+        }
+    }
+}
+
+sub Summary() {
+    if ( !$json ) {
+        print "
+=== Summary ", '=' x 68, "
+Messages scanned: $total";
+        printf " [ %s / %s ]
+", $timeStamp{'start'}, $timeStamp{'end'}
+          if defined $timeStamp{'start'};
+        say '';
+        printf "%11s: %6.2f%%, %d\n", $_, 100 * $action{$_} / $total, $action{$_} for sort keys %action;
+        say '';
+        printf "scan time min/avg/max = %.2f/%.2f/%.2f s
+", $scanTime{'min'} / 1000, ($total) ? $scanTime{'total'} / $total / 1000 : undef, $scanTime{'max'} / 1000
+          if exists $scanTime{'min'};
+        say '=' x 80;
+    }
+    else {
+        JsonObjectElt( "total", $total, "%d" );
+        print ",";
+
+        if ( defined $timeStamp{'start'} ) {
+            JsonObjectElt( "start", $timeStamp{'start'} );
+            print ",";
+        }
+
+        if ( defined $timeStamp{'end'} ) {
+            JsonObjectElt( "end", $timeStamp{'end'} );
+            print ",";
+        }
+
+        print "\"actions\":{";
+
+        my $has_comma = 0;
+        foreach my $a ( sort keys %action ) {
+            if ($has_comma) {
+                print ",";
+            }
+            else {
+                $has_comma = 1;
+            }
+            JsonObjectElt( $a, $action{$a}, "%d" );
+        }
+        print "},";
+    }
+}
+
+sub ProcessRelated {
+    my ( $symbols, $target, $source ) = @_;
+
+    foreach my $s ( @{$symbols} ) {
+        $s =~ /^([^\(]+)(\(([^\)]+)\))?/;
+        my $sym_name  = $1;
+        my $sym_score = 0;
+
+        if ( $groups{$sym_name} ) {
+            $sym_name = $groups{$sym_name};
+        }
+
+        next if ( $source eq $sym_name );
+
+        next if IsIgnored($sym_name);
+
+        if ($2) {
+            $sym_score = $3 * ($symbols_mult{$sym_name} or 1.0);
+
+            if ( abs($sym_score) < $diff_alpha ) {
+                next;
+            }
+
+            my $bm = $bidir_match{$sym_name};
+            if ($bm) {
+                if ( $sym_score >= 0 ) {
+                    $sym_name = $bm->{'spam'};
+                }
+                else {
+                    $sym_name = $bm->{'ham'};
+                }
+            }
+        }
+
+        if ( exists( $target->{$sym_name} ) ) {
+            $target->{$sym_name}++;
+        }
+        else {
+            $target->{$sym_name} = 1;
+        }
+    }
+}
+
+sub ProcessLog {
+    my ( $ts_format, @line ) = &log_time_format($rspamd_log);
+
+    while () {
+        last if eof $rspamd_log;
+        $_ = (@line) ? shift @line : <$rspamd_log>;
+
+        if ( !$enabled && ( $search_pattern eq "" || /$search_pattern/ ) ) {
+            $enabled = 1;
+        }
+
+        next if !$enabled;
+
+        if (/^.*rspamd_task_write_log.*$/) {
+            &spinner;
+            my $ts;
+            if ( $ts_format eq 'syslog' ) {
+                $ts = syslog2iso( join ' ', ( split /\s+/ )[ 0 .. 2 ] );
+            }
+            elsif ( $ts_format eq 'syslog5424' ) {
+                /^([0-9-]+)T([0-9:]+)/;
+                $ts = "$1 $2";
+            }
+            else {
+                $ts = join ' ', ( split /\s+/ )[ 0 .. 1 ];
+            }
+
+            next if ( $ts lt $startTime );
+            next if ( defined $endTime && $ts gt $endTime );
+
+            if ( $_ !~
+                /\(([^()]+)\): \[(NaN|-?\d+(?:\.\d+)?)\/(-?\d+(?:\.\d+)?)\]\s+\[([^\]]+)\].+? time: (\d+\.\d+)ms/ )
+            {
+                #print "BAD: $_\n";
+                next;
+            }
+
+            my @symbols   = split /(?:\{[^}]*\})?(?:$|,)/, $4;
+            my $scan_time = $5;
+            my $act       = $1;
+            my $score     = $2 * 1.0;
+            my $skip      = 0;
+
+            foreach my $ex (@symbols_exclude) {
+                my @found = grep { /^$ex/ } @symbols;
+
+                if ( scalar(@found) > 0 ) {
+                    $skip = 1;
+                    last;
+                }
+            }
+
+            next if ( $skip != 0 );
+
+            if ( defined( $timeStamp{'end'} ) ) {
+                $timeStamp{'end'} = $ts if ( $ts gt $timeStamp{'end'} );
+            }
+            else {
+                $timeStamp{'end'} = $ts;
+            }
+
+            if ( defined( $timeStamp{'start'} ) ) {
+                $timeStamp{'start'} = $ts if ( $ts lt $timeStamp{'start'} );
+            }
+            else {
+                $timeStamp{'start'} = $ts;
+            }
+
+            $scanTime{'min'} = $scan_time if ( !exists $scanTime{'min'} || $scanTime{'min'} > $scan_time );
+            $scanTime{'max'} = $scan_time if ( $scanTime{'max'} < $scan_time );
+            $scanTime{'total'} += $scan_time;
+
+            $action{$act}++;
+            $total++;
+
+            if ( $score >= $reject_score ) {
+                $total_spam++;
+            }
+            elsif ( $score >= $junk_score ) {
+                $total_junk++;
+            }
+
+            my @sym_names;
+
+            foreach my $s (@symbols_search) {
+                my @selected = grep /$s/, @symbols;
+
+                if ( scalar(@selected) > 0 ) {
+
+                    foreach my $sym (@selected) {
+                        $sym =~ /^([^\(]+)(\(([^\)]+)\))?/;
+                        my $sym_name  = $1;
+                        my $sym_score = 0;
+                        my $orig_name = $sym_name;
+
+                        if ($2) {
+                            $sym_score = $3 * ($symbols_mult{$sym_name} or 1.0);
+
+                            if ( abs($sym_score) < $diff_alpha ) {
+                                next;
+                            }
+
+                            my $bm = $bidir_match{$sym_name};
+                            if ($bm) {
+                                if ( $sym_score >= 0 ) {
+                                    $sym_name = $bm->{'spam'};
+                                }
+                                else {
+                                    $sym_name = $bm->{'ham'};
+                                }
+                            }
+                        }
+
+                        next if $orig_name !~ /^$s/;
+
+                        if ( $groups{$s} ) {
+
+                            # Replace with group
+                            $sym_name = $groups{$s};
+                        }
+
+                        push @sym_names, $sym_name;
+
+                        if ( !$sym_res{$sym_name} ) {
+                            $sym_res{$sym_name} = {
+                                hits             => 0,
+                                spam_hits        => 0,
+                                junk_hits        => 0,
+                                spam_change      => 0,
+                                junk_change      => 0,
+                                weight           => 0,
+                                corr             => {},
+                                symbols_met_spam => {},
+                                symbols_met_ham  => {},
+                                symbols_met_junk => {},
+                            };
+                        }
+
+                        my $r = $sym_res{$sym_name};
+
+                        $r->{hits}++;
+                        $r->{weight} += $sym_score;
+                        my $is_spam = 0;
+                        my $is_junk = 0;
+
+                        if ( $score >= $reject_score ) {
+                            $is_spam = 1;
+                            $r->{spam_hits}++;
+                            if ($correlations) {
+                                ProcessRelated( \@symbols, $r->{symbols_met_spam}, $sym_name );
+                            }
+                        }
+                        elsif ( $score >= $junk_score ) {
+                            $is_junk = 1;
+                            $r->{junk_hits}++;
+                            if ($correlations) {
+                                ProcessRelated( \@symbols, $r->{symbols_met_junk}, $sym_name );
+                            }
+                        }
+                        else {
+                            if ($correlations) {
+                                ProcessRelated( \@symbols, $r->{symbols_met_ham}, $sym_name );
+                            }
+                        }
+
+                        if ( $sym_score != 0 ) {
+                            my $score_without = $score - $sym_score;
+
+                            if ( $sym_score > 0 ) {
+                                if ( $is_spam && $score_without < $reject_score ) {
+                                    $r->{spam_change}++;
+                                }
+                                if ( $is_junk && $score_without < $junk_score ) {
+                                    $r->{junk_change}++;
+                                }
+                            }
+                            else {
+                                if ( !$is_spam && $score_without >= $reject_score ) {
+                                    $r->{spam_change}++;
+                                }
+                                if ( !$is_junk && $score_without >= $junk_score ) {
+                                    $r->{junk_change}++;
+                                }
+                            }
+                        }
+                    }    # End foreach symbols selected
+                }
+            }
+
+            if ($correlations) {
+                foreach my $sym (@sym_names) {
+                    next if IsIgnored($sym);
+                    my $r = $sym_res{$sym};
+
+                    foreach my $corr_sym (@sym_names) {
+                        if ( $corr_sym ne $sym ) {
+                            if ( $r->{'corr'}->{$corr_sym} ) {
+                                $r->{'corr'}->{$corr_sym}++;
+                            }
+                            else {
+                                $r->{'corr'}->{$corr_sym} = 1;
+                            }
+                        }
+                    }
+                }    # End of correlations check
+            }
+        }
+    }
+}
+
+sub JsonObjectElt() {
+    my ( $k, $v ) = @_;
+    my $f = defined $_[2] ? $_[2] : '%s';
+
+    if ( $f eq "%s" ) {
+        $f = "\"%s\"";
+    }
+
+    printf "\"%s\":$f", $k, $v;
+}
+
+sub GetLogfilesList {
+    my ($dir) = @_;
+    opendir( my $fh, $dir ) or die $!;
+
+    my $pattern = join( '|', keys %decompressor );
+    my $re      = qr/\.[0-9]+(?:\.(?:$pattern))?/;
+
+    # Add unnumbered logs first
+    my @logs =
+      grep { -f "$dir/$_" && !/$re/ } readdir($fh);
+
+    # Add numbered logs
+    rewinddir($fh);
+    push( @logs, ( sort numeric ( grep { -f "$dir/$_" && /$re/ } readdir($fh) ) ) );
+
+    closedir($fh);
+
+    # Select required logs and revers their order
+    @logs =
+      reverse splice( @logs, $exclude_logs, $num_logs ||= @logs - $exclude_logs );
+
+    # Loop through array printing out filenames
+    print { interactive(*STDERR) } "\nLog files to process:\n";
+    foreach my $file (@logs) {
+        print { interactive(*STDERR) } "  $file\n";
+    }
+    print { interactive(*STDERR) } "\n";
+
+    return @logs;
+}
+
+sub log_time_format {
+    my $fh = shift;
+    my ( $format, $line );
+    while (<$fh>) {
+        $line = $_;
+
+        # 2017-08-08 00:00:01 #66984(
+        # 2017-08-08 00:00:01.001 #66984(
+        if (/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d{3,5})? #\d+\(/) {
+            $format = 'rspamd';
+            last;
+        }
+
+        # Aug  8 00:02:50 #66986(
+        elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d #\d+\(/) {
+            $format = 'syslog';
+            last;
+        }
+
+        # Aug  8 00:02:50 hostname rspamd[66986]
+        elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d \S+ rspamd\[\d+\]/) {
+            $format = 'syslog';
+            last;
+        }
+
+        # 2018-04-16T06:25:46.012590+02:00 rspamd rspamd[12968]
+        elsif (/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(Z|[-+]\d{2}:\d{2}) \S+ rspamd\[\d+\]/) {
+            $format = 'syslog5424';
+            last;
+        }
+
+        # Skip newsyslog messages
+        # Aug  8 00:00:00 hostname newsyslog[63284]: logfile turned over
+        elsif (/^\w{3} (?:\s?\d|\d\d) \d\d:\d\d:\d\d\ \S+ newsyslog\[\d+\]: logfile turned over$/) {
+            next;
+        }
+
+        # Skip journalctl messages
+        # -- Logs begin at Mon 2018-01-15 11:16:24 MSK, end at Fri 2018-04-27 09:10:30 MSK. --
+        elsif (
+/^-- Logs begin at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}, end at \w{3} \d{4}-\d\d-\d\d \d\d:\d\d:\d\d [A-Z]{3}\. --$/
+          )
+        {
+            next;
+        }
+        else {
+            print "Unknown log format\n";
+            exit 1;
+        }
+    }
+    return ( $format, $line );
+}
+
+sub normalized_time {
+    return
+      if !defined( $_ = shift );
+
+    /^\d\d(?::\d\d){0,2}$/
+      ? sprintf '%04d-%02d-%02d %s', 1900 + (localtime)[5], 1 + (localtime)[4], (localtime)[3], $_
+      : $_;
+}
+
+sub numeric {
+    $a =~ /\.(\d+)\./;
+    my $a_num = $1;
+    $b =~ /\.(\d+)\./;
+    my $b_num = $1;
+
+    $a_num <=> $b_num;
+}
+
+sub spinner {
+    my @spinner = qw{/ - \ |};
+    return
+      if ( ( time - $spinner_update_time ) < 1 );
+    $spinner_update_time = time;
+    printf { interactive(*STDERR) } "%s\r", $spinner[ $spinner_update_time % @spinner ];
+    select()->flush();
+}
+
+# Convert syslog timestamp to "ISO 8601 like" format
+# using current year as syslog does not record the year (nor the timezone)
+# or the last year if the guessed time is in the future.
+sub syslog2iso {
+    my %month_map;
+    @month_map{qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)} = 0 .. 11;
+
+    my ( $month, @t ) = $_[0] =~ m/^(\w{3}) \s\s? (\d\d?) \s (\d\d):(\d\d):(\d\d)/x;
+    my $epoch =
+      timelocal( ( reverse @t ), $month_map{$month}, 1900 + (localtime)[5] );
+    sprintf '%04d-%02d-%02d %02d:%02d:%02d', 1900 + (localtime)[5] - ( $epoch > time ), $month_map{$month} + 1, @t;
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+sub is_interactive {
+    ## no critic (ProhibitInteractiveTest)
+
+    my ($out_handle) = ( @_, select );    # Default to default output handle
+
+    # Not interactive if output is not to terminal...
+    return 0 if not -t $out_handle;
+
+    # If *ARGV is opened, we're interactive if...
+    if ( tied(*ARGV) or defined( fileno(ARGV) ) ) {    # this is what 'Scalar::Util::openhandle *ARGV' boils down to
+
+        # ...it's currently opened to the magic '-' file
+        return -t *STDIN if defined $ARGV && $ARGV eq '-';
+
+        # ...it's at end-of-file and the next file is the magic '-' file
+        return @ARGV > 0 && $ARGV[0] eq '-' && -t *STDIN if eof *ARGV;
+
+        # ...it's directly attached to the terminal
+        return -t *ARGV;
+    }
+
+    # If *ARGV isn't opened, it will be interactive if *STDIN is attached
+    # to a terminal.
+    else {
+        return -t *STDIN;
+    }
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+local ( *DEV_NULL, *DEV_NULL2 );
+my $dev_null;
+
+BEGIN {
+    pipe *DEV_NULL, *DEV_NULL2
+      or die "Internal error: can't create null filehandle";
+    $dev_null = \*DEV_NULL;
+}
+
+### Imported from IO::Interactive 1.022 Perl module
+sub interactive {
+    my ($out_handle) = ( @_, \*STDOUT );    # Default to STDOUT
+    return &is_interactive ? $out_handle : $dev_null;
+}
+
+__END__
+
+=head1 NAME
+
+rspamd_stats - analyze Rspamd rules by parsing log files
+
+=head1 SYNOPSIS
+
+rspamd_stats [options] [--symbol=SYM1 [--symbol=SYM2...]] [--log file]
+
+ Options:
+   --log=file             log file or directory to read (stdin by default)
+   --reject-score=score   set reject threshold (15 by default)
+   --junk-score=score     set junk score (6.0 by default)
+   --symbol=sym           check specified symbol (perl regexps, '.*' by default)
+   --alpha-score=score    set ignore score for symbols (0.1 by default)
+   --correlations         enable correlations report
+   --nrelated=integer     show that amount of related symbols (10 by default)
+   --search-pattern       do not process input unless the desired pattern is found
+   --start                starting time (oldest) for log parsing
+   --end                  ending time (newest) for log parsing
+   --num-logs=integer     number of recent logfiles to analyze (all files in the directory by default)
+   --exclude-logs=integer number of latest logs to exclude (0 by default)
+   --json                 print json output instead of human readable
+   --help                 brief help message
+   --mult=sym=number      multiply symbol score
+   --man                  full documentation
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--log>
+
+Specifies log file or directory to read data from. If a directory is specified B<rspamd_stats> analyses files in the
+directory including known compressed file types. Number of log files can be limited using B<--num-logs> and
+B<--exclude-logs> options. This assumes that files in the log directory have B<newsyslog(8)>- or B<logrotate(8)>-like
+name format with numeric indexes. Files without indexes (generally it is merely one file) are considered the most
+recent and files with lower indexes are considered newer.
+
+=item B<--reject-score>
+
+Specifies the reject (spam) threshold.
+
+=item B<--junk-score>
+
+Specifies the junk (add header or rewrite subject) threshold.
+
+=item B<--alpha-score>
+
+Specifies the minimum score for a symbol to be considered by this script.
+
+=item B<--symbol>
+
+Add symbol or pattern (pcre format) to analyze.
+
+=item B<--num-logs>
+
+If set, limits number of analyzed logfiles in the directory to the specified value.
+
+=item B<--exclude-logs>
+
+Number of latest logs to exclude (0 by default).
+
+=item B<--correlations>
+
+Additionally print correlation rate for each symbol displayed. This routine calculates merely paired correlations
+between symbols.
+
+=item B<--search-pattern>
+
+Do not process input unless finding the specified regular expression. Useful to skip logs to a certain position.
+
+=item  B<--exclude>
+
+Exclude log lines if certain symbols are fired (e.g. GTUBE). You may specify this option multiple time to skip multiple
+symbols.
+
+=item B<--start>
+
+Select log entries after this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used
+with B<--end> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you
+supply the time.
+
+=item B<--end>
+
+Select log entries before this time. Format: C<YYYY-MM-DD HH:MM:SS> (can be truncated to any desired accuracy). If used
+with B<--start> select entries between B<--start> and B<--end>. The omitted date defaults to the current date if you
+supply the time.
+
+=item B<--mult=symbol=number>
+
+Multiplies score for the named symbol by the provided multiplier.
+
+=item B<--help>
+
+Print a brief help message and exits.
+
+=item B<--man>
+
+Prints the manual page and exits.
+
+=back
+
+=head1 DESCRIPTION
+
+B<rspamd_stats> will read the given log file (or standard input) and provide statistics for the specified symbols:
+
+    Symbol: BAYES_SPAM (weight 3.763) (381985 hits, 26.827%)
+    Ham hits: 184557 (48.315%), total ham: 1095487 (ham with BAYES_SPAM: 16.847%)
+    Spam hits: 15134 (3.962%), total spam: 16688 (spam with BAYES_SPAM: 90.688%)
+    Junk hits: 182294 (47.723%), total junk: 311699 (junk with BAYES_SPAM: 58.484%)
+    Spam changes (ham/junk -> spam): 7026 (1.839%), total percentage (changes / spam hits): 42.102%
+    Junk changes (ham -> junk): 95192 (24.920%), total percentage (changes / junk hits): 30.540%
+
+Where there are the following attributes:
+
+=over 4
+
+=item *
+
+B<Weight>: average score for a symbols
+
+=item *
+
+B<Total hits>: total number of hits and percentage of symbol hits divided by total number of messages
+
+=item *
+
+B<HAM hits>: provides the following information about B<HAM> messages with the specified symbol (from left to right):
+
+=over 4
+
+=item 1.
+
+B<total symbol hits>: number of messages that has this symbol and are B<HAM>
+
+=item 2.
+
+B<ham percentage>: number of symbol hits divided by overall B<HAM> messages count
+
+=item 3.
+
+B<total ham hits>: overall number of B<HAM> messages
+
+=item 4.
+
+B<ham with symbol percentage>: percentage of number of hits with specified symbol in B<HAM> messages divided by total
+number of B<HAM> messages.
+
+=back
+
+=item *
+
+B<SPAM hits>: provides the following information about B<SPAM> messages - same as previous but for B<SPAM> class.
+
+=item *
+
+B<Junk hits>: provides the following information about B<Junk> messages - same as previous but for B<JUNK> class.
+
+=item *
+
+B<Spam changes>: displays data about how much messages switched their class because of the specific symbol weight.
+
+=item *
+
+B<Junk changes>: displays data about how much messages switched their class because of the specific symbol weight.
+
+=back
+
+=cut