Changeset 60

Timestamp:

27.11.2006 16:53:35 (2 years ago)

Author:

decoder

Message:

New Scanset and Preprocessor System

Works at first sight but still needs some work and lots of testing
Error handling at some points still missing

Files:

• trunk/devel/FuzzyOcr.cf (modified) (1 diff)
• trunk/devel/FuzzyOcr.pm (modified) (4 diffs)
• trunk/devel/FuzzyOcr.preps (added)
• trunk/devel/FuzzyOcr.scansets (added)
• trunk/devel/FuzzyOcr/Config.pm (modified) (7 diffs)
• trunk/devel/FuzzyOcr/Misc.pm (modified) (1 diff)
• trunk/devel/FuzzyOcr/Preprocessor.pm (modified) (3 diffs)
• trunk/devel/FuzzyOcr/Scanset.pm (modified) (4 diffs)

trunk/devel/FuzzyOcr.cf

@@ -56 +56 @@
-############################################################################################
-
-
-
-
-
+
+
-#
-
-
+
-#
- Another one for ocrad only
-s $ocrad -s5 -T 0.5 $pfile
+focr_preprocessor_file /etc/mail/spamassassin/FuzzyOcr.preps
+_file /etc/mail/spamassassin/FuzzyOcr.scansets
-#
-
-
-
-
-
-
-
-
-
+
+
+
-
-##### Various Score/Scan settings #####

trunk/devel/FuzzyOcr.pm

@@ -140 +140 @@
-
-        $imgdir = Mail::SpamAssassin::Util::secure_tmpdir();
-
+
-        set_tmpdir($imgdir);
-
@@ -602 +602 @@
-        my $scansets = get_scansets();
-        foreach my $scanset (@$scansets) {
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-            if ($retcode<0) {
-set
-
-set
+cmd
+\"$scanlabel\" 
+cmd
-                next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-
-
-
-
-
+
+
-                debuglog($errstr);
-                debuglog("Skipping scanset because of errors, trying next...");
@@ -630 +621 @@
-            }
-
-ocrdata
-ocrdata
+result
+result
-            push( @used_scansets, $scanset );
-        }
@@ -654 +645 @@
-                            "Found word \"$w\" in line\n \"$_\" \n with fuzz of "
-                            . sprintf("%0.4f",$matched)
-$used_scansets[$gcnt]
+\"$used_scansets[$gcnt]->{label}\"
-                        );
-                    }

trunk/devel/FuzzyOcr/Config.pm

@@ -1 +1 @@
-use strict;
-package FuzzyOcr::Config;
+
+use FuzzyOcr::Scanset;
+use FuzzyOcr::Preprocessor;
-
-use base 'Exporter';
@@ -13 +16 @@
-    get_timeout
-    get_scansets 
+    get_preprocessor 
-    get_thresholds 
-    get_config 
@@ -39 +43 @@
-our %words = ();
-our @scansets;
+our @preprocessors;
-our $conf;
-our $pms;
@@ -128 +133 @@
-}
-
+sub get_preprocessor {
+    my ($label) = @_;
+    foreach (@preprocessors) {
+        if ($_->{label} eq $label) {
+            return $_;
+        }
+    }
+    return 0;
+}
+
-sub get_thresholds {
-    return \%Threshold;
@@ -352 +367 @@
-
-    push (@cmds, {
-
-
+
+
+
+
+
+
+
-        type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
-        });
@@ -441 +461 @@
-        debuglog(" $k => ".$conf->{$k});
-    }
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    foreach my $scan (@scansets) {
-
-
-
-
+
+
+
-    }
-
@@ -702 +739 @@
-}
-
+sub parse_scansets {
+    my ($file) = @_;
+
+    open(SFILE, "<$file");
+    if ($?) {
+        debuglog("Failed to open scanset file \"$file\", aborting...");
+        return 1;
+    }
+
+    our @slabels;
+    our @scansets;
+    my $scanset;
+
+    while(<SFILE>) {
+        # We are in the middle of a scanset
+        if(defined $scanset) {
+            # Strip comments and ignore blank lines
+            chomp($_);
+            $_ =~ s/(\s)*#(.*)//;
+            unless ($_) {
+                next;
+            }
+            if ($_ =~ /^(\s)*preprocessors(\s)*=(\s)*(.*)$/i) {
+                my $prep = $4;
+                $scanset->{preprocessors} = $prep;
+                $prep =~ s/ //g;
+                my @preps = split(',', $prep);
+                foreach (@preps) {
+                    unless(get_preprocessor($_)) {
+                        debuglog("Unknown preprocessor \"$_\" used in scansets line $., aborting...");
+                        return 1;
+                    }
+                }
+            } elsif ($_ =~ /^(\s)*ocr_command(\s)*=(\s)*(.*)$/i) {
+                my $cmd = $4;
+                if ($cmd =~ /(<|>|\||;)/) {
+                    debuglog("OCR Command may not contain \"< > | ;\", aborting...");
+                    return 1;
+                }
+                $scanset->{ocr_command} = $cmd;
+            } elsif ($_ =~ /^(\s)*force_output_in(\s)*=(\s)*(.*)$/i) {
+                $scanset->{force_output_in} = $4;
+            # Scanset is closing
+            } elsif ($_ =~ /^(\s)*\}/) {
+                unless ($scanset->{ocr_command}) {
+                    my $l = $scanset->{label};
+                    debuglog("Scanset \"$l\" is missing ocr_command line, aborting...");
+                    return 1;
+                }
+                push(@scansets, $scanset);
+                $scanset = undef;
+            } else {
+                debuglog("Unknown token at line $., aborting...");
+                return 1;
+            }
+        # Start a new scanset
+        } elsif ($_ =~ /^(\s)*scanset(\s)+(.+?)(\s)+\{$/i) {
+            if (grep $_ eq $3, @slabels) {
+                debuglog("Error, label already used earlier in line $., aborting...");
+                return 1;
+            }
+            $scanset = FuzzyOcr::Scanset->new($3);
+            push(@slabels, $3);
+        }
+    }
+    close(SFILE);
+    return (0, @scansets);
+}
+
+sub parse_preprocessors {
+    my ($file) = @_;
+
+    open(PFILE, "<$file");
+    if ($?) {
+        debuglog("Failed to open preprocessor file \"$file\", aborting...");
+        return 1;
+    }
+
+    our @plabels;
+    our @preprocessors;
+    my $preprocessor;
+
+    while(<PFILE>) {
+        chomp($_);
+        $_ =~ s/(\s)*#(.*)//;
+        unless ($_) {
+            next;
+        }
+        # We are in the middle of a preprocessor
+        if(defined $preprocessor) {
+            if ($_ =~ /^(\s)*command(\s)*=(\s)*(.*)$/i) {
+                my $cmd = $4;
+                if ($cmd =~ /(<|>|\||;)/) {
+                    debuglog("Preprocessor Command may not contain \"< > | ;\", aborting...");
+                    return 1;
+                }
+                $preprocessor->{command} = $cmd;
+            # Preprocessor is closing
+            } elsif ($_ =~ /^(\s)*\}/) {
+                unless ($preprocessor->{command}) {
+                    my $l = $preprocessor->{label};
+                    debuglog("Preprocessor \"$l\" is missing command line, aborting...");
+                    return 1;
+                }
+                push(@preprocessors, $preprocessor);
+                $preprocessor = undef;
+            } else {
+                debuglog("Unknown token at line $., aborting...");
+                return 1;
+            }
+        # Start a new preprocessor
+        } elsif ($_ =~ /^(\s)*preprocessor(\s)+(.+?)(\s)+\{$/i) {
+            if (grep $_ eq $3, @plabels) {
+                debuglog("Error, label already used earlier in line $., aborting...");
+                return 1;
+            }
+            $preprocessor = FuzzyOcr::Preprocessor->new($3);
+            push(@plabels, $3);
+        }
+    }
+    close(PFILE);
+    return (0, @preprocessors);
+}
+
-1;

trunk/devel/FuzzyOcr/Misc.pm

@@ -6 +6 @@
-
-use lib "../";
-;
+
-use Time::HiRes qw( time usleep ualarm gettimeofday tv_interval );
-

trunk/devel/FuzzyOcr/Preprocessor.pm

@@ -2 +2 @@
-
-use lib "../";
-
-
+
-
-sub new {
-
-
-
-
+
-
-    bless {
@@ -19 +15 @@
-sub run {
-    my ($self, $input) = @_;
-
+FuzzyOcr::Config::
-    my $label = $self->{label};
-    my $output = "$tmpdir/prep.$label.out";
@@ -43 +39 @@
-
-    # Run processor
-
+FuzzyOcr::Misc::
-
-    # Return code

trunk/devel/FuzzyOcr/Scanset.pm

@@ -2 +2 @@
-
-use lib "../";
-
-
+
-
-sub new {
-
-
-
-
-
+
-
-    bless {
-        "label"         => $label,
-        "preprocessors" => $preprocessors,
-
+
+
-    }, $class;
-}
@@ -21 +17 @@
-sub run {
-    my ($self, $input) = @_;
-
+
+
-    my $label = $self->{label};
-    my $output = "$tmpdir/scanset.$label.out";
@@ -32 +29 @@
-    # First, run all preprocessors
-    my $preprocessors = $self->{preprocessors};
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-        }
-        # Input of next processor is output of last
-        $input = "$tmpdir/prep.$label.out";
-    }
-
@@ -60 +59 @@
-    }
-
+    #Replace supported scanner macros by full path
+    $rcmd =~ s/\$gocr/$conf->{focr_bin_gocr}/;
+    $rcmd =~ s/\$ocrad/$conf->{focr_bin_ocrad}/;
+
-    # Run scanner
-
+
+
+
-
+    # Scanset enforces OCR output in file $out_in (for example TesserAct has multiple files as output)
+    if ($out_in) {
+        $retcode = FuzzyOcr::Misc::save_execute($rcmd, $stdin, $stdout, $stderr);
+        open(INFILE, "<$out_in");
+        @result = <INFILE>;
+        close(INFILE);
+    } else {
+        ($retcode, @result) = FuzzyOcr::Misc::save_execute($rcmd, $stdin, $stdout, $stderr, 1);
+    }
+
+    # If there were errors in the scan, return the errors instead of OCR results
+    if ($retcode>0) {
+        $stderr =~ tr/>|</   /;
+        open(INFILE, "<$stderr");
+        @result = <INFILE>;
+        close(INFILE);
+    }
-    # Return scanner results and return code
+
-    return ($retcode, @result);
-}