Changeset 70

Timestamp:

30.11.2006 18:47:07 (2 years ago)

Author:

jorge

Message:

Helper application configuration is now dynamic (see FuzzyOcr?.cf)
Changed format slightly for scansets and preprocessors (see FuzzyOcr?.preps and FuzzyOcr?.scansets)
Updated FuzzyOcr?.cf to better inform of plugin options.
Logging now shows error/info/debug lines in output to stderr.

Files:

• trunk/devel/FuzzyOcr.cf (modified) (5 diffs)
• trunk/devel/FuzzyOcr.pm (modified) (43 diffs)
• trunk/devel/FuzzyOcr.preps (modified) (1 diff)
• trunk/devel/FuzzyOcr.scansets (modified) (1 diff)
• trunk/devel/FuzzyOcr/Config.pm (modified) (26 diffs)
• trunk/devel/FuzzyOcr/Deanimate.pm (modified) (6 diffs)
• trunk/devel/FuzzyOcr/Hashing.pm (modified) (25 diffs)
• trunk/devel/FuzzyOcr/Logging.pm (modified) (2 diffs)
• trunk/devel/FuzzyOcr/Misc.pm (modified) (7 diffs)
• trunk/devel/FuzzyOcr/Preprocessor.pm (modified) (3 diffs)
• trunk/devel/FuzzyOcr/Scanset.pm (modified) (6 diffs)
• trunk/devel/FuzzyOcr/Scoring.pm (modified) (3 diffs)

trunk/devel/FuzzyOcr.cf

@@ +1 @@
+# Syntax:
+# loadplugin <Plugin_Name> <Location>
+#  <Location> path where Plugin resides.
-loadplugin FuzzyOcr FuzzyOcr.pm
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#focr_log_stderr 1
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-#focr_global_wordlist /etc/mail/spamassassin/FuzzyOcr.words
-#
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-# These parameters can be used to change other detection settings
-# If you leave these commented out, the defaults will be used.
-# Do not use " " around any parameters!
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#focr_bin_gifsicle /usr/bin/gifsicle
-#focr_bin_giffix /usr/bin/giffix
@@ -51 +104 @@
-#focr_bin_gocr /usr/bin/gocr
-#focr_bin_ocrad /usr/bin/ocrad
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-#
-#focr_preprocessor_file /etc/mail/spamassassin/FuzzyOcr.preps
-#focr_scanset_file /etc/mail/spamassassin/FuzzyOcr.scansets
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#focr_autosort_buffer 10
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#focr_max_size_gif 80000
-#focr_max_size_jpeg 100000
@@ -96 +161 @@
-#focr_max_size_bmp 500000
-#focr_max_size_tiff 500000
-
-
+
+
+
-#focr_skip_gif 1
-#focr_skip_jpeg 1
@@ -103 +169 @@
-#focr_skip_bmp 1
-#focr_skip_tiff 1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
-# Number of minimum matches before the rule scores (Default value: 2)
-#focr_counts_required 3
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#focr_enable_image_hashing 3
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-focr_db_max_days 15
-
-
+
+
+
+
+
-#focr_mysql_db FuzzyOcr
-#focr_mysql_hash Hash
@@ -164 +271 @@
-#focr_mysql_user fuzzyocr
-#focr_mysql_pass fuzzyocr
+#focr_mysql_host localhost
-#focr_mysql_port 3306
-#focr_mysql_socket /tmp/mysql.sock
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-#  1 = keep only if error
-#  2 = always keep
+#--
+# Keeping these intermediate files could fill your HDD _very_ fast!
+# Make shure you periodically empty your temp dir (usually: /tmp) or
+# suffer the conscecuences.  You've been warned!!
+#--
-#focr_keep_bad_images 1
-
-
-
-
-
+
+
-# DO NOT REMOVE THIS LINE, IT IS REQUIRED UNDER ALL CIRCUMSTANCES
-focr_end_config

trunk/devel/FuzzyOcr.pm

@@ -20 +20 @@
-use POSIX;
-
- /etc/mail/spamassassin
-
-
+
+
+ errorlog infolog
-use FuzzyOcr::Config qw(kill_pid
-    get_tmpdir
@@ -70 +70 @@
-    if ($conf->{focr_global_timeout}) {
-        my $t = get_timeout();
-debug
+info
-        $t->run(sub {
-            $end = fuzzyocr_do( $self, $conf, $pms );
-        });
-        if ($t->timed_out()) {
-debug
-debug
+info
+info
-            my ($ret, $pid) = kill_pid();
-            if ($ret > 0) {
-debug
+info
-            } elsif ($ret < 0) {
-debug
+info
-            } else {
-debug
+info
-            }
-            return 0;
@@ -90 +90 @@
-        $end = fuzzyocr_do( $self, $conf, $pms );
-    }
-debug
+info
-        sprintf("%.6f",tv_interval($begin, [gettimeofday]))
-        ." sec.");
@@ -103 +103 @@
-
-    if ( $current_score > $score ) {
-debug
+info
-        return 0;
-    }
@@ -109 +109 @@
-    $score = $conf->{focr_autodiable_negative_score} || -100;
-    if ( $current_score < $score ) {
-debug
+info
-        return 0;
-    }
-
-    my $ddb;
-    my $imgdir;
-    my %imgfiles = ();
@@ -123 +122 @@
-
-    debuglog("Starting FuzzyOcr...");
-    debuglog("Attempting to load personal wordlist...");
-    if ($conf->{focr_personal_wordlist} =~ m/^\//) {
+        debuglog("Attempting to load personal wordlist...");
-        read_words( $conf->{focr_personal_wordlist} );
-    } else {
-        my $peruserlist = $main->sed_path($conf->{focr_personal_wordlist});
-        if (-r $peruserlist) {
-            read_words($peruserlist);
-        } else {
-            debuglog("Error getting personal wordlist, skipping...");
-        }
-    }
-
@@ -153 +145 @@
-
-        if ($test == 0) {
-debug
+info
-            next;
-        }
@@ -162 +154 @@
-
-        unless ($imgdir) {
-debug
+error
-            return 0;
-        }
@@ -194 +186 @@
-            $imgfiles{$imgfilename}{ftype} = 1; 
-            ($w,$h) = unpack("vv",substr($pdata,6,4));
-$imgfilename '${h}x${w}'
+[${h}x${w}] $imgfilename
-            $imgfiles{$imgfilename}{width}  = $w;
-            $imgfiles{$imgfilename}{height} = $h;
@@ -204 +196 @@
-                my ($b,$m) = unpack("CC",substr($pdata,$pos,2)); $pos += 2;
-                if ($b != 0xff) {
-debug
+info
-                   $pos = $pdatalen + 1;
-                   last;
@@ -218 +210 @@
-            }
-            if ($pos > $pdatalen) {
-debug
+error
-            } else {
-                ($h,$w) = unpack("nn",substr($pdata,$pos+3,4));
-$imgfilename '${h}x${w}'
+[${h}x${w}] $imgfilename
-                $imgfiles{$imgfilename}{ftype} = 2;
-                $imgfiles{$imgfilename}{height} = $h;
@@ -232 +224 @@
-            $imgfiles{$imgfilename}{width}  = $w;
-            $imgfiles{$imgfilename}{height} = $h;
-$imgfilename '${h}x${w}'
+[${h}x${w}] $imgfilename
-        } elsif ( substr($pdata,0,2) eq "BM" ) {
-            ## BMP File
@@ -239 +231 @@
-            $imgfiles{$imgfilename}{width}  = $w;
-            $imgfiles{$imgfilename}{height} = $h;
-$imgfilename '${h}x${w}'
+[${h}x${w}] $imgfilename
-        } elsif (
-            (substr($pdata,0,4) eq "\x4d\x4d\x00\x2a") or
@@ -250 +242 @@
-                my $add = 2 + ($n * 12);
-                my ($id,$tag,$cnt,$val)  = unpack($worder?"vvVV":"nnNN",substr($pdata,$offset+$add,12));
-tag
-tag
+id
+id
-                last if ($h != 0 and $w != 0);
-            }
-
-
-
+
+
-            $imgfiles{$imgfilename}{ftype}  = 5;
-            $imgfiles{$imgfilename}{width}  = $w ? $w : 1;
@@ -266 +257 @@
-        $imgfiles{$imgfilename}{fsize} = $pdatalen;
-        unless (open PICT, ">$imgfilename") {
-debug
+error
-            delete $imgfiles{$imgfilename};
-            next;
@@ -278 +269 @@
-
-    if ($cnt == 0) {
-debug
+info
-        removedir($imgdir) if (defined($imgdir) and ($conf->{focr_keep_bad_images}<2));
-        return 0;
-    }
-debug
+info
-    if ($conf->{focr_enable_image_hashing} == 3) {
-ddb = $
+
-    }
-
@@ -293 +284 @@
-    foreach my $file (keys %imgfiles) {
-        my $pic = $imgfiles{$file};
-debug
+info
-        my @used_scansets = ();
-        my $corrupt = 0;
@@ -304 +295 @@
-
-        if ( $$pic{ftype} == 1 ) {
-debug
+info
-            if ($conf->{focr_skip_gif}) {
-debug
+info
-                next IMAGE;
-            }
-            if (defined($conf->{focr_max_size_gif}) and ($$pic{fsize} > $conf->{focr_max_size_gif})) {
-debug
+info
-                next;
-            }
@@ -322 +313 @@
-            foreach my $a (qw/gifsicle giftext giffix gifinter giftopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-debug
+error
-                    next IMAGE;
-                }
@@ -336 +327 @@
-            if ($retcode<0) { # only care if we timed out
-                chomp $retcode;
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
@@ -351 +342 @@
-            }
-            if ($interlaced_gif or ($image_count > 1)) {
-debug
+info
-            }
-            else {
-debug
+info
-                $tfile .= "-fixed.gif";
-                printf RAWERR "## $conf->{focr_bin_giffix} $file >$tfile 2>>$efile\n" if ($haserr>0);
@@ -362 +353 @@
-                if ($retcode<0) { # only care if we timed out
-                    chomp $retcode;
-debug
+error
-                    printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-                    ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
@@ -381 +372 @@
-            if ($corrupt) {
-                if ($interlaced_gif or ($image_count > 1)) {
-debug
+info
-                    corrupt_img($conf->{focr_corrupt_unfixable_score}, $corrupt);
-                    next;
-                }
-                if (-z $tfile) {
-debug
+info
-                    corrupt_img($conf->{focr_corrupt_unfixable_score}, $corrupt);
-                    next;
-                }
-debug
+info
-                corrupt_img($conf->{focr_corrupt_score}, $corrupt);
-            }
-
-            if ($image_count > 1) {
-debug
+info
-                $tfile = deanimate($tfile);
-            }
-
-            if ($interlaced_gif) {
-debug
+info
-                my $cfile = $tfile;
-                if ($tfile =~ m/\.gif$/i) {
@@ -414 +405 @@
-                    chomp $retcode;
-                    printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                    ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-                } elsif ($retcode>0) {
-                    chomp $retcode;
-                    printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_gifinter}\n" if ($haserr>0);
-debug
+error
-                    ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-                }
@@ -431 +422 @@
-                chomp $retcode;
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-                printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_giftopnm}\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
-        }
-        elsif ( $$pic{ftype} == 2 ) {
-debug
+info
-            if ($conf->{focr_skip_jpeg}) {
-debug
+info
-                next IMAGE;
-            }
-
-            if (defined($conf->{focr_max_size_jpeg}) and ($$pic{fsize} > $conf->{focr_max_size_jpeg})) {
-debug
+info
-                next;
-            }
@@ -456 +447 @@
-            foreach my $a (qw/jpegtopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-debug
+error
-                    next IMAGE;
-                }
@@ -466 +457 @@
-                chomp $retcode;
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-                printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_jpegtopnm}\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
-        }
-        elsif ( $$pic{ftype} == 3 ) {
-debug
+info
-            if ($conf->{focr_skip_png}) {
-debug
+info
-                next IMAGE;
-            }
-            if (defined($conf->{focr_max_size_png}) and ($$pic{fsize} > $conf->{focr__max_size_png})) {
-debug
+info
-                next;
-            }
@@ -490 +481 @@
-            foreach my $a (qw/pngtopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-debug
+error
-                    next IMAGE;
-                }
@@ -501 +492 @@
-                chomp $retcode;
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-                printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_pngtopnm}\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
-        }
-        elsif ( $$pic{ftype} == 4 ) {
-debug
+info
-            if ($conf->{focr_skip_bmp}) {
-debug
+info
-                next IMAGE;
-            }
-            if (defined($conf->{focr_max_size_bmp}) and ($$pic{fsize} > $conf->{focr_max_size_bmp})) {
-debug
+info
-                next;
-            }
@@ -525 +516 @@
-            foreach my $a (qw/bmptopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-debug
+error
-                    next IMAGE;
-                }
@@ -535 +526 @@
-                chomp $retcode;
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-                printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_bmptopnm}\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
-        }
-        elsif ( $$pic{ftype} == 5 ) {
-debug
+info
-            if ($conf->{focr_skip_tiff}) {
-debug
+info
-                next IMAGE;
-            }
-            if (defined($conf->{focr_max_size_tiff}) and ($$pic{fsize} > $conf->{focr_max_size_tiff})) {
-debug
+info
-                next;
-            }
@@ -560 +551 @@
-            foreach my $a (qw/tifftopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-debug
+error
-                    next IMAGE;
-                }
@@ -570 +561 @@
-                chomp $retcode;
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            } elsif ($retcode>0) {
-                chomp $retcode;
-                printf RAWERR "?? [$retcode] returned from $conf->{focr_bin_tifftopnm}\n" if ($haserr>0);
-debug
+error
-                ++$imgerr if $conf->{focr_keep_bad_images}>0; next;
-            }
-        }
-        else {
-debug
+error
-            next;
-        }
-
-        if($conf->{focr_enable_image_hashing}) {
-debuglog("Calculating the image hash
+infolog("Calculating image hash for
-            ($corrupt, $digest) = calc_image_hash($pfile,$pic);
-            if ($corrupt) {
-debug
+info
-            } else {
-                my ($score, $dinfo, $whash);
@@ -597 +588 @@
-                if ($score > 0) {
-                    known_img_hash($score,$dinfo);
-debug
+info
-                    removedir($imgdir);
-                    return 0;
@@ -606 +597 @@
-                ($score,$dinfo) = check_image_hash_db($digest, $whash, $$pic{fname}, $$pic{ctype}, $$pic{ftype});
-                if ($score > 0) {
-debug
+info
-                    next IMAGE;
-                }
-            }
-            if ($digest eq '') {
-debug
+info
-                next IMAGE;
-            }
-        } else {
-debug
+info
-        }
-        my @ocr_results = ();
@@ -623 +614 @@
-        my %words = %$wref;
-        foreach my $scanset (@$scansets) {
+            my $scanlabel = $scanset->{label};
+            my $scancmd   = $scanset->{command};
+            if ($scancmd} =~ m/^\$/) {
+                infolog("Skipping $scanlabel, invalid command '$scancmd'");
+                next;
+            }
-            my $cmcnt = 0;
-            my @cfound;
-
-
+
+
+
-            printf RAWERR qq(## $scancmd\n) if ($haserr>0);
-            my ($retcode, @result) = $scanset->run($pfile);
-            if ($retcode<0) {
-
-
-
+
+
+