Changeset 42

Timestamp:

20.11.2006 18:54:51 (2 years ago)

Author:

jorge

Message:

Don't need pamfile helper app any more, since we now determine size, height, width directly from the image header.
Fixed bug in FuzzyOcr?.pm where it did not save the file-type in the @hashes array, resulting in erroneous hash generation.
Make shure that the OCR application is executable when generating the scansets array, and skip a configuration if it cannot find an executable.
Updated default scanset configuration to include ocrad pass.

Files:

• trunk/devel/FuzzyOcr.pm (modified) (29 diffs)
• trunk/devel/FuzzyOcr/Config.pm (modified) (8 diffs)
• trunk/devel/FuzzyOcr/Hashing.pm (modified) (15 diffs)

trunk/devel/FuzzyOcr.pm

@@ -49 +49 @@
-    save_pms($pms);
-
-'focr_autodisable_score'
-'focr_autodisable_score'
+focr_autodisable_score
+focr_autodisable_score
-        return 0;
-    }
@@ -63 +63 @@
-    #debuglog("Starting FuzzyOcr...");
-    #debuglog("Attempting to load personal wordlist...");
-"focr_personal_wordlist"
-"focr_personal_wordlist"
+focr_personal_wordlist
+focr_personal_wordlist
-    } else {
-        my $homedir  = (getpwuid($<))[7];
-        if ($homedir) {
-'focr_personal_wordlist'
+focr_personal_wordlist
-        } elsif (defined($ENV{HOME})) {
-'focr_personal_wordlist'
+focr_personal_wordlist
-        } else {
-            debuglog("Variable \$ENV{HOME} not defined and getpwuid failed, personal wordlist function not available...");
@@ -146 +146 @@
-                if ($b != 0xff) {
-                   debuglog("Invalid JPEG image");
+                   $pos = $pdatalen + 1;
-                   last;
-                }
@@ -161 +162 @@
-            } else {
-                ($h,$w) = unpack("nn",substr($pdata,$pos+3,4));
-}
-debuglog("JPEG: $imgfilename '${h}x${w}'")
-$imgfiles{$imgfilename}{ftype} = 2
-$imgfiles{$imgfilename}{height} = $h
-$imgfiles{$imgfilename}{width}  = $w;
+    debuglog("JPEG: $imgfilename '${h}x${w}'");
+    $imgfiles{$imgfilename}{ftype} = 2
+    $imgfiles{$imgfilename}{height} = $h
+    $imgfiles{$imgfilename}{width}  = $w
+}
-        } elsif ( substr($pdata,0,4) eq "\x89\x50\x4e\x47" ) {
-            # PNG File
@@ -201 +202 @@
-            $imgfiles{$imgfilename}{height} = $h ? $h : 1;
-        }
+        next unless defined $imgfiles{$imgfilename}{ftype};
-        $imgfiles{$imgfilename}{fname} = $fname;
-        $imgfiles{$imgfilename}{ctype} = $ctype;
@@ -218 +220 @@
-    if ($cnt == 0) {
-        debuglog("Skipping OCR, no image files found...");
-"focr_keep_bad_images"
+focr_keep_bad_images
-        return 0;
-    }
-    debuglog("Found: $cnt images"); $cnt = 0;
-"focr_timeout"
+focr_timeout
-    my $retcode;
-
@@ -247 +249 @@
-                next IMAGE;
-            }
-"focr_max_size_gif"}) and ($$pic{fsize} > $conf->{"focr_max_size_gif"
-s
+focr_max_size_gif}) and ($$pic{fsize} > $conf->{focr_max_size_gif
+$pic{fsize}
-                next;
-            }
@@ -268 +270 @@
-
-            $retcode = $t->run_and_catch(sub {
-"focr_bin_giftext"
+focr_bin_giftext
-            });
-            if ($retcode) {
-                chomp $retcode;
-'focr_bin_giftext'
-"focr_keep_bad_images"
+focr_bin_giftext
+focr_keep_bad_images
-            }
-            foreach (@stdout_data) {
@@ -291 +293 @@
-                debuglog("Image is single non-interlaced...");
-                $tfile .= "-fixed.gif";
-'focr_bin_giffix'
+focr_bin_giffix
-                $retcode = $t->run_and_catch(sub {
-"focr_bin_giffix"
+focr_bin_giffix
-                });
-                if ($retcode) {
-                    chomp $retcode;
-'focr_bin_giffix'
+focr_bin_giffix
-                    printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-"focr_keep_bad_images"
+focr_keep_bad_images
-                }
-                if (open ERR, $efile) {
@@ -316 +318 @@
-                if ($interlaced_gif or ($image_count > 1)) {
-                    debuglog("Skipping corrupted interlaced image...");
-'focr_corrupt_unfixable_score'
+focr_corrupt_unfixable_score
-                    next;
-                }
-                if (-z $tfile) {
-                    debuglog("Uncorrectable corruption detected, skipping non-interlaced image...");
-'focr_corrupt_unfixable_score'
+focr_corrupt_unfixable_score
-                    next;
-                }
-                debuglog("Image is corrupt, but seems fixable, continuing...");
-'focr_corrupt_score'
+focr_corrupt_score
-            }
-
@@ -341 +343 @@
-                    $tfile .= ".gif";
-                }
-'focr_bin_gifinter'
+focr_bin_gifinter
-                $retcode = $t->run_and_catch(sub{
-"focr_bin_gifinter"
+focr_bin_gifinter
-                });
-                if ($retcode) {
-                    chomp $retcode;
-                    debuglog("$conf->{'focr_bin_gifinter'}: Timed out [$retcode], skipping...");
-                    printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-
-
-
-
-
+
+
+
+
+
+
-            $retcode = $t->run_and_catch(sub {
-"focr_bin_giftopnm"
+focr_bin_giftopnm
-            });
-            if ($retcode) {
-                chomp $retcode;
-                debuglog("$conf->{'focr_bin_giftopnm'}: Timed out [$retcode], skipping...");
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-
+
+
-            }
-        }
@@ -371 +373 @@
-            }
-
-"focr_max_size_jpeg"}) and ($$pic{fsize} > $conf->{"focr_max_size_jpeg"
-s
+focr_max_size_jpeg}) and ($$pic{fsize} > $conf->{focr_max_size_jpeg
+$pic{fsize}
-                next;
-            }
@@ -384 +386 @@
-                }
-            }
-'focr_bin_jpegtopnm'
+focr_bin_jpegtopnm
-            $retcode = $t->run_and_catch(sub {
-"focr_bin_jpegtopnm"
+focr_bin_jpegtopnm
-            });
-            if ($retcode) {
-                chomp $retcode;
-
-
+
+
+
-            }
-        }
@@ -400 +403 @@
-                next IMAGE;
-            }
-"focr_max_size_png"}) and ($$pic{fsize} > $conf->{"focr__max_size_png"
-s
+focr_max_size_png}) and ($$pic{fsize} > $conf->{focr__max_size_png
+$pic{fsize}
-                next;
-            }
@@ -413 +416 @@
-                }
-            }
-'focr_bin_pngtopnm'
+focr_bin_pngtopnm
-            $retcode = $t->run_and_catch(sub {
-"focr_bin_pngtopnm"
+focr_bin_pngtopnm
-            });
-            if ($retcode) {
-                chomp $retcode;
-                debuglog("$conf->{'focr_bin_pngtopnm'}: Timed out [$retcode], skipping...");
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-
+
+
-            }
-        }
@@ -430 +433 @@
-                next IMAGE;
-            }
-"focr_max_size_bmp"}) and ($$pic{fsize} > $conf->{"focr_max_size_bmp"
-s
+focr_max_size_bmp}) and ($$pic{fsize} > $conf->{focr_max_size_bmp
+$pic{fsize}
-                next;
-            }
@@ -443 +446 @@
-                }
-            }
-'focr_bin_bmptopnm'
+focr_bin_bmptopnm
-            $retcode = $t->run_and_catch(sub {
-                qx($conf->{"focr_bin_bmptopnm"} $file >$pfile 2>>$efile);
@@ -449 +452 @@
-            if ($retcode) {
-                chomp $retcode;
-                debuglog("$conf->{'focr_bin_bmptopnm'}: Timed out [$retcode], skipping...");
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-
+
+
-            }
-        }
@@ -460 +463 @@
-                next IMAGE;
-            }
-"focr_max_size_tiff"}) and ($$pic{fsize} > $conf->{"focr_max_size_tiff"
-s
+focr_max_size_tiff}) and ($$pic{fsize} > $conf->{focr_max_size_tiff
+$pic{fsize}
-                next;
-            }
@@ -468 +471 @@
-            }
-
-
+
+
+
+
+
+
+
-            $retcode = $t->run_and_catch(sub {
-                qx($conf->{'focr_bin_tifftopnm'} $file >$pfile 2>>$efile);
@@ -474 +483 @@
-            if ($retcode) {
-                chomp $retcode;
-                debuglog("$conf->{'focr_bin_tifftopnm'}: Timed out [$retcode], skipping...");
-                printf RAWERR "?? Timed out > $retcode\n" if ($haserr>0);
-
+
+
-            }
-        }
@@ -484 +493 @@
-        }
-
-"focr_enable_image_hashing"
+focr_enable_image_hashing
-            debuglog("Calculating the image hash: $pfile");
-
+,$pic
-            if ($corrupt) {
-                debuglog("Error calculating the image hash, skipping hash check...");
@@ -497 +506 @@
-                if ($score > 0) {
-                    known_img_hash($score,$dinfo);
-
+ if ($conf->{focr_enable_image_hashing} < 3)
-                    removedir($imgdir);
-                    return 0;
@@ -521 +530 @@
-        foreach my $scanset (@$scansets) {
-            my $scan = $scanset;
-"focr_bin_gocr"
-"focr_bin_ocrad"
+focr_bin_gocr
+focr_bin_ocrad
-            $scan =~ s/\$pfile/$pfile/;
-            $scan =~ s/\$efile/$efile/g;
@@ -577 +586 @@
-            $cnt  += $wcnt;
-            $mcnt += $wcnt;
-"focr_verbose"
+focr_verbose
-                push( @found, "\"$w\" in $wcnt lines" );
-            }
-        }
-"focr_enable_image_hashing"
-
+focr_enable_image_hashing
+$pic{ftype},$
-            push(@hashes, $info);
-        }
@@ -589 +598 @@
-    
-    if ($cnt == 0) {
-"focr_enable_image_hashing"
+focr_enable_image_hashing
-            debuglog("Message is ham, saving...");
-            foreach my $h (@hashes) {
@@ -606 +615 @@
-            . join( "\n", @found )
-            . "\n($cnt word occurrences found)" );
-"focr_counts_required"
-'focr_base_score'} + (( $cnt - $conf->{"focr_counts_required"} ) * $conf->{'focr_add_score'
+focr_counts_required
+focr_base_score} + (( $cnt - $conf->{focr_counts_required} ) * $conf->{focr_add_score
-            debuglog("Message is spam, score = $score");
-        } else {
-'focr_add_score'} * $cnt) if $conf->{"focr_score_ham"
+focr_add_score} * $cnt) if $conf->{focr_score_ham
-            debuglog("Message is ham, score = $score");
-        }
-"focr_enable_image_hashing"
-"focr_hashing_learn_scanned"
+focr_enable_image_hashing
+focr_hashing_learn_scanned
-            $score > 0) {
-            foreach my $h (@hashes) {
@@ -625 +634 @@
-            }
-        }
-"focr_verbose"} > 0
+focr_verbose} > 0 and $conf->{focr_verbose} < 3
-            debuglog($debuginfo);
-        }
@@ -634 +643 @@
-            $pms->{conf}->{descriptions}->{FUZZY_OCR} . "\n$debuginfo" );
-    }
-"focr_keep_bad_images"
+focr_keep_bad_images
-        removedir($imgdir);
-    }

trunk/devel/FuzzyOcr/Config.pm

@@ -125 +125 @@
-                return $Mail::SpamAssassin::Conf::INVALID_VALUE;
-            }
-'focr_verbose'
+focr_verbose
-        }
-    });
@@ -151 +151 @@
-                return $Mail::SpamAssassin::Conf::INVALID_VALUE;
-            }
-'focr_enable_image_hashing'
+focr_enable_image_hashing
-        }
-       });
@@ -208 +208 @@
-                return $Mail::SpamAssassin::Conf::INVALID_VALUE;
-            }
-'focr_keep_bad_images'
+focr_keep_bad_images
-        }
-       });
@@ -281 +281 @@
-    push (@cmds, {
-        setting => 'focr_scansets',
-gocr -l 140 -d 2 -i
+ocrad -s5 -T 0.5
-        type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
-        });
@@ -334 +334 @@
-    my ($self, $opts) = @_;
-    $conf = $opts->{conf};
-
-    # Use specified scansets
-    @scansets = split(/,\s*/, $conf->{'focr_scansets'});
-    debuglog("Using scan: $_") foreach @scansets;
-
-    # find external binaries
@@ -360 +356 @@
-        }
-    }
+
+    # Use specified scansets
+    @scansets = split(/,\s*/, $conf->{'focr_scansets'});
+    foreach my $scan (@scansets) {
+        if ($scan =~ m/(gocr|ocrad)/) {
+            next unless -x $conf->{"focr_bin_$1"};
+        }
+        debuglog("Using scan: $scan");
+    }
+
-    if ($conf->{focr_enable_image_hashing} == 2 and -r $conf->{focr_digest_db}) {
-        my %DB; my $dbm; my $err = 0;
-        my $now = time - ($conf->{focr_db_max_days}*86400);
+        debuglog($conf->{focr_db_hash});
-        tie %DB, 'MLDBM', $conf->{focr_db_hash} or $err++;
-        if ($err) {
@@ -571 +578 @@
-    $logtext =~ s/\n/\n                      /g;
-
-'focr_logfile'
-'focr_logfile'
+focr_logfile
+focr_logfile
-    }
-    flock( LOGFILE, LOCK_EX );
@@ -586 +593 @@
-    my $limit = defined $_[1] ? $_[1] : 1;
-    foreach (@lines) { dbg ("FuzzyOcr: $_"); }
-"focr_verbose"
-"focr_logfile"
+focr_verbose
+focr_logfile
-    }
-}

trunk/devel/FuzzyOcr/Hashing.pm

@@ -78 +78 @@
-        my $db   = $conf->{focr_mysql_db};
-        my $sql  = qq(select * from $db.$dbfile where $dbfile.key='$key');
-        debuglog($sql);
-        my @data = $ddb->selectrow_array($sql);
-        my $next = 0;
-        my $when = 0;
-        if (scalar(@data)>0) {
-
+[$dbfile]
-            $next  = $data[5]; $next++;
-6
+7
-            $ret   = $data[8] == 0 ? 0.001 : $data[8];
-            $dinfo = $data[9] || '';
@@ -101 +100 @@
-            }
-        } else {
-"focr_db_max_days"
+focr_db_max_days
-            $sql = qq(select * from $db.$dbfile);
-            my $sth  = $ddb->prepare($sql); $sth->execute;
@@ -112 +111 @@
-                    $next  = $row[5] + 1;
-                    $when  = $row[7] || $now;
-"focr_mysql_hash"
+focr_mysql_hash
-                    $dinfo = $row[9] || '';
- in Table:'$dbfile
+[$dbfile]: Score='$row[8]' Info: '$row[9]
-                    last;
-                }
@@ -124 +123 @@
-        }
-        if ($ret > 0) {
-"focr_mysql_hash"
+focr_mysql_hash
-                debuglog("Found Score <$ret> for $txt Image Hash");
-            }
-
-conf->{focr_mysql_db}
+now - $
+db
-            debuglog($sql);
-            $ddb->do($sql);
@@ -134 +133 @@
-        return ($ret,$dinfo);
-    }
-"focr_enable_image_hashing"
+focr_enable_image_hashing
-        my %DB = (); my $dbm;
-        tie %DB, 'MLDBM', $dbfile, O_RDWR or $ret++;
@@ -153 +152 @@
-        }
-        if ($ret == 0) {
-"focr_db_max_days"
+focr_db_max_days
-            foreach my $k (keys %DB) {
-                $dbm  = $DB{$k};
-                $hash = $dbm->{basic} ? $dbm->{basic} : "0:0:0:0::$k";
-                if (within_threshold($digest,$hash)) {
-"focr_db_hash"
+focr_db_hash
-                    $txt  = 'Approx'; $dinfo = $dbm->{dinfo} || '';
-                    debuglog("Found in: <$dbfile>");
@@ -166 +165 @@
-                $dbm->{check} = $now - 1 unless defined $dbm->{check};
-                if ($dbm->{check} < $then) {
-'focr_db_max_days'
+focr_db_max_days
-                    delete $DB{$k};
-                }
@@ -173 +172 @@
-        if ($ret>0) {
-            $dbm->{match}++;
-"focr_db_hash"
+focr_db_hash
-                $ret = sprintf("%0.3f",$dbm->{score});
-                debuglog("Found Score <$ret> for $txt Image Hash");
@@ -184 +183 @@
-        return ($ret,$dinfo);
-    }
-"focr_enable_image_hashing"
-"focr_digest_db"
+focr_enable_image_hashing
+focr_digest_db
-        unless($ret) {
-'focr_digest_db'
+focr_digest_db
-            return (0,'');
-        }
@@ -225 +224 @@
-            my @data = $ddb->selectrow_array($sql);
-            unless (scalar(@data)) {
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-                debuglog($sql);
-                $ddb->do($sql);
-            } else {
-                debuglog("Hash already in $db.$table skipping...");
-
-
-
-
-
+
+
+
+
+
+
-        my %DB = ();
-        tie %DB, 'MLDBM', $dbfile or $ret++;
@@ -260 +258 @@
-            $dbm->{input} = 
-            $dbm->{check} = time;
-"focr_db_hash"
+focr_db_hash
-            $DB{$key} = $dbm;
-        }
-        untie %DB;
-    }
-"focr_enable_image_hashing"
-"focr_digest_db"
-'focr_digest_db'
+focr_enable_image_hashing
+focr_digest_db
+focr_digest_db
-        } else {
-'focr_digest_db'
+focr_digest_db
-        }
-        unless ($ret) {
-'focr_digest_db'
+focr_digest_db
-            return;
-        }
-'focr_digest_db'
+focr_digest_db
-        flock( DB, LOCK_EX );
-        seek( DB, 0, 2 );
@@ -290 +288 @@
-    my %Threshold = %$thresref;
-    my $pfile = $_[0];
+    my $pic   = $_[1];
-    my ($rcode, $hash);
-
-amfile ppmhist/) {
+pmhist/) { #pamfile
-        unless (defined $conf->{"focr_bin_$a"}) {
-            info("FuzzyOcr: calc_image_hash cannot exec $a");
@@ -305 +304 @@
-
-    my @stdout_data;
-
-
+
-    $rcode = $t->run_and_catch(sub {
-"focr_bin_pamfile"}
+focr_bin_ppmhist} -noheader
-    });
-    if ($rcode) {
-        chomp $rcode;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-        return (1, '');
-    }
-    my $cnt = 0;
-    my $c = scalar(@stdout_data);
-
+
+
+
+
+
-    if ($Threshold{max_hash}) {
-        foreach (@stdout_data) {
@@ -346 +335 @@
-
-sub fmt_time {
-time - 
+
-    my $ret;
-
+    debuglog("fmt_time: When =  $when");
-    if ($when>86400) {
-        my $d = int($when/86400);
@@ -357 +347 @@
-        my $h = int($when/3600);
-        $when -= $h*3600;
-ours
+rs.
-    }
-    if ($when>60) {
-        my $m = int($when/60);
-        $when -= $m*60;
-utes
+.
-    }
-    if ($when>0) {
-onds
+.
-    }
-    $ret .= " ago";