root/trunk/devel/patchset2.patch

FuzzyOcr.cf

@@ -33 +33 @@
-# Default value: 1
-#focr_verbose 3
-
+# Log Message-Id, From, To
+# Default: 1
+#focr_log_pmsinfo 0
+
-# Send logging output to stderr.
-# Default value: 1
-#focr_log_stderr 0
@@ -163 +167 @@
-# Default value: 0
-#focr_global_timeout 1
-
+# Minimum image size to scan. Images with dimensions smaller than the
+# ones specified here will be skipped:
+# Default: Height:4 Width:4
+#focr_min_height 4
+#focr_min_width 4
+
-# Maximum file size for different formats in byte, bigger pictures 
-# will not be scanned 
-# Default values: Unlimited)

FuzzyOcr.pm

@@ -124 +124 @@
-    my $imgerr   = 0;
-    my $main     = $self->{main};
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-    chomp($from, $to, $msgid);
-
-    debuglog("Starting FuzzyOcr...");
-    infolog("Processing Message with ID \"$msgid\" ($from -> $to)");
-    foreach my $p (
-        $pms->{msg}->find_parts(qr(^image\b)i),
-        $pms->{msg}->find_parts(qr(Application/Octet-Stream)i)
@@ -146 +149 @@
-            $fname =~ tr/\@\$\%\&/_/s;
-        }
-
-
+
+
-        my $pdata = $p->decode();
-        my $pdatalen = length($pdata);
-        my $w = 0; my $h = 0;
-
-        my $blah = substr($pdata,0,3);
-
-        if ( substr($pdata,0,3) eq "\x47\x49\x46" ) {
-            ## GIF File
-            $imgfiles{$filename}{ftype} = 1; 
-            ($w,$h) = unpack("vv",substr($pdata,6,4));
-
+ ($pdatalen)
-            $imgfiles{$filename}{width}  = $w;
-            $imgfiles{$filename}{height} = $h;
-        } elsif ( substr($pdata,0,2) eq "\xff\xd8" ) {
@@ -184 +186 @@
-                errorlog("Cannot find image dimensions");
-            } else {
-                ($h,$w) = unpack("nn",substr($pdata,$pos+3,4));
-
+ ($pdatalen)
-                $imgfiles{$filename}{ftype} = 2;
-                $imgfiles{$filename}{height} = $h;
-                $imgfiles{$filename}{width}  = $w;
@@ -195 +197 @@
-            $imgfiles{$filename}{ftype}  = 3;
-            $imgfiles{$filename}{width}  = $w;
-            $imgfiles{$filename}{height} = $h;
-
+ ($pdatalen)
-        } elsif ( substr($pdata,0,2) eq "BM" ) {
-            ## BMP File
-NN
+VV
-            $imgfiles{$filename}{ftype}  = 4;
-            $imgfiles{$filename}{width}  = $w;
-            $imgfiles{$filename}{height} = $h;
-
+ ($pdatalen)
-        } elsif (
-            ## TIFF File
-            (substr($pdata,0,4) eq "\x4d\x4d\x00\x2a") or
@@ -218 +220 @@
-                $w = $val if ($id == 257);
-                last if ($h != 0 and $w != 0);
-            }
-
+pdatalen) ($
-            infolog("Cannot determine size of TIFF image, setting to '1x1'") if ($h == 0 and $w == 0);
-            $imgfiles{$filename}{ftype}  = 5;
-            $imgfiles{$filename}{width}  = $w ? $w : 1;
@@ -230 +232 @@
-            infolog("Skipping file with content-type=\"$ctype\" name=\"$fname\"");
-            delete $imgfiles{$filename};
-            next;
+        }
-
+        #Skip images that cannot contain text
+        if ($imgfiles{$filename}{height} < $conf->{focr_min_height}) {
+            infolog("Skipping image: height < $conf->{focr_min_height}");
+            delete $imgfiles{$filename};
+            next;
-        }
-
+        #Skip images that cannot contain text
+        if ($imgfiles{$filename}{width} < $conf->{focr_min_width}) {
+            infolog("Skipping image: width < $conf->{focr_min_width}");
+            delete $imgfiles{$filename};
+            next;
+        }
+
-        #Found Image!! Get a temporary dir to save image
-        $imgdir = Mail::SpamAssassin::Util::secure_tmpdir();
-        unless ($imgdir) {
@@ -243 +258 @@
-
-        #Generete unique filename to store image
-        my $imgfilename = Mail::SpamAssassin::Util::untaint_file_path(
-
+ile
-        );
-        my $unique = 0;
-        while (-e $imgfilename) {
-            $imgfilename = Mail::SpamAssassin::Util::untaint_file_path(
-
+ile
-            );
-            $unique++;
-        }
@@ -308 +323 @@
-        }
-    }
-
-    IMAGE:
-    my $haserr;
-    foreach my $filename (keys %imgfiles) {
-        my $pic = $imgfiles{$filename};
@@ -351 +365 @@
-            infolog("Found GIF header name=\"$$pic{fname}\"");
-            if ($conf->{focr_skip_gif}) {
-                infolog("Skipping image check");
- IMAGE
+
-            }
-            if (defined($conf->{focr_max_size_gif}) and ($$pic{fsize} > $conf->{focr_max_size_gif})) {
-                infolog("GIF file size ($$pic{fsize}) exceeds maximum file size for this format, skipping...");
@@ -374 +388 @@
-            foreach my $a (qw/gifsicle giftext giffix gifinter giftopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-                    errorlog("Cannot exec $a, skipping image");
- IMAGE
+
-                }
-            }
+
-            my @stderr_data;
-
-            my ($retcode, @stdout_data) = save_execute(
-                "$conf->{focr_bin_giftext} $file",
-                undef,
@@ -429 +443 @@
-                    }
-                }
-            }
-
-((stat($tfile))[7])
-$pic{fsize}
+
+$fixedsize
+fixedsize
-                next;
-            }
-
@@ -503 +517 @@
-            infolog("Found JPEG header name=\"$$pic{fname}\"");
-            if ($conf->{focr_skip_jpeg}) {
-                infolog("Skipping image check");
- IMAGE
+
-            }
-
-            if (defined($conf->{focr_max_size_jpeg}) and ($$pic{fsize} > $conf->{focr_max_size_jpeg})) {
@@ -523 +537 @@
-            foreach my $a (qw/jpegtopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-                    errorlog("Cannot exec $a, skipping image");
- IMAGE
+
-                }
-            }
-            printf RAWERR qq(## $conf->{focr_bin_jpegtopnm} $file >$pfile 2>>$efile\n) if ($haserr>0);
@@ -545 +559 @@
-            infolog("Found PNG header name=\"$$pic{fname}\"");
-            if ($conf->{focr_skip_png}) {
-                infolog("Skipping image check");
- IMAGE
+
-            }
-            if (defined($conf->{focr_max_size_png}) and ($$pic{fsize} > $conf->{focr__max_size_png})) {
-                infolog("PNG file size ($$pic{fsize}) exceeds maximum file size for this format, skipping...");
@@ -562 +576 @@
-            foreach my $a (qw/pngtopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-                    errorlog("Cannot exec $a, skipping image");
- IMAGE
+
-                }
-            }
-
@@ -585 +599 @@
-            infolog("Found BMP header name=\"$$pic{fname}\"");
-            if ($conf->{focr_skip_bmp}) {
-                infolog("Skipping image check");
- IMAGE
+
-            }
-            if (defined($conf->{focr_max_size_bmp}) and ($$pic{fsize} > $conf->{focr_max_size_bmp})) {
-                infolog("BMP file size ($$pic{fsize}) exceeds maximum file size for this format, skipping...");
@@ -602 +616 @@
-            foreach my $a (qw/bmptopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-                    errorlog("Cannot exec $a, skipping image");
- IMAGE
+
-                }
-            }
-            printf RAWERR qq(## $conf->{focr_bin_bmptopnm} $file >$pfile 2>>$efile\n) if ($haserr>0);
@@ -624 +638 @@
-            infolog("Found TIFF header name=\"$$pic{fname}\"");
-            if ($conf->{focr_skip_tiff}) {
-                infolog("Skipping image check");
- IMAGE
+
-            }
-            if (defined($conf->{focr_max_size_tiff}) and ($$pic{fsize} > $conf->{focr_max_size_tiff})) {
-                infolog("TIFF file size ($$pic{fsize}) exceeds maximum file size for this format, skipping...");
@@ -642 +656 @@
-            foreach my $a (qw/tifftopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
-                    errorlog("Cannot exec $a, skipping image");
- IMAGE
+
-                }
-            }
-            printf RAWERR qq(## $conf->{focr_bin_tifftopnm} $file >$pfile 2>>$efile\n) if ($haserr>0);
@@ -688 +702 @@
-                ($score,$dinfo) = check_image_hash_db($digest, $whash, $$pic{fname}, $$pic{ctype}, $$pic{ftype});
-                if ($score > 0) {
-                    infolog("Image in KNOWN_GOOD. Skipping OCR checks...");
- IMAGE
+
-                }
-            }
-            if ($digest eq '') {
-                infolog("Empty Hash, skipping...");
- IMAGE
+
-            }
-        } else {
-            infolog("Image hashing disabled in configuration, skipping...");
@@ -711 +725 @@
-
-        my @ocr_results = ();
-        my $scansets = get_scansets();
+        my $newlist = '';
+        foreach my $s (@$scansets) {
+            $newlist .= ' ' . $s->{label} . '(' . $s->{hit_counter} . ')';
+        }
+        infolog("Scanset Order:$newlist");
-        my $mcnt = 0;
-        my $modus = 0;
-        my $modus_match = 0;
@@ -832 +851 @@
-                            }
-                        }
-                    }
-
-
+
-                }
-                last;
-            }

FuzzyOcr/Config.pm

@@ -130 +130 @@
-}
-
-sub get_scansets {
+    if ($conf->{focr_autosort_scanset}) {
+        @scansets = sort { $b->{hit_counter} <=> $a->{hit_counter} } @scansets;
+    }
-    return \@scansets;
-}
-
@@ -194 +197 @@
-            type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
-        });
-    }
+    foreach my $t (qw/height width/) {
+        push (@cmds, {
+            setting => 'focr_min_'.$t,
+            default => 4,
+            type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+        });
+    }
-    push (@cmds, {
-        setting => 'focr_threshold',
-        default => 0.25,
@@ -245 +255 @@
-    });
-
-    push (@cmds, {
+        setting => 'focr_log_pmsinfo',
+        default => 1,
+        type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL
+    });
+
+    push (@cmds, {
-        setting => 'focr_enable_image_hashing',
-        default => 0,
-        code => sub {

FuzzyOcr/Hashing.pm

@@ -94 +94 @@
-            $dinfo = $data[9] || '';
-            if ($data[2] eq '') {
-                infolog("Updating $txt info File-Name:'$fname'");
-'$fname' where $dbfile.key='$key')
+? where $dbfile.key='$key'),undef,$fname
-            }
-            if ($data[3] eq '') {
-                infolog("Updating $txt info Content-Type:'$ctype'");
-'$ctype' where $dbfile.key='$key')
+? where $dbfile.key='$key'),undef,$ctype
-            }
-            if ($data[4] != $ftype) {
-                infolog("Updating $txt info File-Type:'$ftype'");
-'$ftype' where $dbfile.key='$key')
+? where $dbfile.key='$key'),undef,$ftype
-            }
-        }
-        unless ($match) {
@@ -134 +134 @@
-            }
-            infolog("Matched [$next] time(s). Prev match: ".fmt_time($now - $when));
-            $sql = qq(update $db.$dbfile set $dbfile.match='$next',$dbfile.check='$now' where $dbfile.key='$key');
-            debuglog($sql,2);
-            $ddb->do($sql);
+            debuglog($sql);
-        }
-        return ($ret,$dinfo);
-    }
@@ -143 +143 @@
-        use MLDBM qw(DB_File Storable);
-        use MLDBM::Sync;
-        my %DB = (); my $dbm; my $sdbm;
-
+CREAT|O_
-        if ($ret>0) {
-            warnlog("No Image Hash database found at \"$dbfile\", or permissions wrong.");
-            return (0,'');
@@ -242 +242 @@
-                if ($conf->{focr_mysql_update_hash}) {
-                    infolog("Hash already in $db.$table updating...");
-                    $sql  = "update $db.$table set ";
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                    $sql  =~ s/,$//;
-                    $sql .= " where $table.key='$key'";
+                    $ddb->do($sql,undef,@params);
+                    foreach my $p (@params) { $sql =~ s/\?/$p/; }
-                    debuglog($sql);
-                    $ddb->do($sql);
-                } else {
-                    infolog("Hash already in $db.$table skipping...");
-                }
-            } else {
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-                debuglog($sql);
-                $ddb->do($sql);
-            }
-        }
-    }