Changeset 81

Timestamp:

03.12.2006 00:03:49 (2 years ago)

Author:

decoder

Message:

Critical bugfix in Preprocessor.pm (Same bug as Scanset.pm earlier)
Modified matching system, the first matching pass is now done without stripping spaces,
a second pass with stripping spaces is only done when the first didn't give enough matches.
Matches from the first pass are multiplied with focr_twopass_scoring_factor.

This allows to reduce false positives caused by space stripping.

New option: Optionally do not strip numbers from OCR results and wordlists to match them

Files:

• trunk/devel/FuzzyOcr.cf (modified) (2 diffs)
• trunk/devel/FuzzyOcr.pm (modified) (4 diffs)
• trunk/devel/FuzzyOcr/Config.pm (modified) (1 diff)
• trunk/devel/FuzzyOcr/Preprocessor.pm (modified) (2 diffs)

trunk/devel/FuzzyOcr.cf

@@ -54 +54 @@
-#focr_personal_wordlist fuzzyocr.words
-#
+## Optionally, disable this option if you want to scan for numbers
+## Setting this to 0 will cause FuzzyOcr not to strip numbers from
+## both the wordlist and the OCR results
+#
+#focr_strip_numbers 1
+
-
-###
@@ -188 +194 @@
-focr_add_score 0.375
-
+# This option defines the factor, which is multiplied with the number
+# of matches, that were made without stripping spaces. FuzzyOcr does two
+# matching attempts on OCR results, one without space strippings and one with.
+# To weight the first match type more, this factor is applied.
+# Default value: 1.5
+#focr_twopass_scoring_factor 1.5
+
-# This is the score to give for a wrong content-type.
-# e.g. JPEG image but content type says GIF

trunk/devel/FuzzyOcr.pm

@@ -657 +657 @@
-        my $scansets = get_scansets();
-        my $mcnt = 0;
+        my $modus = 0;
+        my $modus_match = 0;
-        my $wref = get_wordlist();
-        my %words = %$wref;
@@ -688 +690 @@
-            } elsif ($retcode>0) {
-                chomp $retcode;
-
+
+
-                infolog("Errors in Scanset \"$scanlabel\"");
-                infolog($errstr);
@@ -697 +700 @@
-
-            debuglog("ocrdata=>>".join("",@result)."<<=end");
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            if ($mcnt >= $conf->{focr_counts_required} and $conf->{focr_minimal_scanset}) {
-                infolog("Scanset \"$scanlabel\" generates enough hits ($mcnt), skipping further scansets...");
@@ -750 +779 @@
-            push(@hashes, $info);
-        }
-
+
+
+
+
+
+
+
-    }
-    close RAWERR if ($haserr>0);

trunk/devel/FuzzyOcr/Config.pm

@@ -295 +295 @@
-            $self->{focr_keep_bad_images} = $value+0;
-        }
+    });
+
+    push (@cmds, {
+        setting => 'focr_strip_numbers',
+        default => 1,
+        type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+    });
+
+    push (@cmds, {
+        setting => 'focr_twopass_scoring_factor',
+        default => 1.5,
+        type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
-    });
-

trunk/devel/FuzzyOcr/Preprocessor.pm

@@ -23 +23 @@
-    my $rcmd = $self->{command};
-
+    if (defined $args) {
+        $rcmd .= ' ' . $args;
+    }
+
-    # Does the processor expect input from file or from stdin?
-    if(defined $args and $args =~ /\$input/) {
-.= " $input"
+=~ s/\$input/$input/
-    } else {
-        $stdin = "<$input";
@@ -32 +36 @@
-    # Does it output to file or to stdout?
-    if(defined $args and $args =~ /\$output/) {
-.= " $output"
+=~ s/\$output/$output/
-    } else {
-        $stdout = ">$output";