Changeset 89

Timestamp:

06.12.2006 23:46:25 (2 years ago)

Author:

jorge

Message:

Update: fuzzy-find can now handle lookups with MySQL hashing
Change: fuzzy-find can specify where to find FuzzyOcr?.cf with --config option
Change: fuzzy-find can specify the score to use when learning ham/spam with --score option (default: 0/ham 10/spam)
Change: FuzzyOcr/Config?.pm Thresholds can be specified as plugin options to make changing these easier
Change: Log with 2 lines instead of 3 when a word match is found (cosmetic)

Files:

• trunk/devel/FuzzyOcr.pm (modified) (3 diffs)
• trunk/devel/FuzzyOcr/Config.pm (modified) (4 diffs)
• trunk/devel/Utils/fuzzy-find (modified) (9 diffs)

trunk/devel/FuzzyOcr.pm

@@ -139 +139 @@
-        my $test  = 0;
-        $test++ if ($ctype =~ /image/i);
-(gif|jpg|jpe
+\.(gif|jpe?
-
-        if ($test == 0) {
@@ -262 +262 @@
-        print PICT $pdata;
-        close PICT;
-info
+debug
-        $cnt++;
-    }
@@ -756 +756 @@
-                            $wcnt++;
-                            infolog(
-Found word \"$w\" in line\n \"$_\" \n
+Scanset \"$scanlabel\" found word \"$w\"
-                                . sprintf("%0.4f",$matched)
- scanned with scanset \"$scanlabel
+\nline: \"$_
-                            );
-                            if ($conf->{focr_unique_matches}) {

trunk/devel/FuzzyOcr/Config.pm

@@ -68 +68 @@
-my @img_types = qw/gif png jpeg bmp tiff/;
-
-# Default thresolds
-$Threshold{s}  = 
-$Threshold{h}  = 
-$Threshold{w}  = 
-$Threshold{cn} = 0.01;
-$Threshold{c}  = 5;
-$Threshold{max_hash} = 5;
-
-sub get_timeout {
-    unless (defined $timeout) {
@@ -164 +156 @@
-    $dsn .= ";port=".$conf->{focr_mysql_port} if $conf->{focr_mysql_port} != 3306;
-    }
-info
+debug
-    my $ddb = DBI->connect($dsn,
-        $conf->{focr_mysql_user},
@@ -176 +168 @@
-    my @cmds = ();
-
+    foreach my $t (qw/s h w cn/) {
+        push (@cmds, {
+            setting => 'focr_threshold_'.$t,
+            default => 0.01,
+            type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+        });
+    }
+    foreach my $t (qw/c max_hash/) {
+        push (@cmds, {
+            setting => 'focr_threshold_'.$t,
+            default => 5,
+            type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+        });
+    }
-    push (@cmds, {
-        setting => 'focr_threshold',
@@ -552 +558 @@
-    }
-
+    # Allow scanning if in debug mode?
+    $conf->{focr_autodisable_score} = 1000
+        if $Mail::SpamAssassin::Logger::LOG_SA{level} == 3;
+
+    # Extract Thresholds
+    foreach my $k (keys %{$conf}) {
+        if ($k =~ m/^focr_threshold_(\S+)/) {
+            $Threshold{$1} = $conf->{$k};
+            debuglog("Threshold[$1] => $conf->{$k}");
+        }
+    }
-    # Display All Options
-    foreach my $k (sort keys %{$conf}) {

trunk/devel/Utils/fuzzy-find

@@ -1 @@
-
+local/
-use Getopt::Long;
+use DBI;
-use MLDBM qw(DB_File Storable);
-my %Files = (
@@ -7 +8 @@
-    );  
-
+my %MySQL = (
+     db   => 'FuzzyOcr'
+    ,hash => 'Hash'
+    ,safe => 'Safe'
+    ,user => 'fuzzyocr'
+    ,pass => 'fuzzyocr'
+    ,host => 'localhost'
+    ,port => 3306
+);
+
-# defaults
-
+/etc/mail/spamassassin/
-my %App;
-my @bin_utils = qw/pamfile ppmhist jpegtopnm giftopnm pngtopnm bmptopnm/;
-foreach (@bin_utils) {
-    $App{$_} = "/usr/bin/$_";
-}
-
-my $delete = 0;
@@ -19 +27 @@
-my $learn_ham = 0;
-my $learn_spam = 0;
+my $score;
-GetOptions(
-    'verbose' => \$verbose,
-    'delete'  => \$delete,
+    'config=s' => \$cfgfile,
+    'score=f' => \$score,
-    'learn-ham' => \$learn_ham,
-    'learn-spam' => \$learn_spam,
@@ -30 +41 @@
-    print "\n";
-    print "Available options:\n";
+    print "--config=s   Specify location of FuzzyOcr.cf\n";
+    print "             Default: /etc/mail/spamassasin/FuzzyOcr.cf\n";
-    print "--delete     Removes the hash from the database\n";
-    print "--learn-ham  Add the hash as ham to the database\n";
-    print "--learn-spam Add the hash as spam to the database\n";
+    print "--score=i    Score to use when adding ham/spam\n";
-    print "--verbose    Show more informations\n";
-    print "\n";
@@ -38 +52 @@
-}
-
+# Setup default score
+unless (defined $score) {
+    $score = $learn_ham ? 10 : 0;
+}
+
-# Read custom paths from FuzzyOcr.cf
+my $app_path = q(/usr/local/netpbm/bin:/usr/local/bin:/usr/bin);
-open CONFIG, "< $cfgfile" or warn "Can't read configuration file, using defaults...\n";
-
@@ -44 +64 @@
-    chomp;
-    if ($_ =~ m/^focr_bin_(\w+) (.+)/) {
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-}
-
-close CONFIG;
+
+# make shure we have this threshold set
+$App{max_hash} = 5 unless defined $App{max_hash};
+
+# search path for bin_util unless already specified in configuration file
+foreach my $app (@bin_utils) {
+    next if defined $App{$app};
+    foreach my $d (split(':',$app_path)) {
+        if (-x "$d/$app") {
+            $App{$app} = "$d/$app";
+            last;
+        }
+    }
+}
+
+sub get_ddb {
+    my %dopts = ( AutoCommit => 1 );
+    my $dsn = "dbi:mysql:database=".$MySQL{db};
+    if (defined $MySQL{socket}) {
+        $dsn .= ";mysql_socket=$MySQL{socket}";
+    } else {
+        $dsn .= ";host=$MySQL{host}";
+        $dns .= ";port=$MySQL{port}" unless $MySQL{port} == 3306;
+    }
+    printf "Connecting to: $dsn\n" if $verbose;
+    return DBI->connect($dsn,$MySQL{user},$MySQL{pass},\%dopts);
+}
-
-while (@ARGV) {
@@ -64 +128 @@
-    my $key = '';
-    my $ctype = '';
+    my $ftype = 0;
-    unless (@data) {
-        my $app;
-        if (($file =~ m/\.jpg$/i) or ($file =~ m/\.jpeg$/i)) {
-            $app = $App{jpegtopnm};
-
+
+
-        } elsif ($file =~ m/\.png$/i) {
-            $app = $App{pngtopnm};
-
+
+
-        } elsif ($file =~ m/\.bmp$/i) {
+            $ctype = "image/bmp";
-            $app = $App{bmptopnm};
-
+
+
+
+
+
+
+
+
+
-        } elsif ($file =~ m/\.pnm$/i) {
-            $ctype = "image/pnm";
-            $app = '/bin/cat';
-
-
-
+
-        } else {
-       
-       
-       
-
-
+        
+        
+        
+2>/dev/null 
+2>/dev/null 
-        my ($h,$w) = (0,0);
-        if ($res[0] =~ m/(\d+) by (\d+)/) {
@@ -99 +172 @@
-            my @d = split(' ',$_);
-            $hash .= sprintf("::%d:%d:%d:%d:%d",@d);
-5
+$App{max_hash}
-        }
-        $key = substr($hash,2);
@@ -106 +179 @@
-    printf "key = <$key>\n" if ($key);
-    if ($learn_spam || $learn_ham) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    } else {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+