Changeset 38

Timestamp:

16.11.2006 23:47:41 (2 years ago)

Author:

jorge

Message:

Added option to store hash in MySQL database making it easier for multiple servers to share the Hash/Safe DB

Files:

• trunk/devel/FuzzyOcr.cf (modified) (2 diffs)
• trunk/devel/FuzzyOcr.mysql (added)
• trunk/devel/FuzzyOcr.pm (modified) (7 diffs)
• trunk/devel/FuzzyOcr/Config.pm (modified) (6 diffs)
• trunk/devel/FuzzyOcr/Hashing.pm (modified) (9 diffs)

trunk/devel/FuzzyOcr.cf

@@ -135 +135 @@
-# Value = 1 ... use digest_hash only
-# Value = 2 ... use digest_db w/digest_hash import
+# Value = 3 ... use mysql database
-#focr_enable_image_hashing 2
-#
@@ -154 +155 @@
-#focr_db_max_days 15
-#
+# MySQL options
+#focr_mysql_db FuzzyOcr
+#focr_mysql_hash Hash
+#focr_mysql_safe Safe
+#focr_mysql_user fuzzyocr
+#focr_mysql_pass fuzzyocr
+#focr_mysql_host localhost
+#focr_mysql_port 3306
+#focr_mysql_socket /tmp/mysql.sock (Default: empty)
+#
-# Automatically add hashes of spam images recognized by OCR to the Image Hash database, to disable, set to 0.0 (Default value: 1)
-#focr_hashing_learn_scanned 1

trunk/devel/FuzzyOcr.pm

@@ -19 +19 @@
-
-use lib qw(.); # Allow placing of FuzzyOcr in siteconfigdir
-
+ddb get_
-use FuzzyOcr::Hashing qw(check_image_hash_db check_image_hash_db add_image_hash_db calc_image_hash);
-use FuzzyOcr::Deanimate qw(deanimate);
@@ -74 +74 @@
-            debuglog("Variable \$ENV{HOME} not defined and getpwuid failed, personal wordlist function not available...");
-        }
+    }
+    if ($conf->{focr_enable_image_hashing} == 3) {
+        $conf->{focr_ddb} = get_ddb();
-    }
-
@@ -428 +431 @@
-                debuglog("Error calculating the image hash, skipping hash check...");
-            } else {
-
-
+
+
+
+
+
-                if ($score > 0) {
-                    known_img_hash($score,$dinfo);
@@ -436 +442 @@
-                    return 0;
-                }
-
+
+
+
+
-                if ($score > 0) {
-                    debuglog("Image in KNOWN_GOOD. Skipping OCR checks...");
@@ -521 +530 @@
-    
-    if ($cnt == 0) {
-== 2
+> 1
-            debuglog("Message is ham, saving...");
-            foreach my $h (@hashes) {
-                my ($mcnt,$fname,$ctype,$digest) = split('::',$h,4);
-                next if $mcnt;
-
+
+
+
+
-            }
-        }
@@ -548 +560 @@
-                my ($mcnt,$fname,$ctype,$digest) = split('::',$h,4);
-                next unless $mcnt;
-
+
+
+
+
-            }
-        }
@@ -562 +577 @@
-    if ($imgerr == 0 and $conf->{"focr_keep_bad_images"}<2) {
-        removedir($imgdir);
+    }
+    if ($conf->{focr_enable_image_hashing} == 3) {
+        $conf->{focr_ddb}->disconnect;
-    }
-    debuglog("FuzzyOcr ending successfully...");

trunk/devel/FuzzyOcr/Config.pm

@@ -10 +10 @@
-    get_wordlist 
-    set_config 
+    get_ddb
-    finish_parsing_end 
-    load_global_words 
@@ -26 +27 @@
-our $conf;
-our $pms;
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-our @paths = qw(/usr/local/netpbm/bin /usr/local/bin /usr/bin);
@@ -75 +79 @@
-}
-
+sub get_ddb {
+    my $conf = get_config();
+    my %dopts = ( AutoCommit => 1 );
+    my $dsn = sprintf "dbi:mysql:%s\@%s:%d",
+        $conf->{focr_mysql_db},
+        $conf->{focr_mysql_host},
+        $conf->{focr_mysql_port}
+        );
+    my $ddb = DBI->connect($dsn,
+        $conf->{focr_mysql_user},
+        $conf->{focr_mysql_pass},
+        \%dopts);
+    return $ddb;
+}
+
-sub set_config {
-    my($self, $conf) = @_;
@@ -125 +144 @@
-                return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
-            }
-
+3
-                return $Mail::SpamAssassin::Conf::INVALID_VALUE;
-            }
@@ -146 +165 @@
-    push (@cmds, {
-        setting => 'focr_global_wordlist',
-/etc/mail/spamassassin
+__userstate__
-        type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
-       });
@@ -260 +279 @@
-        default => '$gocr -i $pfile, $gocr -l 180 -d 2 -i $pfile, $gocr -l 140 -d 2 -i $pfile',
-        type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-    $conf->{parser}->register_commands(\@cmds);

trunk/devel/FuzzyOcr/Hashing.pm

@@ -10 +10 @@
-use lib "../";
-use FuzzyOcr::Config qw(get_thresholds get_config set_config debuglog logfile);
+use DBI;
-use MLDBM qw(DB_File Storable);
-use Fcntl;
@@ -58 +59 @@
-sub check_image_hash_db {
-    my $conf = get_config();
-
-
-
-
+
-    my $digest = $_[0];
-    my $dbfile = $_[1] || $conf->{"focr_db_hash"};
@@ -68 +66 @@
-    my ($img, $key) = split('::', $digest,2);
-    return (0,'') unless defined $key;
+    my $now = time;
-    my $hash = $digest;
-    my $ret = 0; my $txt = 'Exact';
-    my $dinfo;
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        tie %DB, 'MLDBM', $dbfile, O_RDWR or $ret++;
-        if ($ret>0) {
@@ -91 +140 @@
-        }
-        if ($ret == 0) {
-now
+then
-            foreach my $k (keys %DB) {
-                $dbm  = $DB{$k};
@@ -103 +152 @@
-                # Has the record expired??
-                $dbm->{check} = $now - 1 unless defined $dbm->{check};
-now
+then
-                    debuglog("Expiring <$k> older than $conf->{'focr_db_max_days'} days");
-                    delete $DB{$k};
@@ -121 +170 @@
-        untie %DB;
-        return ($ret,$dinfo);
-
+
+
-        $ret = open HASH, $conf->{"focr_digest_db"};
-        unless($ret) {
@@ -142 +192 @@
-sub add_image_hash_db {
-    my $conf = get_config();
-
-
-
-
+
-    my $digest = $_[0];
-    my $score  = $_[1];
-    my $ret = 0;
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        my $dbfile = $_[2] || $conf->{"focr_db_hash"};
-        my %DB = ();
@@ -159 +224 @@
-        }
-        debuglog("Adding Hash to \"$dbfile\"");
-
-        my ($img,$key) = split('::',$digest,2);
-        if (defined $key) {
@@ -174 +238 @@
-        }
-        untie %DB;
-
+
+
-        if (-e $conf->{"focr_digest_db"}) {
-            $ret = open DB, ">>$conf->{'focr_digest_db'}";