Changeset 80

Timestamp:

02.12.2006 21:39:04 (2 years ago)

Author:

decoder

Message:

Content-Type application/octet-stream is now considered generic (always accepted)
New rule FUZZY_OCR_WRONG_EXTENSION which hits files with fake extensions (e.g. GIF file blah.jpeg)

+ Scoring config switch for this rule

This is an improved and generalized version of Rosenbaum's patch

Files:

• trunk/devel/FuzzyOcr.cf (modified) (2 diffs)
• trunk/devel/FuzzyOcr.pm (modified) (10 diffs)
• trunk/devel/FuzzyOcr/Config.pm (modified) (1 diff)
• trunk/devel/FuzzyOcr/Scoring.pm (modified) (2 diffs)

trunk/devel/FuzzyOcr.cf

@@ -4 +4 @@
-loadplugin FuzzyOcr FuzzyOcr.pm
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-priority FUZZY_OCR 900
@@ -191 +193 @@
-#focr_wrongctype_score 1.5
-
+# This is the score to give for a wrong file extension.
+# e.g. JPEG image but file extension says GIF
+# Default value: 1.5
+#focr_wrongext_score 1.5
+
-# This is the score to give for a corrupted image.
-# This currently affects only GIF images

trunk/devel/FuzzyOcr.pm

@@ -39 +39 @@
-use FuzzyOcr::Hashing qw(check_image_hash_db add_image_hash_db calc_image_hash);
-use FuzzyOcr::Deanimate qw(deanimate);
-
+wrong_extension 
-use FuzzyOcr::Misc qw(max removedir save_execute);
-
@@ -137 +137 @@
-        my $test  = 0;
-        $test++ if ($ctype =~ /image/i);
-
+?
-
-        if ($test == 0) {
@@ -177 +177 @@
-        my $pdatalen = length($pdata);
-        my $w = 0; my $h = 0;
+
-        if ( substr($pdata,0,3) eq "\x47\x49\x46" ) {
-            ## GIF File
@@ -297 +298 @@
-        my @used_scansets = ();
-        my $corrupt = 0;
+        my $suffix = 0;
+        my $generic_ctype = 0;
-        my $digest;
-        my $tfile = $file;
@@ -304 +307 @@
-        debuglog("efile => $efile");
-
+        my $mimetype = $$pic{ctype};
+        if($mimetype =~ m'application/octet-stream'i) {
+            $generic_ctype = 1;
+        }
+
+        if($$pic{fname} =~ /\.([\w-]+)$/) {
+            $suffix = $1;
+        }
+        if ($suffix) {
+            debuglog("File has Content-Type \"$mimetype\" and File Extension \"$suffix\"");
+        } else {
+            debuglog("File has Content-Type \"$mimetype\" and no File Extension");
+        }
+
-        if ( $$pic{ftype} == 1 ) {
-            infolog("Found GIF header name=\"$$pic{fname}\"");
@@ -315 +332 @@
-            }
-
-$$pic{ctype} !~ /gif/i 
+($$pic{ctype} !~ /gif/i) and not $generic_ctype
-                wrong_ctype( "GIF", $$pic{ctype} );
-            }
+
+            if ( $suffix and $suffix !~ /gif/i) {
+                wrong_extension( "GIF", $suffix);
+            }
+
-            my $interlaced_gif = 0;
-            my $image_count = 0;
@@ -452 +474 @@
-                next;
-            }
-$$pic{ctype} !~ /(jpeg|jpg)/i 
+($$pic{ctype} !~ /(jpeg|jpg)/i) and not $generic_ctype
-                wrong_ctype( "JPEG", $$pic{ctype} );
-            }
+
+            if ( $suffix and $suffix !~ /(jpeg|jpg|jfif)/i) {
+                wrong_extension( "JPEG", $suffix);
+            }
+
-            foreach my $a (qw/jpegtopnm/) {
-                unless (defined $conf->{"focr_bin_$a"}) {
@@ -486 +513 @@
-                next;
-            }
-$$pic{ctype} !~ /png/i 
+($$pic{ctype} !~ /png/i) and not $generic_ctype
-                wrong_ctype( "PNG", $$pic{ctype} );
+            }
+            if ( $suffix and $suffix !~ /(png)/i) {
+                wrong_extension( "PNG", $suffix);
-            }
-            foreach my $a (qw/pngtopnm/) {
@@ -521 +551 @@
-                next;
-            }
-$$pic{ctype} !~ /bmp/i 
+($$pic{ctype} !~ /bmp/i) and not $generic_ctype
-                wrong_ctype( "BMP", $$pic{ctype} );
+            }
+            if ( $suffix and $suffix !~ /(bmp)/i) {
+                wrong_extension( "BMP", $suffix);
-            }
-            foreach my $a (qw/bmptopnm/) {
@@ -555 +588 @@
-                next;
-            }
-$$pic{ctype} !~ /tiff/i 
+($$pic{ctype} !~ /tif/i) and not $generic_ctype
-                wrong_ctype( "TIFF", $$pic{ctype} );
+            }
+            if ( $suffix and $suffix !~ /tif/i) {
+                wrong_extension( "TIFF", $suffix);
-            }
-

trunk/devel/FuzzyOcr/Config.pm

@@ -329 +329 @@
-    push (@cmds, {
-        setting => 'focr_wrongctype_score',
+        default => 1.5,
+        type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+    });
+
+    push (@cmds, {
+        setting => 'focr_wrongext_score',
-        default => 1.5,
-        type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC

trunk/devel/FuzzyOcr/Scoring.pm

@@ -3 +3 @@
-
-use base 'Exporter';
-
+ wrong_extension
-
-use lib qw(..);
@@ -29 +29 @@
-            $conf->{'focr_wrongctype_score'}, "BODY: ",
-            $pms->{conf}->{descriptions}->{FUZZY_OCR_WRONG_CTYPE} . "\n$debuginfo" );
+    }
+}
+
+sub wrong_extension {
+    my $conf = get_config();
+    my $pms = get_pms();
+    my ( $format, $suffix ) = @_;
+    if ($conf->{'focr_wrongext_score'}) {
+        my $debuginfo = "";
+        if ( $conf->{"focr_verbose"} > 0 ) {
+            $debuginfo = 
+              ("Image has format \"$format\" but file extension is \"$suffix\"");
+        }
+        infolog($debuginfo);
+        my $ws = sprintf( "%0.3f", $conf->{'focr_wrongext_score'} );
+        for my $set ( 0 .. 3 ) {
+            $pms->{conf}->{scoreset}->[$set]->{"FUZZY_OCR_WRONG_EXTENSION"} = $ws;
+        }
+        $pms->_handle_hit( "FUZZY_OCR_WRONG_EXTENSION",
+            $conf->{'focr_wrongext_score'}, "BODY: ",
+            $pms->{conf}->{descriptions}->{FUZZY_OCR_WRONG_EXTENSION} . "\n$debuginfo" );
-    }
-}