Ticket #270: send-quarantine-digests-refactor.2.patch

File send-quarantine-digests-refactor.2.patch, 15.3 kB (added by erik@…, 5 years ago)

[IGNORE LAST BASED ON DEBUGGING] patch send...digests.pl bug fixes, refactoring per #270

  • send-quarantine-digests.pl

    old new  
    9898    #           = "received_date DIRECTION" 
    9999    #           = "recipient_id DIRECTION" 
    100100    # Where DIRECTION is ASC or DESC 
    101     my %sort = (); 
    102     $sort{'ham'} = "score DESC";   # puts the high scores at the top 
    103     $sort{'spam'} = "score ASC";   # puts the low scroes at the top 
    104     $sort{'virus'} = "received_date DESC"; 
    105     $sort{'banned_file'} = "received_date DESC"; 
    106     $sort{'bad_header'} = "received_date DESC"; 
     101my %sort = ( 
     102            ham         => "score DESC",  # puts the high scores at the top 
     103            spam        => "score ASC",   # puts the low scroes at the top 
     104            virus       => "received_date DESC", 
     105            banned_file => "received_date DESC", 
     106            bad_header  => "received_date DESC", 
     107           ); 
    107108     
    108109    my $titles = {  'spam'        =>  "Spam Quarantine", 
    109110                    'virus'       =>  "Virus Quarantine", 
    … …  
    121122# End of user-configurable portion.  There should be no need to modify # 
    122123# anything below this point.                                           # 
    123124######################################################################## 
    124  
    125 # The organization of this file makes this a bit obtuse 
    126 my $isPg = 0; 
    127  
    128     # Retrieve the string value associated with a key in the database.cfg file. 
    129     sub get_string_key($$) 
    130     { 
    131        my ($file, $key) = @_; 
    132  
    133        if ($file =~ /\n[ \t]*$key[ \t]*=[ \t]*\"(.*)\"/) { 
    134           return ($1); 
    135        } else { 
    136           die ("Maia: [get_db_string_key] Unable to find $key value in $file\n"); 
    137        } 
    138     } 
    139  
    140     # Read a single value from Maia's configuration table. 
    141     sub get_config_value($$) { 
    142         my($dbh, $key) = @_; 
    143         my($sth, @row, $select); 
    144         my $value = undef; 
    145  
    146         $select = "SELECT " . $key . " FROM maia_config WHERE id = 0"; 
    147         $select = $1 if $select =~ /^(.*)$/si; # untaint 
    148         $sth = $dbh->prepare($select) 
    149                    or die (sprintf("Maia: [get_config_value] Couldn't prepare query: %s", $dbh->errstr)); 
    150         $sth->execute() 
    151             or die (sprintf("Maia: [get_config_value] Couldn't execute query: %s", $dbh->errstr)); 
    152         if (@row = $sth->fetchrow_array()) { 
    153             $value = $row[0]; 
    154         } 
    155         $sth->finish; 
    156  
    157         return $value; 
    158     } 
    159  
    160     #random phrase generated from password generator 
    161     #credit: http://web.uconn.edu/~cdavid/cgi-bin/book/make_password_html.pl 
    162     sub phrase_generate(){   
    163       my $ug    = new Data::UUID;   
    164       my $uuid = $ug->create_hex();   
    165       $uuid =~ s/0x(.*)/$1/;   
    166      
    167       my (@passset,$rnd_passwd,$randum_num);   
    168       my ($randum_num);   
    169       @passset = ('A'..'Z','0'..'9');   
    170       $rnd_passwd = "";   
    171       for (my $i = 0; $i<32;$i++){   
    172         $randum_num = int(rand($#passset+1));   
    173         $rnd_passwd .= @passset[$randum_num];   
    174       }   
    175      
    176       return $uuid . $rnd_passwd ;  
    177     } 
    178  
    179     sub generate_confirm_token($$) 
    180     { 
    181       my ($dbh, $maia_user_id) = @_; 
    182       my $spamexpiry = get_config_value($dbh, "expiry_period"); 
    183       my $hamexpiry = get_config_value($dbh, "ham_cache_expiry_period"); 
    184       my $days = $spamexpiry > $hamexpiry ? $spamexpiry : $hamexpiry; 
    185  
    186       my $unique_string = phrase_generate(); 
    187  
    188       my $insert = "INSERT INTO maia_tokens (token_system, token, data, expires) " . 
    189                    "VALUES ('digest',?,?, " . 
    190                        ($isPg ? "NOW() + INTERVAL ? DAY" 
    191                               : "DATE_ADD(NOW(), INTERVAL ? DAY)") . 
    192                            ")"; 
    193       my $sth = $dbh->prepare($insert); 
    194       $sth->execute($unique_string, $maia_user_id,$days) or die (sprintf("Maia: [send-quarantine-reminders] Couldn't execute query: %s", $dbh->errstr));; 
    195  
    196       return $unique_string; 
    197     } 
     125sub get_string_key($$); 
     126sub get_config_value($$); 
     127sub phrase_generate(); 
     128sub generate_confirm_token($$); 
    198129 
    199130    # Read the database configuration file into memory once 
    200131    open DB_CFG, "<" . $cfg 
    … …  
    209140    # Connect to the database 
    210141    my $dsn = get_string_key($db_cfg, "dsn"); 
    211142    # The organization of this file makes this a bit obtuse 
    212     $isPg = $dsn =~ /^dbi:Pg/; 
     143my $isPg = $dsn =~ /^dbi:Pg/; 
    213144    my $username = get_string_key($db_cfg, "username"); 
    214145    my $password = get_string_key($db_cfg, "password"); 
    215146    my $dbh = DBI->connect($dsn, $username, $password) 
    216147                       or die("Can't connect to SQL database"); 
    217148 
    218     my($select, $sth, $sth2, @row, @row2, $user_id, $user_email); 
     149my($query, $sth, @row, %config_value_cache); 
    219150    my($admin_email, $smtp_server, $smtp_port); 
    220     $select = "SELECT admin_email, " . 
    221                      "smtp_server, " . 
    222                      "smtp_port " . 
    223                "FROM maia_config WHERE id = 0"; 
    224     $sth = $dbh->prepare($select) 
    225                or die (sprintf("Maia: [send-quarantine-reminders] Couldn't prepare query: %s", $dbh->errstr)); 
     151 
     152$query = <<"endSQL;"; 
     153  SELECT admin_email, smtp_server, smtp_port 
     154  FROM   maia_config 
     155  WHERE  id = 0 
     156endSQL; 
     157 
     158$sth = $dbh->prepare($query) 
     159    or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
    226160    $sth->execute() 
    227         or die (sprintf("Maia: [send-quarantine-reminders] Couldn't execute query: %s", $dbh->errstr)); 
     161    or die (sprintf("Maia: [send-quarantine-digests] Couldn't execute query: %s", $dbh->errstr)); 
     162 
    228163    if (@row = $sth->fetchrow_array()) { 
    229164        $admin_email = $1 if $row[0] =~ /^(.+@.+\..+)$/si; # untaint 
    230165        $smtp_server = $1 if $row[1] =~ /^(.+)$/si; # untaint 
    231166        $smtp_port = $1 if $row[2] =~ /^([1-9]+[0-9]*)$/si; # untaint 
    232167    } 
    233     $sth->finish; 
     168$sth->finish(); 
    234169 
    235170    my $bgcolor; 
    236     my $rowcount; 
    237     my $token; 
    238     my $received_date; 
    239     my $score; 
    240     my $sender; 
    241     my $subject; 
    242171    my $at_least_one = 0; 
    243172    my $timestamp = time(); 
    244173    my ($secs, $mins, $hours, $day, $mon, $year) = localtime($timestamp); 
    245174    my $dbtimestamp = sprintf("%04d-%02d-%02d %02d:%02d:%02d", $year+1900, $mon+1, $day, $hours, $mins, $secs); 
    246175 
    247     $select = "SELECT maia_users.id, users.email " . 
    248               "FROM maia_users, users " . 
    249               "WHERE maia_users.primary_email_id = users.id " . 
    250               "AND maia_users.quarantine_digest_interval > 0 " . 
    251               "AND (maia_users.quarantine_digest_interval <= " . 
    252               ($isPg ? "(( ROUND(DATE_PART('epoch', NOW())) - ROUND(DATE_PART('epoch', maia_users.last_digest_sent))) / 60)" 
    253                      : "((UNIX_TIMESTAMP() - UNIX_TIMESTAMP(maia_users.last_digest_sent)) / 60) ") . 
    254                    "OR maia_users.last_digest_sent IS NULL) " . 
    255               "ORDER BY maia_users.id ASC"; 
    256     my $sth3 = $dbh->prepare($select) 
     176my $unixTime = $isPg ? "( ROUND(DATE_PART('epoch', NOW())) - ROUND(DATE_PART('epoch', maia_users.last_digest_sent)))" 
     177                     : "(UNIX_TIMESTAMP() - UNIX_TIMESTAMP(maia_users.last_digest_sent)"; 
     178 
     179$query = <<"endSQL;"; 
     180  SELECT   maia_users.id, users.email 
     181  FROM     maia_users, users 
     182  WHERE    maia_users.primary_email_id = users.id 
     183           AND maia_users.quarantine_digest_interval > 0 
     184           AND (maia_users.quarantine_digest_interval <= ($unixTime / 60) 
     185                OR maia_users.last_digest_sent IS NULL) 
     186  ORDER BY maia_users.id ASC 
     187endSQL; 
     188 
     189my $users_sth = $dbh->prepare($query) 
    257190                or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
    258     $sth3->execute() 
     191$users_sth->execute() 
    259192        or die (sprintf("Maia: [send-quarantine-digests] Couldn't execute query: %s", $dbh->errstr)); 
    260193 
    261     while (my @row3 = $sth3->fetchrow_array()) { 
    262         $user_id = $1 if $row3[0] =~ /^(\d+)$/si; # untaint 
    263         $user_email = $1 if $row3[1] =~ /^(.+@.+\..+)$/si; # untaint 
     194# Preparing the same statement over & over is wasteful. Granted, 
     195# performance doesn't really matter in this application, but there is no 
     196# sense in wasting db resources (which COULD be at a premium). 
     197 
     198my %report_statements; 
     199while (my($element, $sort) = each(%sort)) { 
     200    next if exists $report_statements{$sort}; 
     201 
     202    $query = <<"endSQL;"; 
     203  SELECT   mmr.token, mm.received_date, mm.score, 
     204           mm.sender_email, mm.subject 
     205  FROM     maia_mail AS mm, maia_mail_recipients AS mmr 
     206  WHERE    mm.id = mmr.mail_id 
     207           AND mmr.type = ? 
     208           AND mmr.recipient_id = ? 
     209  ORDER BY mm.$sort 
     210endSQL; 
     211 
     212    $report_statements{$sort} = $dbh->prepare($query) 
     213        or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
     214} 
     215 
     216$query = <<"endSQL;"; 
     217  UPDATE maia_users SET last_digest_sent = ? WHERE id = ? 
     218endSQL; 
     219my $update_sth = $dbh->prepare($query) 
     220    or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
     221 
     222my $date_add = $isPg ? "NOW() + INTERVAL ? DAY" 
     223                     : "DATE_ADD(NOW(), INTERVAL ? DAY)"; 
     224$query = <<"endSQL;"; 
     225  INSERT INTO maia_tokens (token_system, token, data, expires) 
     226  VALUES ('digest', ?, ?, $date_add) 
     227endSQL; 
     228 
     229my $confirm_sth = $dbh->prepare($query); 
     230 
     231while (my @row3 = $users_sth->fetchrow_array()) { 
     232    my $user_id = $1 if $row3[0] =~ /^(\d+)$/si; # untaint 
     233    my $user_email = $1 if $row3[1] =~ /^(.+@.+\..+)$/si; # untaint 
    264234 
    265235        #regenerate template vars for every user, to minimize memory usage 
    266236        my %vars = ( 
    … …  
    281251         
    282252        # We need to use an array to Separate Report Elements, since hashes can't keep reliable ordering 
    283253        my $report_count = 0; 
     254    $at_least_one = 0; 
    284255        for my $report_element (@report_order) { 
     256        $sth = $report_statements{$sort{$report_element}}; 
    285257         
    286             $select = "SELECT maia_mail_recipients.token, maia_mail.received_date, maia_mail.score, " . 
    287                 "maia_mail.sender_email, maia_mail.subject " . 
    288                 "FROM maia_mail, maia_mail_recipients " . 
    289                 "WHERE maia_mail.id = maia_mail_recipients.mail_id " . 
    290                 "AND maia_mail_recipients.type = '$report_type{$report_element}' " . 
    291                 "AND maia_mail_recipients.recipient_id = ? " . 
    292                 "ORDER BY maia_mail.$sort{$report_element}"; 
    293  
    294             $sth = $dbh->prepare($select) 
    295                 or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
    296             $sth->execute($user_id) 
     258        $sth->execute($report_type{$report_element}, $user_id) 
    297259                or die (sprintf("Maia: [send-quarantine-digests] Couldn't execute query: %s", $dbh->errstr)); 
    298260            if ($sth->rows > 0) { 
    299261                $at_least_one = 1; 
    300                 $rowcount = 0; 
     262            my $rowcount = 0; 
    301263                while (@row = $sth->fetchrow_array()) { 
    302                     $token = $1 if $row[0] =~  /^([a-zA-Z0-9]+)$/si; # untaint 
    303                     $received_date = $1 if $row[1] =~ /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/si; # untaint 
    304                     $score = $1 if $row[2] =~ /^(-?\d+\.\d+)$/si; # untaint 
    305                     $sender = $1 if $row[3] =~ /^(.+\@.+\..+)$/si; # untaint 
    306                     $subject = $1 if $row[4] =~ /^(.*)$/si; # untaint 
     264                my $token = $1 if $row[0] =~  /^([a-zA-Z0-9]+)$/si; # untaint 
     265                my $received_date = $1 if $row[1] =~ /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/si; # untaint 
     266                my $score = $1 if $row[2] =~ /^(-?\d+\.\d+)$/si; # untaint 
     267                my $sender = $1 if $row[3] =~ /^(.+\@.+\..+)$/si; # untaint 
     268                my $subject = $1 if $row[4] =~ /^(.*)$/si; # untaint 
    307269                    if ($subject eq "") { 
    308270                        $subject = "(no subject)"; 
    309271                    } 
    … …  
    313275                    if ($report_element eq 'ham' || $report_element eq 'spam') { 
    314276                        $vars{'list'}[$report_count]{$report_element}[$rowcount]{'score'} =  $score; 
    315277                    } 
     278 
    316279                    $vars{'list'}[$report_count]{$report_element}[$rowcount]{'sender'} =  $sender; 
    317280                    $vars{'list'}[$report_count]{$report_element}[$rowcount]{'subject'} =  $subject; 
    318281                     
    319282                    $rowcount++; 
    320283                } 
     284            $report_count++ 
    321285            } 
    322286            $sth->finish(); 
    323             $report_count++ 
    324287        } 
    325288         
     289    # Send out the e-mail 
     290    if ($at_least_one) { 
    326291        $vars{'confirm_token'} = generate_confirm_token($dbh, $user_id); 
    327292        my $output = ''; 
    328293        my $template = Template->new({INCLUDE_PATH => $template_dir, 
    … …  
    330295        $template->process("digest.tpl", \%vars)  
    331296       || die "Template process failed: ", $template->error(), "\n"; 
    332297 
    333         # Send out the e-mail 
    334         if ($at_least_one) { 
    335298            print "Sending quarantine digest to <" . $user_email . ">\n"; 
     299 
    336300            my($smtp) = Net::SMTP->new($smtp_server, Port => $smtp_port); 
    337301            die "Couldn't connect to SMTP server" unless $smtp; 
    338302            $smtp->mail($admin_email); 
    … …  
    342306            $smtp->dataend(); 
    343307            $smtp->quit(); 
    344308 
    345             my $update = "UPDATE maia_users SET last_digest_sent = ? WHERE id = ?"; 
    346             $sth = $dbh->prepare($update) 
    347                        or die (sprintf("Maia: [send-quarantine-digests] Couldn't prepare query: %s", $dbh->errstr)); 
    348             $sth->execute($dbtimestamp, $user_id) 
     309        $update_sth->execute($dbtimestamp, $user_id) 
    349310                or die (sprintf("Maia: [send-quarantine-digests] Couldn't execute query: %s", $dbh->errstr)); 
    350      } 
    351  
     311        $update_sth->finish(); 
    352312 
    353313    } 
    354     $sth3->finish(); 
     314} 
     315 
     316# not strictly necessary, since we're about to disconnect, but good 
     317# policy on the whole. 
     318$users_sth->finish(); 
    355319  
    356320    # Disconnect from the database 
    357321    $dbh->disconnect; 
    358322 
    359323    # We're done. 
    360324    exit; 
     325 
     326# Retrieve the string value associated with a key in the database.cfg file. 
     327sub get_string_key($$) { 
     328    my ($file, $key) = @_; 
     329 
     330    if ($file =~ /\n[ \t]*$key[ \t]*=[ \t]*\"(.*)\"/) { 
     331        return ($1); 
     332    } else { 
     333        die ("Maia: [get_db_string_key] Unable to find $key value in $file\n"); 
     334    } 
     335} 
     336 
     337# Read a single value from Maia's configuration table. 
     338sub get_config_value($$) { 
     339    my($dbh, $key) = @_; 
     340    my($sth, @row, $select); 
     341    my $value = undef; 
     342 
     343    return $config_value_cache{$key} if (exists $config_value_cache{$key}); 
     344 
     345    $select = "SELECT " . $key . " FROM maia_config WHERE id = 0"; 
     346    $select = $1 if $select =~ /^(.*)$/si; # untaint 
     347    $sth = $dbh->prepare($select) 
     348        or die (sprintf("Maia: [get_config_value] Couldn't prepare query: %s", $dbh->errstr)); 
     349    $sth->execute() 
     350        or die (sprintf("Maia: [get_config_value] Couldn't execute query: %s", $dbh->errstr)); 
     351    if (@row = $sth->fetchrow_array()) { 
     352        $value = $row[0]; 
     353    } 
     354    $sth->finish; 
     355    $config_value_cache{$key} = $value; 
     356    return $value; 
     357} 
     358 
     359#random phrase generated from password generator 
     360#credit: http://web.uconn.edu/~cdavid/cgi-bin/book/make_password_html.pl 
     361sub phrase_generate() { 
     362    my $ug    = new Data::UUID; 
     363    my $uuid = $ug->create_hex(); 
     364    $uuid =~ s/0x(.*)/$1/; 
     365 
     366    my (@passset,$rnd_passwd,$randum_num); 
     367    @passset = ('A'..'Z','0'..'9'); 
     368    $rnd_passwd = ""; 
     369    for (my $i = 0; $i<32;$i++) { 
     370        $randum_num = int(rand($#passset+1)); 
     371        $rnd_passwd .= $passset[$randum_num]; 
     372    } 
     373 
     374    return $uuid . $rnd_passwd ; 
     375} 
     376 
     377sub generate_confirm_token($$) { 
     378    my ($dbh, $maia_user_id) = @_; 
     379    my $spamexpiry = get_config_value($dbh, "expiry_period"); 
     380    my $hamexpiry = get_config_value($dbh, "ham_cache_expiry_period"); 
     381    my $days = $spamexpiry > $hamexpiry ? $spamexpiry : $hamexpiry; 
     382 
     383    my $unique_string = phrase_generate(); 
     384 
     385    $confirm_sth->execute($unique_string, $maia_user_id,$days) or die (sprintf("Maia: [send-quarantine-reminders] Couldn't execute query: %s", $dbh->errstr));; 
     386    $confirm_sth->finish(); 
     387    return $unique_string; 
     388}