Initial commit.

This commit is contained in:
2021-05-24 22:18:33 +03:00
commit e2954d55f4
3701 changed files with 330017 additions and 0 deletions

View File

@@ -0,0 +1 @@
identity-ruD5ziim06

View File

@@ -0,0 +1,2 @@
pass = E0poZPVgJgr1HMfB9rfpeROXhPnL
user = ru5QBQp4Pq

View File

@@ -0,0 +1,2 @@
pass = DDLJDlYq4FlQUs0yFF1BUctXWg54
user = ruD5ziim06

View File

@@ -0,0 +1,26 @@
#
# Razor2 config file
#
# Autogenerated by Razor-Agents v2.84
# Thu Jan 19 12:44:12 2012
# Non-default values taken from /etc/mail/spamassassin/.razor/razor-agent.conf
#
# see razor-agent.conf(5) man page
#
debuglevel = 3
identity = identity
ignorelist = 0
listfile_catalogue = servers.catalogue.lst
listfile_discovery = servers.discovery.lst
listfile_nomination = servers.nomination.lst
logfile = razor-agent.log
logic_method = 4
min_cf = ac
razordiscovery = discovery.razor.cloudmark.com
razorhome = /etc/mail/spamassassin/.razor
rediscovery_wait = 172800
report_headers = 1
turn_off_discovery = 0
use_engines = 4,8
whitelist = razor-whitelist

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,36 @@
#
# Autogenerated by Razor-Agents v2.84, Sun Apr 26 14:03:30 2015
ac = 21
bql = 50
bqs = 329
cp = 7 1 6 5 4 3 2 0
crt = 90
cs = 1
dre = 8
ep10 = 20
ep16 = 5
ep19 = 5
ep20 = 5
ep21 = 5
ep22 = 5
ep4 = 7542-10
ep8 = 20
immdi = 1
imsio = 0
lm = 4
logic_method = 4
lsp = 3
lsp10 = 0
mhs = 15
mphs = 4
mps = 300
nmp = 2
sa = 2FB
se = 2380
sn = C
srf = FF
srl = 25305
sv = 4.007
to = 15
zone = razor2.cloudmark.com

View File

@@ -0,0 +1,36 @@
#
# Autogenerated by Razor-Agents v2.84, Sat Apr 25 22:44:21 2015
ac = 21
bql = 50
bqs = 329
cp = 7 1 6 5 4 3 2 0
crt = 90
cs = 1
dre = 8
ep10 = 20
ep16 = 5
ep19 = 5
ep20 = 5
ep21 = 5
ep22 = 5
ep4 = 7542-10
ep8 = 20
immdi = 1
imsio = 0
lm = 4
logic_method = 4
lsp = 3
lsp10 = 0
mhs = 15
mphs = 4
mps = 300
nmp = 2
sa = 2FB
se = 2380
sn = C
srf = FF
srl = 25304
sv = 4.007
to = 15
zone = razor2.cloudmark.com

View File

@@ -0,0 +1,36 @@
#
# Autogenerated by Razor-Agents v2.84, Sun Apr 26 00:23:15 2015
ac = 21
bql = 50
bqs = 329
cp = 7 1 6 5 4 3 2 0
crt = 90
cs = 1
dre = 8
ep10 = 20
ep16 = 5
ep19 = 5
ep20 = 5
ep21 = 5
ep22 = 5
ep4 = 7542-10
ep8 = 20
immdi = 1
imsio = 0
lm = 4
logic_method = 4
lsp = 3
lsp10 = 0
mhs = 15
mphs = 4
mps = 300
nmp = 2
sa = 2FB
se = 2380
sn = C
srf = FF
srl = 25304
sv = 4.007
to = 15
zone = razor2.cloudmark.com

View File

@@ -0,0 +1,31 @@
#
# Autogenerated by Razor-Agents v2.84, Thu Jan 19 12:44:03 2012
ac = 21
bql = 50
bqs = 257
dre = 4
ep10 = 5
ep19 = 5
ep20 = 5
ep21 = 5
ep4 = 7542-10
ep8 = 20
immdi = 1
imsio = 0
lm = 4
logic_method = 4
lsp = 3
lsp10 = 0
mhs = 15
mphs = 4
mps = 128
nmp = 2
sa = 2FB
se = 23C0
sn = N
srf = FF
srl = 9966
sv = 3.112
to = 15
zone = razor2.cloudmark.com

View File

@@ -0,0 +1,3 @@
c301.cloudmark.com
c302.cloudmark.com
c303.cloudmark.com

View File

@@ -0,0 +1 @@
discovery.razor.cloudmark.com

View File

@@ -0,0 +1,4 @@
n004.cloudmark.com
n002.cloudmark.com
n003.cloudmark.com
n001.cloudmark.com

View File

@@ -0,0 +1,5 @@
urirhssub URIBL_BLACK multi.uribl.com. A 2
body URIBL_BLACK eval:check_uridnsbl('URIBL_BLACK')
describe URIBL_BLACK Contains an URL listed in the URIBL blacklist
tflags URIBL_BLACK net
score URIBL_BLACK 3.0

View File

@@ -0,0 +1,8 @@
body LOCAL_ALLOW_RULE /AllowThisEmail/i
score LOCAL_ALLOW_RULE -10
describe LOCAL_ALLOW_RULE Local rule to let emails in.
body LOCAL_ALLOW_RULE_DISCLAIMER /bla bla/i
score LOCAL_ALLOW_RULE_DISCLAIMER -10
describe LOCAL_ALLOW_RULE_DISCLAIMER Local rule to let emails in, if they are a reply with our disclaimer.

View File

@@ -0,0 +1,313 @@
# 2006-10-01 <pille@struction.de>
# URIBL
urirhssub URIBL_BLACK multi.uribl.com. A 2
body URIBL_BLACK eval:check_uridnsbl('URIBL_BLACK')
describe URIBL_BLACK Contains an URL listed in the URIBL blacklist (http://uribl.com)
tflags URIBL_BLACK net
score URIBL_BLACK 2.0
# NIX_SPAM (heise.de)
header NIX_SPAM eval:check_rbl('nix-spam', 'ix.dnsbl.manitu.net')
describe NIX_SPAM Listed in NIX_SPAM DNSBL
tflags NIX_SPAM net
score NIX_SPAM 2.0
# VIRBL (virus sender blacklist) http://virbl.bit.nl
header RCVD_IN_VIRBL eval:check_rbl_txt('virbl', 'virbl.dnsbl.bit.nl')
describe RCVD_IN_VIRBL Listed in virbl.dnsbl.bit.nl
tflags RCVD_IN_VIRBL net
score RCVD_IN_VIRBL 1.0
# 2006-12-19 <pille@struction.de>
# deactivated, since this DB has vanished as of 2006-12-18
# ORDB (open relays) http://ordb.org
#header RCVD_IN_ORDB eval:check_rbl_txt('ordb', 'relays.ordb.org')
#describe RCVD_IN_ORDB Listed in relays.ordb.org
#tflags RCVD_IN_ORDB net
#score RCVD_IN_ORDB 0.5
# CBL (open relays/proxys) http://cbl.abuseat.org
header RCVD_IN_CBL eval:check_rbl_txt('cbl', 'cbl.abuseat.org')
describe RCVD_IN_CBL Listed in cbl.abuseat.org
tflags RCVD_IN_CBL net
score RCVD_IN_CBL 2.0
# UCEPROTECT1 (open relays/proxys/dialups) http://uceprotect.net
header RCVD_IN_UCEPROTECT1 eval:check_rbl_txt('uceprotect1', 'dnsbl-1.uceprotect.net')
describe RCVD_IN_UCEPROTECT1 Listed in dnsbl-1.uceprotect.net
tflags RCVD_IN_UCEPROTECT1 net
score RCVD_IN_UCEPROTECT1 1.0
# UCEPROTECT2 (open relays/proxys/dialups networks) http://uceprotect.net
header RCVD_IN_UCEPROTECT2 eval:check_rbl_txt('uceprotect1', 'dnsbl-2.uceprotect.net')
describe RCVD_IN_UCEPROTECT2 Network listed in dnsbl-2.uceprotect.net
tflags RCVD_IN_UCEPROTECT2 net
score RCVD_IN_UCEPROTECT2 0.5
# UCEPROTECT3 (bad networks) http://uceprotect.net
header RCVD_IN_UCEPROTECT3 eval:check_rbl_txt('uceprotect1', 'dnsbl-3.uceprotect.net')
describe RCVD_IN_UCEPROTECT3 Network listed in dnsbl-3.uceprotect.net
tflags RCVD_IN_UCEPROTECT3 net
score RCVD_IN_UCEPROTECT3 0.1
# DSBL-multihop (multihop open relays) http://dsbl.org
header RCVD_IN_DSBL_MULTIHOP eval:check_rbl_txt('dsblmultihop', 'multihop.dsbl.org')
describe RCVD_IN_DSBL_MULTIHOP Listed in multihop.dsbl.org
tflags RCVD_IN_DSBL_MULTIHOP net
score RCVD_IN_DSBL_MULTIHOP 0.1
# DSBL-unconfirmed (open relays) http://dsbl.org
header RCVD_IN_DSBL_UNCONFIRMED eval:check_rbl_txt('dsblunconfirmed', 'unconfirmed.dsbl.org')
describe RCVD_IN_DSBL_UNCONFIRMED Listed in unconfirmed.dsbl.org
tflags RCVD_IN_DSBL_UNCONFIRMED net
score RCVD_IN_DSBL_UNCONFIRMED 0.001
# AHBL-tor (TOR relays) http://ahbl.org
header RCVD_IN_AHBL_TOR eval:check_rbl_txt('ahbltor', 'tor.ahbl.org')
describe RCVD_IN_AHBL_TOR Listed in tor.ahbl.org
tflags RCVD_IN_AHBL_TOR net
score RCVD_IN_AHBL_TOR 0.001
# AHBL-exemptions (whitelist) http://ahbl.org
header RCVD_IN_AHBL_WHITELIST eval:check_rbl_txt('ahblwhite', 'exemptions.ahbl.org')
describe RCVD_IN_AHBL_WHITELIST WhiteListed in exemptions.ahbl.org
tflags RCVD_IN_AHBL_WHITELIST net
score RCVD_IN_AHBL_WHITELIST -0.01
# from http://www.ahbl.org/docs/mailservers/spamassassin.txt
header RCVD_IN_AHBL eval:check_rbl('AHBL', 'dnsbl.ahbl.org.')
describe RCVD_IN_AHBL AHBL: sender is listed in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL 1.0
tflags RCVD_IN_AHBL net
header RCVD_IN_AHBL_UNKNOWN_1 eval:check_rbl_sub('AHBL', '127.0.0.1')
describe RCVD_IN_AHBL_UNKNOWN_1 AHBL: Unknown Category 1 in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_UNKNOWN_1 0.01
tflags RCVD_IN_AHBL_UNKNOWN_1 net
header RCVD_IN_AHBL_SMTP eval:check_rbl_sub('AHBL', '127.0.0.2')
describe RCVD_IN_AHBL_SMTP AHBL: Open SMTP relay in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_SMTP 0.5
tflags RCVD_IN_AHBL_SMTP net
header RCVD_IN_AHBL_PROXY eval:check_rbl_sub('AHBL', '127.0.0.3')
describe RCVD_IN_AHBL_PROXY AHBL: Open Proxy server in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_PROXY 0.5
tflags RCVD_IN_AHBL_PROXY net
header RCVD_IN_AHBL_SPAM eval:check_rbl_sub('AHBL', '127.0.0.4')
describe RCVD_IN_AHBL_SPAM AHBL: Spam Source in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_SPAM 0.5
tflags RCVD_IN_AHBL_SPAM net
header RCVD_IN_AHBL_RTB eval:check_rbl_sub('AHBL', '127.0.0.5')
describe RCVD_IN_AHBL_RTB AHBL: Real-Time Blocked in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_RTB 0.01
tflags RCVD_IN_AHBL_RTB net
header RCVD_IN_AHBL_FORMMAIL eval:check_rbl_sub('AHBL', '127.0.0.6')
describe RCVD_IN_AHBL_FORMMAIL AHBL: Abuseable Form Mail in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_FORMMAIL 0.5
tflags RCVD_IN_AHBL_FORMMAIL net
header RCVD_IN_AHBL_SPAM_SUPPORT eval:check_rbl_sub('AHBL', '127.0.0.7')
describe RCVD_IN_AHBL_SPAM_SUPPORT AHBL: Spam Supporter in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_SPAM_SUPPORT 0.5
tflags RCVD_IN_AHBL_SPAM_SUPPORT net
header RCVD_IN_AHBL_I_SPAM_SUPPORT eval:check_rbl_sub('AHBL', '127.0.0.8')
describe RCVD_IN_AHBL_I_SPAM_SUPPORT AHBL: Indirect Spam supporter in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_I_SPAM_SUPPORT 0.5
tflags RCVD_IN_AHBL_I_SPAM_SUPPORT net
header RCVD_IN_AHBL_ENDUSER eval:check_rbl_sub('AHBL', '127.0.0.9')
describe RCVD_IN_AHBL_ENDUSER AHBL: End User (non mail system) in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_ENDUSER 0.5
tflags RCVD_IN_AHBL_ENDUSER net
header RCVD_IN_AHBL_SOS eval:check_rbl_sub('AHBL-notfirsthop', '127.0.0.10')
describe RCVD_IN_AHBL_SOS AHBL: Shoot On Sight in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_SOS 0.5
tflags RCVD_IN_AHBL_SOS net
header RCVD_IN_AHBL_RFCI_PA eval:check_rbl_sub('AHBL', '127.0.0.11')
describe RCVD_IN_AHBL_RFCI_PA AHBL: Missing Postmaster or Abuse Address in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_RFCI_PA 0.5
tflags RCVD_IN_AHBL_RFCI_PA net
header RCVD_IN_AHBL_5XXI eval:check_rbl_sub('AHBL', '127.0.0.12')
describe RCVD_IN_AHBL_5XXI AHBL: Does not properly handle 5xx errors in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_5XXI 0.5
tflags RCVD_IN_AHBL_5XXI net
header RCVD_IN_AHBL_RFCI_MISC eval:check_rbl_sub('AHBL', '127.0.0.13')
describe RCVD_IN_AHBL_RFCI_MISC AHBL: Other Non-RFC Compliant in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_RFCI_MISC 0.5
tflags RCVD_IN_AHBL_RFCI_MISC net
header RCVD_IN_AHBL_COMP_DDOS eval:check_rbl_sub('AHBL', '127.0.0.14')
describe RCVD_IN_AHBL_COMP_DDOS AHBL: Compromised System - DDoS in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_COMP_DDOS 0.5
tflags RCVD_IN_AHBL_COMP_DDOS net
header RCVD_IN_AHBL_COMP_RELAY eval:check_rbl_sub('AHBL', '127.0.0.15')
describe RCVD_IN_AHBL_COMP_RELAY AHBL: Compromised System - Relay in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_COMP_RELAY 0.5
tflags RCVD_IN_AHBL_COMP_RELAY net
header RCVD_IN_AHBL_COMP_SCANNER eval:check_rbl_sub('AHBL', '127.0.0.16')
describe RCVD_IN_AHBL_COMP_SCANNER AHBL: Compromised System - Autorooter/Scanner in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_COMP_SCANNER 0.5
tflags RCVD_IN_AHBL_COMP_SCANNER net
header RCVD_IN_AHBL_COMP_WORM eval:check_rbl_sub('AHBL', '127.0.0.17')
describe RCVD_IN_AHBL_COMP_WORM AHBL: Compromised System - Worm or mass mailing virus in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_COMP_WORM 0.5
tflags RCVD_IN_AHBL_COMP_WORM net
header RCVD_IN_AHBL_COMP_VIRUS eval:check_rbl_sub('AHBL', '127.0.0.18')
describe RCVD_IN_AHBL_COMP_VIRUS AHBL: Compromised System - Other Virus in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_COMP_VIRUS 0.5
tflags RCVD_IN_AHBL_COMP_VIRUS net
header RCVD_IN_AHBL_PROXY eval:check_rbl_sub('AHBL', '127.0.0.19')
describe RCVD_IN_AHBL_PROXY AHBL: Open Proxy in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_PROXY 0.5
tflags RCVD_IN_AHBL_PROXY net
header RCVD_IN_AHBL_BLOG eval:check_rbl_sub('AHBL', '127.0.0.19')
describe RCVD_IN_AHBL_BLOG AHBL: Blog/Wiki/Comment Spammer in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_BLOG 0.5
tflags RCVD_IN_AHBL_BLOG net
header RCVD_IN_AHBL_MISC eval:check_rbl_sub('AHBL', '127.0.0.127')
describe RCVD_IN_AHBL_MISC AHBL: Misc (other) in BlackList / BlockList dnsbl.ahbl.org
score RCVD_IN_AHBL_MISC 0.5
tflags RCVD_IN_AHBL_MISC net
# bondedsender whitelist (commercial?) http://www.returnpath.org/senderscorecertified
header RCVD_IN_BONDEDSENDER_WHITELIST eval:check_rbl('bondedsender', 'sa.bondedsender.org')
describe RCVD_IN_BONDEDSENDER_WHITELIST Received via a whitelisted Bonded Sender address
score RCVD_IN_BONDEDSENDER_WHITELIST -0.001
tflags RCVD_IN_BONDEDSENDER_WHITELIST net
header RCVD_IN_BONDEDSENDER_WHITELIST1 eval:check_rbl('bondedsender1', 'query.bondedsender.org', '127.0.0.10')
describe RCVD_IN_BONDEDSENDER_WHITELIST1 Received via a whitelisted Bonded Sender address
score RCVD_IN_BONDEDSENDER_WHITELIST1 -0.001
tflags RCVD_IN_BONDEDSENDER_WHITELIST1 net
# test, if we catch dialup-relays (additional to standard spamassassin)
header RCVD_IN_NJABL_DUL2 eval:check_rbl('njabl2-notfirsthop', 'combined.njabl.org.', '127.0.0.3')
describe RCVD_IN_NJABL_DUL2 NJABL: dialup sender did non-local SMTP
score RCVD_IN_NJABL_DUL2 0.1
tflags RCVD_IN_NJABL_DUL2 net
header RCVD_IN_MAPS_DUL2 eval:check_rbl('dialup2-notfirsthop', 'dialups.mail-abuse.org.')
describe RCVD_IN_MAPS_DUL2 Relay in DUL, http://www.mail-abuse.org/dul/
score RCVD_IN_MAPS_DUL2 0.1
tflags RCVD_IN_MAPS_DUL2 net
header RCVD_IN_SORBS_DUL2 eval:check_rbl('sorbs2-notfirsthop', 'dnsbl.sorbs.net.', '127.0.0.10')
describe RCVD_IN_SORBS_DUL2 SORBS: sent directly from dynamic IP address
tflags RCVD_IN_SORBS_DUL2 net
score RCVD_IN_SORBS_DUL2 0.1
# FIVETENSG http://www.five-ten-sg.com
header RCVD_IN_FIVETENSG eval:check_rbl('FIVETENSG', 'blackholes.five-ten-sg.com.')
describe RCVD_IN_FIVETENSG sender is listed in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG 1.0
tflags RCVD_IN_FIVETENSG net
header RCVD_IN_FIVETENSG_UNKNOWN_1 eval:check_rbl_sub('FIVETENSG', '127.0.0.1')
describe RCVD_IN_FIVETENSG_UNKNOWN_1 Unknown Category 1 in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_UNKNOWN_1 0.001
tflags RCVD_IN_FIVETENSG_UNKNOWN_1 net
header RCVD_IN_FIVETENSG_SPAM eval:check_rbl_sub('FIVETENSG', '127.0.0.2')
describe RCVD_IN_FIVETENSG_SPAM Spammer in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_SPAM 0.5
tflags RCVD_IN_FIVETENSG_SPAM net
header RCVD_IN_FIVETENSG_DUL eval:check_rbl_sub('FIVETENSG', '127.0.0.3')
describe RCVD_IN_FIVETENSG_DUL Dialup in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_DUL 0.01
tflags RCVD_IN_FIVETENSG_DUL net
header RCVD_IN_FIVETENSG_BULK eval:check_rbl_sub('FIVETENSG', '127.0.0.4')
describe RCVD_IN_FIVETENSG_BULK Bulk-Mailer in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_BULK 0.01
tflags RCVD_IN_FIVETENSG_BULK net
header RCVD_IN_FIVETENSG_MULTISTAGE eval:check_rbl_sub('FIVETENSG', '127.0.0.5')
describe RCVD_IN_FIVETENSG_MULTISTAGE Multistage Open Relay in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_MULTISTAGE 0.1
tflags RCVD_IN_FIVETENSG_MULTISTAGE net
header RCVD_IN_FIVETENSG_SINGLESTAGE eval:check_rbl_sub('FIVETENSG', '127.0.0.6')
describe RCVD_IN_FIVETENSG_SINGLESTAGE Singlestage Open Relay in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_SINGLESTAGE 0.1
tflags RCVD_IN_FIVETENSG_SINGLESTAGE net
header RCVD_IN_FIVETENSG_SUPPORT eval:check_rbl_sub('FIVETENSG', '127.0.0.7')
describe RCVD_IN_FIVETENSG_SUPPORT Spam-Supporter in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_SUPPORT 0.1
tflags RCVD_IN_FIVETENSG_SUPPORT net
header RCVD_IN_FIVETENSG_WEBFORM eval:check_rbl_sub('FIVETENSG', '127.0.0.8')
describe RCVD_IN_FIVETENSG_WEBFORM Web2Mail in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_WEBFORM 0.1
tflags RCVD_IN_FIVETENSG_WEBFORM net
header RCVD_IN_FIVETENSG_SUSPECT eval:check_rbl_sub('FIVETENSG', '127.0.0.9')
describe RCVD_IN_FIVETENSG_SUSPECT Suspected system in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_SUSPECT 0.01
tflags RCVD_IN_FIVETENSG_SUSPECT net
header RCVD_IN_FIVETENSG_KLEZ eval:check_rbl_sub('FIVETENSG', '127.0.0.10')
describe RCVD_IN_FIVETENSG_KLEZ Virus Notification Sender in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_KLEZ 0.01
tflags RCVD_IN_FIVETENSG_KLEZ net
header RCVD_IN_FIVETENSG_FREEMAIL eval:check_rbl_sub('FIVETENSG', '127.0.0.12')
describe RCVD_IN_FIVETENSG_FREEMAIL Freemailer in blackholes.five-ten-sg.com
score RCVD_IN_FIVETENSG_FREEMAIL 0.01
tflags RCVD_IN_FIVETENSG_FREEMAIL net
# bl.csma.biz - Repeat SPAM Sources
header RCVD_IN_BLCSMA eval:check_rbl('blcsma', 'bl.csma.biz.')
describe RCVD_IN_BLCSMA Received via a blocked site in bl.csma.biz
score RCVD_IN_BLCSMA 0.5
tflags RCVD_IN_BLCSMA net
# sbl.csma.biz - Suspect SPAM Sources
header RCVD_IN_SBLCSMA eval:check_rbl('sblcsma', 'sbl.csma.biz.')
describe RCVD_IN_SBLCSMA Received via a blocked site in sbl.csma.biz
score RCVD_IN_SBLCSMA 0.1
tflags RCVD_IN_SBLCSMA net

View File

@@ -0,0 +1,45 @@
# 2006-11-09 <pille@struction.de>
# these rules check for headers placed by exim
header EXIM_SENDER_VERIFY_FAILED X-Sender-Verify =~ /FAILED/
describe EXIM_SENDER_VERIFY_FAILED Sender Address does not accept mail
score EXIM_SENDER_VERIFY_FAILED 2.0
header EXIM_SENDER_VERIFY_SUCCEEDED X-Sender-Verify =~ /SUCCEEDED/
describe EXIM_SENDER_VERIFY_SUCCEEDED Sender Address accepts mail
score EXIM_SENDER_VERIFY_SUCCEEDED -0.1
#header EXIM_SENDER_VERIFY_HEADER exists:X-Sender-Verify
#describe EXIM_SENDER_VERIFY_HEADER header Sender Verify exists
#score EXIM_SENDER_VERIFY_HEADER 0.1
header __EXIM_AUTH1 exists:X-Authenticated-User
header __EXIM_AUTH2 exists:X-Authenticator
meta EXIM_AUTH __EXIM_AUTH1 && __EXIM_AUTH2
describe EXIM_AUTH Sender is authenticated
score EXIM_AUTH -4.0
header __EXIM_HELO_MISSING X-Invalid-HELO =~ /no HELO/
header __EXIM_HELO_NO_FQDN X-Invalid-HELO =~ /HELO is no FQDN/
meta EXIM_HELO_MISSING __EXIM_HELO_MISSING
describe EXIM_HELO_MISSING (E)HELO is missing
score EXIM_HELO_MISSING 0.1
# as exim identifies no FQDN by using "negative hits", we have to ensure, a helo was issued
meta EXIM_HELO_NO_FQDN __EXIM_HELO_NO_FQDN && !__EXIM_HELO_MISSING
describe EXIM_HELO_NO_FQDN (E)HELO is no Fully Qualified Domain Name
score EXIM_HELO_NO_FQDN 1.5
header EXIM_HELO_IP X-Invalid-HELO =~ /HELO is IP only/
describe EXIM_HELO_IP (E)HELO is IP only (not in brackets)
score EXIM_HELO_IP 1.0
header EXIM_HELO_IMPERSONTING X-Invalid-HELO =~ /Host impersonating /
describe EXIM_HELO_IMPERSONTING (E)HELO is impersonating our mailserver
score EXIM_HELO_IMPERSONTING 4.0
header EXIM_HELO_MY_ADDRESS X-Invalid-HELO =~ /is _my_ address/
describe EXIM_HELO_MY_ADDRESS (E)HELO using mailserver's address
score EXIM_HELO_MY_ADDRESS 4.0

View File

@@ -0,0 +1,25 @@
loadplugin Mail::SpamAssassin::Plugin::iXhash /etc/mail/spamassassin/iXhash.pm
# This makes DNS queries time out after 10 seconds (2x default)
ixhash_timeout 10
# This list uses iX Magazine's spam as datasource.
body IXHASH1 eval:ixhashtest('ix.dnsbl.manitu.net')
describe IXHASH1 This mail has been classified as spam @ iX Magazine, Germany
tflags IXHASH1 net
score IXHASH1 2.5
# This list comes in @ spamtraps run by former LogIn & Solutions AG, Germany
body IXHASH2 eval:ixhashtest('generic.ixhash.net')
describe IXHASH2 mail has been classified as spam @ former LogIn&Solutions AG, Germany
tflags IXHASH2 net
score IXHASH2 1.5
body IXHASH3 eval:ixhashtest('ctyme.ixhash.net')
describe IXHASH3 mail has been classified as spam @ JunkEmailFilter, Germany
tflags IXHASH3 net
score IXHASH3 1.0
body IXHASH4 eval:ixhashtest('hosteurope.ixhash.net')
describe IXHASH4 mail has been classified as spam @ HostEurope, Germany
tflags IXHASH4 net
score IXHASH4 1.0

View File

@@ -0,0 +1,67 @@
#*************************************************************************
# Bayes OCR Plugin, version 0.1
#*************************************************************************
# Copyright 2007 P.R.A. Group - D.I.E.E. - University of Cagliari (ITA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#*************************************************************************
loadplugin BayesOCR_PLG BayesOCR_PLG.pm
# Cerberus guarded the gate to Hades and ensured
# that spirits of the dead could enter...
# BayesOCR Plugin guards the inboxes and ensures
# that only legitimate images can enter,
# spam images are detected and eated..
# Rule: BayesOCR_check(thr)
# Categorisation of text embedded in images with TextCategorisation techniques.
# Require gocr, convert (imagemagick)
body BayesOCR_PLG40 eval:BayesOCR_check(0.40, 0.50)
body BayesOCR_PLG50 eval:BayesOCR_check(0.50, 0.60)
body BayesOCR_PLG60 eval:BayesOCR_check(0.60, 0.70)
body BayesOCR_PLG70 eval:BayesOCR_check(0.70, 0.80)
body BayesOCR_PLG80 eval:BayesOCR_check(0.80, 0.90)
body BayesOCR_PLG90 eval:BayesOCR_check(0.90, 0.95)
body BayesOCR_PLG95 eval:BayesOCR_check(0.95, 0.99)
body BayesOCR_PLG99 eval:BayesOCR_check(0.99, 1.00)
describe BayesOCR_PLG40 Bayesian ImageSpam probability is 40% to 50%
describe BayesOCR_PLG50 Bayesian ImageSpam probability is 50% to 60%
describe BayesOCR_PLG60 Bayesian ImageSpam probability is 60% to 70%
describe BayesOCR_PLG70 Bayesian ImageSpam probability is 70% to 80%
describe BayesOCR_PLG80 Bayesian ImageSpam probability is 80% to 90%
describe BayesOCR_PLG90 Bayesian ImageSpam probability is 90% to 95%
describe BayesOCR_PLG95 Bayesian ImageSpam probability is 95% to 99%
describe BayesOCR_PLG99 Bayesian ImageSpam probability is 99% to 100%
add_header all BayesOCR-OUT _PLGBAYESOCROUT_
priority BayesOCR_PLG40 1000
priority BayesOCR_PLG50 1000
priority BayesOCR_PLG60 1000
priority BayesOCR_PLG70 1000
priority BayesOCR_PLG80 1000
priority BayesOCR_PLG90 1000
priority BayesOCR_PLG95 1000
priority BayesOCR_PLG99 1000
score BayesOCR_PLG40 0 0 0.5 0.5
score BayesOCR_PLG50 0 0 1.0 1.0
score BayesOCR_PLG60 0 0 1.5 1.5
score BayesOCR_PLG70 0 0 2.0 2.0
score BayesOCR_PLG80 0 0 2.7 2.7
score BayesOCR_PLG90 0 0 3.5 3.5
score BayesOCR_PLG95 0 0 4.0 4.0
score BayesOCR_PLG99 0 0 4.5 4.5

View File

@@ -0,0 +1,400 @@
#*************************************************************************
# Bayes OCR Plugin, version 0.1
#*************************************************************************
# Copyright 2007 P.R.A. Group - D.I.E.E. - University of Cagliari (ITA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#*************************************************************************
package BayesOCR_PLG;
use strict;
use Mail::SpamAssassin;
use Mail::SpamAssassin::Util;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
our @ISA = qw (Mail::SpamAssassin::Plugin);
# constructor: register the eval rule
sub new {
my ( $class, $mailsa ) = @_;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsa);
bless( $self, $class );
dbg("PLG-BayesOCR:: new:: register_eval_rule");
$self->register_eval_rule("BayesOCR_check");
$self->{'imgTxt_classifierOut'} = -1;
$self->{'imgTxt_tagmsg'} = ""; #msg to be saved in e-mail tag when $self->{'imgTxt_classifierOut'} <= 0
return $self;
}
#===========================================================================
#===========================================================================
sub check_start{
# Called before eval rule
my ( $self, $pms ) = @_;
dbg("PLG-BayesOCR:: check_start:: init score");
#Init outNB_imgTxt
$self->{'imgTxt_classifierOut'} = -1;
$self->{'imgTxt_tagmsg'} = "";
}
sub isValidUser{
my ($pms) = @_;
my $username = $pms->{main}->{username};
dbg("PLG-BayesOCR:: isValidUser:: Username: $username");
return 1;
}
sub BayesOCR_check {
# BayesOCR_check(thr)
# Return an hit when (outNB > thr)
# The score is computed as (weigth * outNB)
#
my ($self, $pms, $unused, $thrL, $thrH) = @_;
my $plgRuleName = $pms->get_current_eval_rule_name();
#if( isValidUser($pms) == 0) { return 0; }
dbg("PLG-BayesOCR:: BayesOCR_check :: Rule: $plgRuleName");
dbg("PLG-BayesOCR:: BayesOCR_check :: thr: ($thrH, $thrL)");
if($self->{'imgTxt_classifierOut'} < 0)
{
#Output
if( $self->imageSpam_OCRTextProcessing($pms ) )
{
$self->{'imgTxt_tagmsg'} = $self->{'imgTxt_classifierOut'};
}
dbg("PLG-BayesOCR:: BayesOCR_check:: Write Mail Header\n\n");
$pms->set_tag ("PLGBAYESOCROUT", $self->{'imgTxt_tagmsg'} );
}
my $resHit = ($self->{'imgTxt_classifierOut'} > $thrL) && ($self->{'imgTxt_classifierOut'} <= $thrH );
return $resHit;
}
1;
#===========================================================================
sub imageSpam_OCRTextProcessing
# boolen $self->imageSpam_OCRTextProcessing($pms)
#
# imageSpam processing by image's text analisys with SA's NaiveBayes
# return 1 : (sucess) image's text has beeen extract and processed by NB
# return 0 : (failed) no images, no text, no NB.
{
my ( $self, $pms ) = @_;
# $self :: Obj Plugin
# $pms :: Obj Mail::SpamAssassin::PerMsgStatus
# $pms->{msg} :: message of class Mail::SpamAssassin::Message
#================================
# Init result
#================================
$self->{'imgTxt_classifierOut'} = 0;
#================================
# Check & Create Classifier
#================================
my $nbSA = $pms->{main}->{bayes_scanner};
#my $nbSA = new Mail::SpamAssassin::Bayes ($pms->{main});
if( $nbSA->is_scan_available() == 0)
{
dbg("PLG-BayesOCR:: imageTextClassifierOutEstimation: NB scan not available");
$self->{'imgTxt_tagmsg'} = "0.0 (NaiveBayes not available)";
return 0;
}
#================================
# Image extraction
#================================
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: Check for Attached Images");
my ($imgTextOcr, $numImages) = imageTextExtractionFromMSG($pms->{msg});
if($numImages == 0)
{
$self->{'imgTxt_tagmsg'} = "0.0 (No images found)";
return 0;
}
# Check extracted text
my $numWord = 0;
while($imgTextOcr =~ /[a-z]{3,}/gi)
{
$numWord++;
}
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: $numWord words (3+ chars) recognised");
if($numWord <= 3)
{
$self->{'imgTxt_tagmsg'} = "0.0 (No usefull text found)";
return 0;
}
#================================
# Classifier's output estimation
#================================
# creation of msg with image's text
my $mailraw = createMSGFromText($pms, $imgTextOcr);
my $msgTmp = $pms->{main}->parse($mailraw,1);
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: Compute score with trained NaiveBayes");
my $pmsTMP = new Mail::SpamAssassin::PerMsgStatus($pms->{main}, $msgTmp);
# Classification
my $outNB = $nbSA->scan($pmsTMP, $msgTmp);
$self->{'imgTxt_classifierOut'} = sprintf("%0.3f", $outNB);
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: classifier's out = $self->{'imgTxt_classifierOut'}" );
return 1; # All OK
}
#===========================================================================
sub imageTextExtractionFromMSG
# ($imgTextOcr, $numImages) = imageTextExtractionFromMSG($msg)
# Extract the text from all attached images
# Return all text anche the number of attached images
{
my $msg = $_[0];
dbg("PLG-BayesOCR:: imageTextExtractionFromMSG:: Extract & Convert Images");
my @mimeStr = ("image/*", "img/*");
my @tmpImgFile;
my $num=0;
my $imgTextOcr = "";
foreach (@mimeStr)
{
# Search all attach with current MIME
my @img_parts = $msg->find_parts($_);
for (my $i=0; $i <= $#img_parts; $i++)
{
my $imagestream = $img_parts[$i]->decode(1048000); # ~ 1 MB
$imgTextOcr = join $imgTextOcr, imageTextExtractionByOCR($imagestream), "\n";
$num++;
}
}
dbg("PLG-BayesOCR:: imageTextExtractionFromMSG:: $num images extracted");
return ($imgTextOcr, $num);
}
#===========================================================================
sub imageTextExtractionByOCR
# $textOut = imageTextExtractionByOCR( $imagestream )
# Text extraction from imge file "" by OCR engine
{
my $imagestream = $_[0];
my $imagelen = length($imagestream) / 1024;
my $tmpDir = "/tmp"; #Get tmp dir
my $tmpFile = "$tmpDir/sa_bayesOCR_tmpImg.$$";
# Zooming small images could improve OCR accuracy
# Byte Check
# > 1000K => no OCR
# < 15K => OCR + zoom 4X
# else => Check resolution
# Check resolution
# res > 1400x1050 => no OCR
# 1024x768 <= res < 1400x1050 => OCR (no zoom)
# 800x600 <= res < 1024x768 => OCR + zoom 2X
# res < 800x600 => OCR + zoom 4X
if ($imagelen > 1000)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Skip, image size = $imagelen");
return "";
}
open (FILE, ">$tmpFile.tmp") or return "";
print FILE "$imagestream \n";
close FILE;
my $convertOPT = "";
my $imageIdentifyTxt = "";
if($imagelen < 20 )
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 4X");
$convertOPT = "-sample 400% -density 280";
}
else
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Check image dim");
# check WxH
open EXEFH, "identify -quiet -ping $tmpFile.tmp |";
$imageIdentifyTxt = join "", <EXEFH>;
close EXEFH;
if( $imageIdentifyTxt =~ s/\s(\d*)x(\d*)\s//i )
{
my $size1 = $1;
my $size2 = $2;
if($size1 * $size2 > 1400*1050 && $size1 > 1280 && $size2 > 1024)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Skip, image dim = $size1 x $size2");
unlink "$tmpFile.tmp";
return "";
}
if( $size1 * $size2 < 800*600)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 4X");
$convertOPT = "-sample 400% -density 280";
}
elsif( $size1 * $size2 < 1024*768)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 2X");
$convertOPT = "-sample 200% -density 280";
}
}
}
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Convert & OCR");
# -append :: concatenate image i layers
# -flatten :: fuse layers
# -density :: set dpi
my $exstatus = system("convert $tmpFile.tmp -append -flatten $convertOPT $tmpFile.pnm");
if($exstatus != 0)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Convert ERROR!!");
#Catturo SDOUT e STERR
open EXEFH, "identify -verbose -strip $tmpFile.tmp 2>&1 |";
$imageIdentifyTxt = join "", <EXEFH>;
close EXEFH;
my $msg = "Stream size (kb): $imagelen\nIdentify output: \n$imageIdentifyTxt\n";
saveLogMsg($tmpDir, "Convert Error", $msg);
unlink "$tmpFile.tmp";
return "";
}
# GOCR call with timeout (thanks to B. Austin for the usefull suggestions)
my $textOut = "";
eval {
local $SIG{ALRM} = sub { die "GOCR_TIMEOUT\n" };
alarm 10;
# Retrieve gocr output
open EXEFH, "gocr $tmpFile.pnm |";
$textOut = join "", <EXEFH>;
close EXEFH;
alarm 0;
};
if ($@) {
die unless $@ eq "GOCR_TIMEOUT\n"; # propagate unexpected errors
# timed out
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: OCR timeout!!");
# Extract the list of all child of this process
open PSFH, "ps -o pid,cmd --ppid $$ |";
my $psOut = join "", <PSFH>;
close PSFH;
#Get the PID of gocr child
if( $psOut =~ s/(\d*) gocr//i)
{
kill 9, $1;
}
my $msg = "Stream size (kb): $imagelen\nPS out:\n $psOut\n";
saveLogMsg($tmpDir, "OCR timeout", $msg);
$textOut = "";
}
unlink "$tmpFile.tmp";
unlink "$tmpFile.pnm";
return $textOut;
}
#===========================================================================
sub createMSGFromText
# msg = createMSGFromText(@img_ocrText)
{
my ($pms, $ocrText) = @_;
dbg("PLG-BayesOCR: createMSGFromText:: Make temp email with OCR's text");
my $subject = "";
my $date = $pms->{msg}->get_pristine_header("Date");
my $from = ""; #$pms->{msg}->get_pristine_header("From");
my $to = ""; #$pms->{msg}->get_pristine_header("To");
my $mailraw = "From: $from\nTo: $to\nSubject: $subject\nDate: $date\nContent-Type: text/plain;\n charset=\"us-ascii\"\nContent-Disposition: inline\n\n$ocrText\n";
return $mailraw
}
#===========================================================================
#===========================================================================
sub saveLogMsg()
{
my ($tmpDir, $title, $msg) = @_;
my $timenow = localtime time;
open (FILE, ">>$tmpDir/sa_bayesOCR.log");
print FILE "#--------------------------------\n";
print FILE " $timenow\n";
print FILE " $title\n";
print FILE "#--------------------------------\n";
print FILE "$msg\n";
close FILE;
}
#===========================================================================

273
mail/spamassassin/DNSWLh.pm Normal file
View File

@@ -0,0 +1,273 @@
# Adds DNSWL.org to recipients of spamassassin --report.
#
# In a SpamAssassin config file, add the lines:
#
# loadplugin Mail::SpamAssassin::Plugin::DNSWLh
# dnswl_address user@example.com
# dnswl_password yourpassword
#
# The last two must be from an account created via
# http://www.dnswl.org/registerreporter.pl
#
#
# 2010-02-26-23 Initial release.
# 2010-02-27-11 Also call report successful on unlisted IPs.
# 2010-02-28-20 State when reported email has trust level "Unlisted".
# 2010-03-02-10 Report the IP DNSWL thought was interesting.
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>
=head1 NAME
Mail::SpamAssassin::Plugin::DNSWL - perform DNSWL reporting of messages
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::DNSWL
=head1 DESCRIPTION
DNSWL is a service which lists known legitimate mail servers.
This module enables automatic reporting of spam to DNSWL, to improve
the accuracy of their database.
Note that spam reports sent by this plugin to DNSWL each include the
entire spam message.
See http://www.dnswl.org/ for more information about DNSWL.
=cut
package Mail::SpamAssassin::Plugin::DNSWLh;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
use IO::Socket;
use strict;
use warnings;
use bytes;
use re 'taint';
use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; };
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
sub new {
my $class = shift;
my $mailsaobject = shift;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsaobject);
bless ($self, $class);
# are network tests enabled?
if (!$mailsaobject->{local_tests_only} && HAS_LWP_USERAGENT) {
$self->{dnswl_available} = 1;
dbg("DNSWL: network tests on, attempting DNSWL");
}
else {
$self->{dnswl_available} = 0;
dbg("DNSWL: local tests only, disabling DNSWL");
}
$self->set_config($mailsaobject->{conf});
return $self;
}
sub set_config {
my($self, $conf) = @_;
my @cmds;
=head1 USER OPTIONS
=over 4
=cut
push (@cmds, {
setting => 'dnswl_address',
default => 'spamassassin-submit@spam.dnswl.chaosreigns.com',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^([^<\s]+\@[^>\s]+)$/) {
$self->{dnswl_address} = $1;
}
elsif ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
else {
return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
},
});
push (@cmds, {
setting => 'dnswl_password',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^(\S+)$/) {
$self->{dnswl_password} = $1;
}
elsif ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
else {
return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
},
});
=item dnswl_max_report_size (default: 50)
Messages larger than this size (in kilobytes) will be truncated in
report messages sent to DNSWL. The default setting is the maximum
size that DNSWL will accept at the time of release.
=cut
push (@cmds, {
setting => 'dnswl_max_report_size',
default => 50,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
});
$conf->{parser}->register_commands(\@cmds);
}
sub plugin_report {
my ($self, $options) = @_;
return unless $self->{dnswl_available};
#dbg("DNSWL: address/pass: " . $options->{report}->{conf}->{dnswl_address}
# .' '. $options->{report}->{conf}->{dnswl_password} );
if (!$options->{report}->{options}->{dont_report_to_dnswl}) {
if ($options->{report}->{conf}->{dnswl_address} and
$options->{report}->{conf}->{dnswl_password}) {
if ($self->dnswl_report($options)) {
$options->{report}->{report_available} = 1;
info("DNSWL: spam reported to DNSWL");
$options->{report}->{report_return} = 1;
} else {
info("DNSWL: could not report spam to DNSWL");
}
} else {
dbg("DNSWL: dnswl_address and/or dnswl_password not defined.");
}
}
}
sub dnswl_report {
my ($self, $options) = @_;
# original text
my $original = ${$options->{text}};
# check date
my $header = $original;
$header =~ s/\r?\n\r?\n.*//s;
my $date = Mail::SpamAssassin::Util::receive_date($header);
if ($date && $date < time - 2*86400) {
warn("DNSWL: Message older than 2 days, not reporting\n");
return 0;
}
# message variables
my $description = "spam report via " . Mail::SpamAssassin::Version();
my $trusted = $options->{msg}->{metadata}->{relays_trusted_str};
my $untrusted = $options->{msg}->{metadata}->{relays_untrusted_str};
# message data
# truncate message
if (length($original) > $self->{main}->{conf}->{dnswl_max_report_size} * 1024) {
substr($original, ($self->{main}->{conf}->{dnswl_max_report_size} * 1024)) =
"\n[truncated by SpamAssassin]\n";
}
my $body = <<"EOM";
Content-Description: $description
X-Spam-Relays-Trusted: $trusted
X-Spam-Relays-Untrusted: $untrusted
$original
EOM
# compose message
my $message;
$message = $body;
# send message
my %form = (
'action', 'save',
'abuseReport',$message,
);
my $ua = LWP::UserAgent->new;
my $netloc = 'www.dnswl.org:80';
my $realm = 'dnswl.org Abuse Reporting';
$ua->credentials( $netloc, $realm, $options->{report}->{conf}->{dnswl_address}, $options->{report}->{conf}->{dnswl_password} );
my $response = $ua->post('http://www.dnswl.org/abuse/report.pl', \%form);
# my $response = $ua->post('http://www.dnswl.org/abuse/report.test.pl', \%form);
# open OUT, ">/tmp/dnswlbody.".time.".txt";
# print OUT $form{'abuseReport'};
# close OUT;
if ($response->is_success) {
#if ( $response->content =~ m#Thank you for your report# ) {
if ( $response->content =~ m#IP ([\d\.]+) matches with DNSWL# ) {
my $reportedip = $1;
dbg("DNSWL: Successfully reported $reportedip.");
print "Successfully reported to DNSWL $reportedip.\n";
return 1;
#} elsif ( $response->content =~ m#No matching entry found for#) {
} elsif ( $response->content =~ m#No matching entry found for IP ([\d\.]+)#) {
my $reportedip = $1;
dbg("DNSWL: Successfully reported $reportedip. Current trust level is: Unlisted.");
print "Successfully reported to DNSWL $reportedip. Current trust level is: Unlisted.\n";
return 1;
} else {
dbg("DNSWL: Failed to report, acknowledgement not received.");
print "Failed to report to DNSWL, acknowledgement not received.\n";
# open OUT, ">/tmp/dnswlerr.".time.".txt";
# print OUT $response->content;
# close OUT;
return 0;
}
} else {
dbg("DNSWL: Failed to report: ". $response->status_line);
print "Failed to report to DNSWL, HTTP error: ". $response->status_line ."\n";
return 0;
}
dbg("DNSWL: Error: This isn't possible.");
return 0;
}
1;
=back
=cut

View File

@@ -0,0 +1,604 @@
loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs /etc/mail/spamassassin/DecodeShortURLs.pm
body HAS_SHORT_URL eval:short_url_tests()
describe HAS_SHORT_URL Message contains one or more shortened URLs
score HAS_SHORT_URL 0.01
body SHORT_URL_CHAINED eval:short_url_tests()
describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners
score SHORT_URL_CHAINED 3.0
body SHORT_URL_MAXCHAIN eval:short_url_tests()
describe SHORT_URL_MAXCHAIN Message has shortened URL that causes more than 10 redirections
score SHORT_URL_MAXCHAIN 5.0
body SHORT_URL_LOOP eval:short_url_tests()
describe SHORT_URL_LOOP Message has short URL that loops back to itself
score SHORT_URL_LOOP 0.01
body SHORT_URL_404 eval:short_url_tests()
describe SHORT_URL_404 Message has short URL that returns 404
score SHORT_URL_404 1.0
uri URI_BITLY_BLOCKED /^http:\/\/bit\.ly\/a\/warning/i
describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse
score URI_BITLY_BLOCKED 10.0
uri URI_SIMURL_BLOCKED /^http:\/\/simurl\.com\/redirect_black\.php/i
describe URI_SIMURL_BLOCKED Message contains a simurl URL that has been disabled due to abuse
score URI_SIMURL_BLOCKED 10.0
uri URI_MIGRE_BLOCKED /^http:\/\/migre\.me\/bloqueado/i
describe URI_MIGRE_BLOCKED Message contains a migre.me URL that has been disabled due to abuse
score URI_MIGRE_BLOCKED 10.0
meta SHORT_URIBL HAS_SHORT_URL && (URIBL_BLACK || URIBL_AB_SURBL || URIBL_WS_SURBL || URIBL_JP_SURBL || URIBL_SC_SURBL || URIBL_RHS_DOB || URIBL_DBL_SPAM || URIBL_SBL)
describe SHORT_URIBL Message contains shortened URL(s) and also hits a URIDNSBL
score SHORT_URIBL 0.01
url_shortener_log /tmp/DecodeShortURLs.txt
url_shortener_cache /tmp/DecodeShortURLs.sq3
#url_shortener_syslog 1
url_shortener 0rz.tw
url_shortener 1l2.us
url_shortener 1u.ro
url_shortener 1url.com
url_shortener 2.gp
url_shortener 2.ly
url_shortener 2chap.it
url_shortener 2pl.us
url_shortener 2su.de
url_shortener 2tu.us
url_shortener 2ze.us
url_shortener 3.ly
url_shortener 301.to
url_shortener 301url.com
url_shortener 307.to
# url_shortener 4sq.com
url_shortener 6url.com
url_shortener 7.ly
url_shortener 9mp.com
url_shortener a.gd
url_shortener a.gg
url_shortener a.nf
url_shortener a2a.me
url_shortener a2n.eu
url_shortener abbr.com
url_shortener abe5.com
url_shortener access.im
url_shortener ad.vu
url_shortener adf.ly
url_shortener adjix.com
url_shortener alturl.com
url_shortener amzn.com
url_shortener amzn.to
url_shortener arm.in
url_shortener asso.in
url_shortener atu.ca
url_shortener aurls.info
url_shortener awe.sm
url_shortener ayl.lv
url_shortener azqq.com
url_shortener b23.ru
url_shortener b65.com
url_shortener b65.us
url_shortener bacn.me
url_shortener beam.to
url_shortener bgl.me
url_shortener bit.ly
url_shortener bkite.com
url_shortener blippr.com
url_shortener bloat.me
url_shortener blu.cc
url_shortener bon.no
url_shortener bt.io
url_shortener budurl.com
url_shortener buk.me
url_shortener burnurl.com
url_shortener c-o.in
url_shortener c.shamekh.ws
url_shortener canurl.com
url_shortener cd4.me
url_shortener chilp.it
url_shortener chopd.it
url_shortener chpt.me
url_shortener chs.mx
url_shortener chzb.gr
url_shortener clck.ru
url_shortener cli.gs
url_shortener cliccami.info
url_shortener clickthru.ca
url_shortener clipurl.us
url_shortener clk.my
url_shortener clop.in
url_shortener clp.ly
url_shortener coge.la
url_shortener cokeurl.com
url_shortener cort.as
url_shortener cot.ag
url_shortener crum.pl
url_shortener curio.us
url_shortener cuthut.com
url_shortener cuturl.com
url_shortener cuturls.com
url_shortener dealspl.us
url_shortener decenturl.com
url_shortener df9.net
url_shortener digbig.com
url_shortener digg.com
url_shortener digipills.com
url_shortener digs.by
url_shortener dld.bz
url_shortener dlvr.it
url_shortener dn.vc
url_shortener doi.org
url_shortener doiop.com
url_shortener dr.tl
url_shortener durl.me
url_shortener durl.us
url_shortener dvlr.it
url_shortener dwarfurl.com
url_shortener easyurl.net
url_shortener eca.sh
url_shortener eclurl.com
url_shortener eepurl.com
url_shortener eezurl.com
url_shortener ewerl.com
url_shortener ezurl.eu
url_shortener fa.by
url_shortener faceto.us
url_shortener fav.me
url_shortener fb.me
url_shortener ff.im
url_shortener fff.to
url_shortener fhurl.com
url_shortener flic.kr
url_shortener flingk.com
url_shortener flq.us
url_shortener fly2.ws
url_shortener fon.gs
url_shortener foxyurl.com
url_shortener fuseurl.com
url_shortener fwd4.me
url_shortener fwdurl.net
url_shortener fwib.net
url_shortener g8l.us
url_shortener get-shorty.com
url_shortener get-url.com
url_shortener get.sh
url_shortener gi.vc
url_shortener gkurl.us
url_shortener gl.am
url_shortener go.9nl.com
url_shortener go.to
url_shortener go2.me
url_shortener golmao.com
url_shortener goo.gl
url_shortener good.ly
url_shortener goshrink.com
url_shortener gri.ms
url_shortener gurl.es
url_shortener hao.jp
url_shortener hellotxt.com
url_shortener hex.io
url_shortener hiderefer.com
url_shortener hop.im
url_shortener hotredirect.com
url_shortener hotshorturl.com
url_shortener href.in
url_shortener ht.ly
url_shortener htxt.it
url_shortener hugeurl.com
url_shortener hurl.it
url_shortener hurl.no
url_shortener hurl.ws
url_shortener icanhaz.com
url_shortener icio.us
url_shortener idek.net
url_shortener ikr.me
url_shortener ir.pe
url_shortener irt.me
url_shortener is.gd
url_shortener iscool.net
url_shortener it2.in
url_shortener ito.mx
url_shortener j.mp
url_shortener j2j.de
url_shortener jdem.cz
url_shortener jijr.com
url_shortener just.as
url_shortener k.vu
url_shortener ketkp.in
url_shortener kisa.ch
url_shortener kissa.be
url_shortener kl.am
url_shortener klck.me
url_shortener kore.us
url_shortener korta.nu
url_shortener kots.nu
url_shortener krz.ch
url_shortener ktzr.us
url_shortener kxk.me
url_shortener l.pr
url_shortener l9k.net
url_shortener liip.to
url_shortener liltext.com
url_shortener lin.cr
url_shortener lin.io
url_shortener linkbee.com
url_shortener linkee.com
url_shortener linkgap.com
url_shortener linkslice.com
url_shortener linxfix.de
url_shortener liteurl.net
url_shortener liurl.cn
url_shortener livesi.de
url_shortener lix.in
url_shortener lk.ht
url_shortener ln-s.net
url_shortener ln-s.ru
url_shortener lnk.by
url_shortener lnk.in
url_shortener lnk.ly
url_shortener lnk.ms
url_shortener lnk.sk
url_shortener lnkurl.com
url_shortener loopt.us
url_shortener lost.in
url_shortener lru.jp
url_shortener lt.tl
url_shortener lu.to
url_shortener lurl.no
url_shortener mavrev.com
url_shortener memurl.com
url_shortener merky.de
url_shortener metamark.net
url_shortener migre.me
url_shortener min2.me
url_shortener minilien.com
url_shortener minilink.org
url_shortener miniurl.com
url_shortener minurl.fr
url_shortener moby.to
url_shortener moourl.com
url_shortener msg.sg
url_shortener murl.kz
url_shortener mv2.me
url_shortener mysp.in
url_shortener myurl.in
url_shortener myurl.si
url_shortener nanoref.com
url_shortener nanourl.se
url_shortener nbx.ch
url_shortener ncane.com
url_shortener ndurl.com
url_shortener ne1.net
url_shortener netnet.me
url_shortener netshortcut.com
url_shortener ni.to
url_shortener nig.gr
url_shortener nm.ly
url_shortener nn.nf
url_shortener notlong.com
url_shortener nutshellurl.com
url_shortener nyti.ms
url_shortener o-x.fr
url_shortener o.ly
url_shortener oboeyasui.com
url_shortener offur.com
url_shortener ofl.me
url_shortener om.ly
url_shortener omf.gd
url_shortener onecent.us
url_shortener onion.com
url_shortener onsaas.info
url_shortener ooqx.com
url_shortener oreil.ly
url_shortener ow.ly
url_shortener oxyz.info
url_shortener p.ly
url_shortener p8g.tw
url_shortener parv.us
url_shortener paulding.net
url_shortener pduda.mobi
url_shortener peaurl.com
url_shortener pendek.in
url_shortener pep.si
url_shortener pic.gd
url_shortener piko.me
url_shortener ping.fm
url_shortener piurl.com
url_shortener plumurl.com
url_shortener plurl.me
url_shortener pnt.me
url_shortener poll.fm
url_shortener pop.ly
url_shortener poprl.com
url_shortener post.ly
url_shortener posted.at
url_shortener pt2.me
url_shortener ptiturl.com
url_shortener puke.it
url_shortener pysper.com
url_shortener qik.li
url_shortener qlnk.net
url_shortener qoiob.com
url_shortener qr.cx
url_shortener quickurl.co.uk
url_shortener qurl.com
url_shortener qurlyq.com
url_shortener quu.nu
url_shortener qux.in
url_shortener r.im
url_shortener rb6.me
url_shortener rde.me
url_shortener readthis.ca
url_shortener reallytinyurl.com
url_shortener redir.ec
url_shortener redirects.ca
url_shortener redirx.com
url_shortener relyt.us
url_shortener retwt.me
url_shortener ri.ms
url_shortener rickroll.it
url_shortener rivva.de
url_shortener rly.cc
url_shortener rnk.me
url_shortener rsmonkey.com
url_shortener rt.nu
url_shortener rubyurl.com
url_shortener rurl.org
url_shortener s.gnoss.us
url_shortener s3nt.com
url_shortener s4c.in
url_shortener s7y.us
url_shortener safe.mn
url_shortener safelinks.ru
url_shortener sai.ly
url_shortener SameURL.com
url_shortener sfu.ca
url_shortener shadyurl.com
url_shortener shar.es
url_shortener shim.net
url_shortener shink.de
url_shortener shorl.com
url_shortener short.ie
url_shortener short.to
url_shortener shorten.ws
url_shortener shortenurl.com
url_shortener shorterlink.com
url_shortener shortio.com
url_shortener shortlinks.co.uk
url_shortener shortn.me
url_shortener shortna.me
url_shortener shortr.me
url_shortener shorturl.com
url_shortener shortz.me
url_shortener shoturl.us
url_shortener shredu
url_shortener shredurl.com
url_shortener shrinkify.com
url_shortener shrinkr.com
url_shortener shrinkster.com
url_shortener shrinkurl.us
url_shortener shrt.fr
url_shortener shrt.ws
url_shortener shrtl.com
url_shortener shrtn.com
url_shortener shrtnd.com
url_shortener shurl.net
url_shortener shw.me
url_shortener simurl.com
url_shortener simurl.net
url_shortener simurl.org
url_shortener simurl.us
url_shortener sitelutions.com
url_shortener siteo.us
url_shortener sl.ly
url_shortener slidesha.re
url_shortener slki.ru
url_shortener smallr.com
url_shortener smallr.net
url_shortener smfu.in
url_shortener smsh.me
url_shortener smurl.com
url_shortener sn.im
url_shortener sn.vc
url_shortener snadr.it
url_shortener snipie.com
url_shortener snipr.com
url_shortener snipurl.com
url_shortener snkr.me
url_shortener snurl.com
url_shortener song.ly
url_shortener sp2.ro
url_shortener spedr.com
url_shortener sqze.it
url_shortener srnk.net
url_shortener srs.li
url_shortener starturl.com
url_shortener stickurl.com
url_shortener stpmvt.com
url_shortener sturly.com
url_shortener su.pr
url_shortener surl.co.uk
url_shortener surl.it
url_shortener t.co
url_shortener t.lh.com
url_shortener ta.gd
url_shortener takemyfile.com
url_shortener tcrn.ch
url_shortener tgr.me
url_shortener th8.us
url_shortener thecow.me
url_shortener thrdl.es
url_shortener tighturl.com
url_shortener timesurl.at
url_shortener tini.us
url_shortener tiniuri.com
url_shortener tiny.cc
url_shortener tiny.pl
url_shortener tinyarro.ws
url_shortener tinylink.com
url_shortener tinypl.us
url_shortener tinysong.com
url_shortener tinytw.it
url_shortener tinyurl.com
url_shortener tl.gd
url_shortener tllg.net
url_shortener tncr.ws
url_shortener tnw.to
url_shortener to.je
url_shortener to.ly
url_shortener to.vg
url_shortener togoto.us
url_shortener tr.im
url_shortener tr.my
url_shortener tra.kz
url_shortener traceurl.com
url_shortener trcb.me
url_shortener trg.li
url_shortener trick.ly
url_shortener trii.us
url_shortener trim.li
url_shortener trumpink.lt
url_shortener trunc.it
url_shortener truncurl.com
url_shortener tsort.us
url_shortener tubeurl.com
# url_shortener tumblr.com
url_shortener turo.us
url_shortener tw0.us
url_shortener tw1.us
url_shortener tw2.us
url_shortener tw5.us
url_shortener tw6.us
url_shortener tw8.us
url_shortener tw9.us
url_shortener twa.lk
url_shortener tweet.me
url_shortener tweetburner.com
url_shortener tweetl.com
url_shortener twi.gy
url_shortener twip.us
url_shortener twirl.at
url_shortener twit.ac
url_shortener twitclicks.com
url_shortener twitterurl.net
url_shortener twitthis.com
url_shortener twittu.ms
url_shortener twiturl.de
url_shortener twitzap.com
url_shortener twlv.net
url_shortener twtr.us
url_shortener twurl.cc
url_shortener twurl.nl
url_shortener u.mavrev.com
url_shortener u.nu
url_shortener u76.org
url_shortener ub0.cc
url_shortener uiop.me
url_shortener ulimit.com
url_shortener ulu.lu
url_shortener unfaker.it
url_shortener updating.me
url_shortener ur.ly
url_shortener ur1.ca
url_shortener urizy.com
url_shortener url.ag
url_shortener url.az
url_shortener url.co.uk
url_shortener url.go.it
url_shortener url.ie
url_shortener url.inc-x.eu
url_shortener url.lotpatrol.com
# url_shortener url4.eu
url_shortener urlao.com
url_shortener urlbee.com
url_shortener urlborg.com
url_shortener urlbrief.com
url_shortener urlcorta.es
url_shortener urlcut.com
url_shortener urlcutter.com
url_shortener urlg.info
url_shortener urlhawk.com
url_shortener urli.nl
url_shortener urlkiss.com
url_shortener urloo.com
url_shortener urlpire.com
url_shortener urltea.com
url_shortener urlu.ms
url_shortener urlvi.b
url_shortener urlvi.be
url_shortener urlx.ie
url_shortener urlz.at
url_shortener urlzen.com
url_shortener usat.ly
url_shortener uservoice.com
url_shortener ustre.am
url_shortener vado.it
url_shortener vb.ly
url_shortener vdirect.com
url_shortener vi.ly
url_shortener viigo.im
url_shortener virl.com
url_shortener vl.am
url_shortener voizle.com
url_shortener vtc.es
url_shortener w0r.me
url_shortener w33.us
url_shortener w34.us
url_shortener w3t.org
url_shortener wa9.la
url_shortener wapurl.co.uk
url_shortener webalias.com
url_shortener welcome.to
url_shortener wh.gov
url_shortener wipi.es
url_shortener wkrg.com
url_shortener woo.ly
url_shortener wp.me
url_shortener x.hypem.com
url_shortener x.se
url_shortener x.vu
url_shortener xeeurl.com
url_shortener xil.in
url_shortener xlurl.de
url_shortener xr.com
url_shortener xrl.in
url_shortener xrl.us
url_shortener xrt.me
url_shortener xurl.jp
url_shortener xxsurl.de
url_shortener xzb.cc
url_shortener yatuc.com
url_shortener ye-s.com
url_shortener yep.it
# url_shortener youtu.be
url_shortener z.pe
url_shortener zapt.in
url_shortener zi.ma
url_shortener zi.me
url_shortener zi.pe
url_shortener zip.li
url_shortener zipmyurl.com
url_shortener zootit.com
url_shortener zud.me
url_shortener zurl.ws
url_shortener zz.gd
url_shortener zzang.kr
url_shortener xn--cwg.ws
url_shortener xn--fwg.ws
url_shortener xn--bih.ws
url_shortener xn--l3h.ws
url_shortener xn--1ci.ws
url_shortener xn--odi.ws
url_shortener xn--rei.ws
url_shortener xn--3fi.ws
url_shortener xn--egi.ws
url_shortener xn--hgi.ws
url_shortener xn--ogi.ws
url_shortener xn--vgi.ws
url_shortener xn--5gi.ws
url_shortener xn--9gi.ws

View File

@@ -0,0 +1,564 @@
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>
# Author: Steve Freegard <steve.freegard@fsl.com>
=head1 NAME
DecodeShortURLs - Expand shortened URLs
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs
url_shortener bit.ly
url_shortener go.to
...
=head1 DESCRIPTION
This plugin looks for URLs shortened by a list of URL shortening services and
upon finding a matching URL will connect using to the shortening service and
do an HTTP HEAD lookup and retrieve the location header which points to the
actual shortened URL, it then adds this URL to the list of URIs extracted by
SpamAssassin which can then be accessed by other plug-ins, such as URIDNSBL.
This plugin also sets the rule HAS_SHORT_URL if any matching short URLs are
found.
Regular 'uri' rules can be used to detect and score links disabled by the
shortening service for abuse and URL_BITLY_BLOCKED is supplied as an example.
It should be safe to score this rule highly on a match as experience shows
that bit.ly only blocks access to a URL if it has seen consistent abuse and
problem reports.
As of version 0.3 this plug-in will follow 'chained' shorteners e.g.
short URL -> short URL -> short URL -> real URL
If this form of chaining is found, then the rule 'SHORT_URL_CHAINED' will be
fired. If a loop is detected then 'SHORT_URL_LOOP' will be fired.
This plug-in limits the number of chained shorteners to a maximim of 10 at
which point it will fire the rule 'SHORT_URL_MAXCHAIN' and go no further.
If a shortener returns a '404 Not Found' result for the short URL then the
rule 'SHORT_URL_404' will be fired.
=head1 NOTES
This plugin runs the parsed_metadata hook with a priority of -1 so that
it may modify the parsed URI list prior to the URIDNSBL plugin which
runs as priority 0.
Currently the plugin queries a maximum of 10 distinct shortened URLs with
a maximum timeout of 5 seconds per lookup. It does not recurse and follow
'chained' shortening as the author has no examples of this happening.
=head1 ACKNOWLEDGEMENTS
A lot of this plugin has been hacked together by using other plugins as
examples. The author would particularly like to tip his hat to Karsten
Bräckelmann for the _add_uri_detail_list() function that he stole from
GUDO.pm for which this plugin would not be possible due to the SpamAssassin
API making no provision for adding to the base list of extracted URIs and
the author not knowing enough about Perl to be able to achieve this without
a good example from someone that does ;-)
=cut
package Mail::SpamAssassin::Plugin::DecodeShortURLs;
my $VERSION = 0.6;
use Mail::SpamAssassin::Plugin;
use strict;
use warnings;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
use constant HAS_LWP_USERAGENT => eval { local $SIG{'__DIE__'}; require LWP::UserAgent; };
use constant HAS_SQLITE => eval { local $SIG{'__DIE__'}; require DBD::SQLite; };
use Fcntl qw(:flock SEEK_END);
use Sys::Syslog qw(:DEFAULT setlogsock);
sub dbg {
my $msg = shift;
return Mail::SpamAssassin::Logger::dbg("DecodeShortURLs: $msg");
}
sub new {
my $class = shift;
my $mailsaobject = shift;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsaobject);
bless ($self, $class);
if ($mailsaobject->{local_tests_only} || !HAS_LWP_USERAGENT) {
$self->{disabled} = 1;
} else {
$self->{disabled} = 0;
}
unless ($self->{disabled}) {
$self->{ua} = new LWP::UserAgent;
$self->{ua}->{max_redirect} = 0;
$self->{ua}->{timeout} = 5;
$self->{ua}->env_proxy;
$self->{logging} = 0;
$self->{caching} = 0;
$self->{syslog} = 0;
}
$self->set_config($mailsaobject->{conf});
$self->register_method_priority ('parsed_metadata', -1);
$self->register_eval_rule('short_url_tests');
return $self;
}
sub set_config {
my($self, $conf) = @_;
my @cmds = ();
push (@cmds, {
setting => 'url_shortener',
default => {},
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
foreach my $domain (split(/\s+/, $value)) {
$self->{url_shorteners}->{lc $domain} = 1;
}
}
});
=cut
=head1 PRIVILEGED SETTINGS
=over 4
=item url_shortener_log (default: none)
A path to a log file to be written to. The file will be created if it does
not already exist and must be writable by the user running spamassassin.
For each short URL found the following will be written to the log file:
[unix_epoch_time] <short url> => <decoded url>
=cut
push (@cmds, {
setting => 'url_shortener_log',
default => '',
is_priv => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
});
=item url_shortener_cache (default: none)
The full path to a database file to write cache entries to. The database will
be created automatically if is does not already exist but the supplied path
and file must be read/writable by the user running spamassassin or spamd.
NOTE: you will need the DBD::SQLite module installed to use this feature.
Example:
url_shortener_cache /tmp/DecodeShortURLs.sq3
=cut
push (@cmds, {
setting => 'url_shortener_cache',
default => '',
is_priv => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
});
=item url_shortener_cache_ttl (default: 86400)
The length of time a cache entry will be valid for in seconds.
Default is 86400 (1 day).
NOTE: you will also need to run the following via cron to actually remove the
records from the database:
echo "DELETE FROM short_url_cache WHERE modified < strftime('%s',now) - <ttl>; | sqlite3 /path/to/database"
NOTE: replace <ttl> above with the same value you use for this option
=cut
push (@cmds, {
setting => 'url_shortener_cache_ttl',
is_admin => 1,
default => 86400,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
});
=item url_shortener_syslog (default: 0 (off))
If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged to syslog (mail.info).
=cut
push (@cmds, {
setting => 'url_shortener_syslog',
is_admin => 1,
default => 0,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL
});
$conf->{parser}->register_commands(\@cmds);
}
sub parsed_metadata {
my ($self, $opts) = @_;
my $pms = $opts->{permsgstatus};
my $msg = $opts->{msg};
return if $self->{disabled};
dbg ('warn: get_uri_detail_list() has been called already')
if exists $pms->{uri_detail_list};
# don't keep dereferencing these
$self->{url_shorteners} = $pms->{main}->{conf}->{url_shorteners};
($self->{url_shortener_log}) = ($pms->{main}->{conf}->{url_shortener_log} =~ /^(.*)$/g);
($self->{url_shortener_cache}) = ($pms->{main}->{conf}->{url_shortener_cache} =~ /^(.*)$/g);
$self->{url_shortener_cache_ttl} = $pms->{main}->{conf}->{url_shortener_cache_ttl};
$self->{url_shortener_syslog} = $pms->{main}->{conf}->{url_shortener_syslog};
# Sort short URLs into hash to de-dup them
my %short_urls;
my $uris = $pms->get_uri_detail_list();
while (my($uri, $info) = each %{$uris}) {
next unless ($info->{domains});
foreach ( keys %{ $info->{domains} } ) {
if (exists $self->{url_shorteners}->{lc $_}) {
# NOTE: $info->{domains} appears to contain all the domains parsed
# from the single input URI with no way to work out what the base
# domain is. So to prevent someone from stuffing the URI with a
# shortener to force this plug-in to follow a link that *isn't* on
# the list of shorteners; we enforce that the shortener must be the
# base URI and that a path must be present.
if ($uri !~ /^http:\/\/(?:www\.)?$_\/.+$/) {
dbg("Discarding URI: $uri");
next;
}
$short_urls{$uri} = 1;
next;
}
}
}
# Make sure we have some work to do
# Before we open any log files etc.
my $count = scalar keys %short_urls;
return undef unless $count gt 0;
# Initialise logging if enabled
if ($self->{url_shortener_log}) {
eval {
local $SIG{'__DIE__'};
open($self->{logfh}, '>>'.$self->{url_shortener_log}) or die $!;
};
if ($@) {
dbg("warn: $@");
} else {
$self->{logging} = 1;
}
}
# Initialise syslog if enabled
if ($self->{url_shortener_syslog}) {
eval {
local $SIG{'__DIE__'};
openlog('DecodeShortURLs','ndelay,pid','mail');
};
if ($@) {
dbg("warn: $@");
} else {
$self->{syslog} = 1;
}
}
# Initialise cache if enabled
if ($self->{url_shortener_cache} && HAS_SQLITE) {
eval {
local $SIG{'__DIE__'};
$self->{dbh} = DBI->connect_cached("dbi:SQLite:dbname=".$self->{url_shortener_cache},"","", {RaiseError => 1, PrintError => 0, InactiveDestroy => 1}) or die $!;
};
if ($@) {
dbg("warn: $@");
} else {
$self->{caching} = 1;
# Create database if needed
eval {
local $SIG{'__DIE__'};
$self->{dbh}->do("
CREATE TABLE IF NOT EXISTS short_url_cache (
short_url TEXT PRIMARY KEY NOT NULL,
decoded_url TEXT NOT NULL,
hits INTEGER NOT NULL DEFAULT 1,
created INTEGER NOT NULL DEFAULT (strftime('%s','now')),
modified INTEGER NOT NULL DEFAULT (strftime('%s','now'))
)
");
$self->{dbh}->do("
CREATE INDEX IF NOT EXISTS short_url_by_modified
ON short_url_cache(short_url, modified)
");
$self->{dbh}->do("
CREATE INDEX IF NOT EXISTS short_url_modified
ON short_url_cache(modified)
");
};
if ($@) {
dbg("warn: $@");
$self->{caching} = 0;
}
}
}
my $max_short_urls = 10;
foreach my $short_url (keys %short_urls) {
next if ($max_short_urls le 0);
my $location = $self->recursive_lookup($short_url, $pms);
$max_short_urls--;
}
# Close log
eval {
local $SIG{'__DIE__'};
close($self->{logfh}) or die $!;
} if $self->{logging};
# Close syslog
eval {
local $SIG{'__DIE__'};
closelog() or die $!;
} if $self->{syslog};
# Don't disconnect cached database handle
# eval { $self->{dbh}->disconnect() or die $!; } if $self->{caching};
}
sub recursive_lookup {
my ($self, $short_url, $pms, %been_here) = @_;
my $count = scalar keys %been_here;
dbg("Redirection count $count") if $count gt 0;
if ($count ge 10) {
dbg("Error: more than 10 shortener redirections");
# Fire test
$pms->got_hit('SHORT_URL_MAXCHAIN');
return undef;
}
my $location;
if ($self->{caching} && ($location = $self->cache_get($short_url))) {
dbg("Found cached $short_url => $location");
eval {
local $SIG{'__DIE__'};
$self->log_to_file("$short_url => $location")
} if $self->{logging};
syslog('info',"Found cached $short_url => $location") if $self->{syslog};
} else {
# Not cached; do lookup
my $response = $self->{ua}->head($short_url);
if (!$response->is_redirect) {
dbg("Skipping URL as not redirect: $short_url = ".$response->status_line);
$pms->got_hit('SHORT_URL_404') if($response->code == '404');
return undef;
}
$location = $response->headers->{location};
# Bail out if $short_url redirects to itself
return undef if ($short_url eq $location);
$self->cache_add($short_url, $location) if $self->{caching};
dbg("Found $short_url => $location");
eval {
local $SIG{'__DIE__'};
$self->log_to_file("$short_url => $location")
} if $self->{logging};
syslog('info',"Found $short_url => $location") if $self->{syslog};
}
# At this point we have a new URL in $response
$pms->got_hit('HAS_SHORT_URL');
_add_uri_detail_list($pms, $location);
# Set chained here otherwise we might mark a disabled page or
# redirect back to the same host as chaining incorrectly.
$pms->got_hit('SHORT_URL_CHAINED') if ($count gt 0);
# Check if we are being redirected to a local page
# Don't recurse in this case...
if($location !~ /^https?:/) {
my($host) = ($short_url =~ /^(https?:\/\/\S+)\//);
$location = "$host/$location";
dbg("Looks like a local redirection: $short_url => $location");
_add_uri_detail_list($pms, $location);
return $location;
}
# Check for recursion
if ((my ($domain) = ($location =~ /^https?:\/\/(\S+)\//))) {
if (exists $been_here{$location}) {
# Loop detected
dbg("Error: loop detected");
$pms->got_hit('SHORT_URL_LOOP');
return $location;
} else {
if (exists $self->{url_shorteners}->{$domain}) {
$been_here{$location} = 1;
# Recurse...
return $self->recursive_lookup($location, $pms, %been_here);
}
}
}
# No recursion; just return the final location...
return $location;
}
sub short_url_tests {
# Set by parsed_metadata
return 0;
}
# Beware. Code copied from PerMsgStatus get_uri_detail_list().
# Stolen from GUDO.pm
sub _add_uri_detail_list {
my ($pms, $uri) = @_;
my $info;
# Cache of text parsed URIs, as previously used by get_uri_detail_list().
push @{$pms->{parsed_uri_list}}, $uri;
$info->{types}->{parsed} = 1;
$info->{cleaned} =
[Mail::SpamAssassin::Util::uri_list_canonify (undef, $uri)];
foreach (@{$info->{cleaned}}) {
my $dom = Mail::SpamAssassin::Util::uri_to_domain($_);
if ($dom && !$info->{domains}->{$dom}) {
$info->{domains}->{$dom} = 1;
$pms->{uri_domain_count}++;
}
}
$pms->{uri_detail_list}->{$uri} = $info;
# And of course, copied code from PerMsgStatus get_uri_list(). *sigh*
dbg ('warn: PMS::get_uri_list() appears to have been harvested'),
push @{$pms->{uri_list}}, @{$info->{cleaned}}
if exists $pms->{uri_list};
}
sub log_to_file {
my ($self, $msg) = @_;
return undef if not $self->{logging};
my $fh = $self->{logfh};
eval {
flock($fh, LOCK_EX) or die $!;
seek($fh, 0, SEEK_END) or die $!;
print $fh '['.time.'] '.$msg."\n";
flock($fh, LOCK_UN) or die $!;
};
}
sub cache_add {
my ($self, $short_url, $decoded_url) = @_;
return undef if not $self->{caching};
eval {
$self->{sth_insert} = $self->{dbh}->prepare_cached("
INSERT INTO short_url_cache (short_url, decoded_url)
VALUES (?,?)
");
};
if ($@) {
dbg("warn: $@");
return undef;
};
$self->{sth_insert}->execute($short_url, $decoded_url);
return undef;
}
sub cache_get {
my ($self, $key) = @_;
return undef if not $self->{caching};
eval {
$self->{sth_select} = $self->{dbh}->prepare_cached("
SELECT decoded_url FROM short_url_cache
WHERE short_url = ? AND modified > (strftime('%s','now') - ?)
");
};
if ($@) {
dbg("warn: $@");
return undef;
}
eval {
$self->{sth_update} = $self->{dbh}->prepare_cached("
UPDATE short_url_cache
SET modified=strftime('%s','now'), hits=hits+1
WHERE short_url = ?
");
};
if ($@) {
dbg("warn: $@");
return undef;
}
$self->{sth_select}->execute($key, $self->{url_shortener_cache_ttl});
my $row = $self->{sth_select}->fetchrow_array();
if($row) {
# Found cache entry; touch it to prevent expiry
$self->{sth_update}->execute($key);
$self->{sth_select}->finish();
$self->{sth_update}->finish();
return $row;
}
$self->{sth_select}->finish();
$self->{sth_update}->finish();
return undef;
}
1;

43
mail/spamassassin/GPG.KEY Normal file
View File

@@ -0,0 +1,43 @@
GPG key used to sign Apache SpamAssassin update channel files of channels
in the sa-update.dostech.net domain as of August 13, 2006.
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.4 (GNU/Linux)
mQGiBETa02gRBADkEGCDrE+q/4B9f4SSe8Zu6Faqj5x94Z6uSDQgiN8tybb2M9GS
nyE4Q2Xi179IYkYhiw9UHYmascM5L39Qfw4bgi9xylGlrmGs/o2QGvSlgkOoA9uu
rAqIp8zW/W5Vtu0+3enm4u48hMfMkJU6wb6y4YkrYwJwr9qOpNIExTc1owCg6pOT
4XczIv6tqtwKj8lpwMtA19cD/3nDABhzeiy/EaCBGzyla97JMUkUXpbbaBeG8YHd
c/DzlaZUtMzhYCSBnY9dw7x5k/nrpCcoHLhuK/CsKqZWHEvF9n6FoDx6yw8hSU3U
oOTiBd7xDvpp5fgHVKoEI9Pq70Am+bv6qWa0VwJjGWHuHJt3/KUxE8IYz9rfNQDB
spmUBACgTLmJlDmWSxUlPITrWb4Lu8it26Nql4pXPtDbhp7pqAH1dfqz4MMHyi2M
pxhkmyAUFrOOSq2PkkmE2YqbEg/uLyG/lFrznRN+H74sk5odLfAFDcLhjzVwNCqN
RlGrzfIkq9Pm2u/AZpn32UzMX6HFhkm1qCtUqgU0TNSR204nZrQsRGFyeWwgQy4g
Vy4gTydTaGVhIDxzcGFtYXNzYXNzaW5AZG9zdGVjaC5jYT6IYAQTEQIAIAUCRNrT
aAIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEDxcBeuFaqiKSG8AniNl9xOy
tcxdqMbJVw45Vl9dyGnKAJ4tJIHqpjB67yivI3Ya6jrJAgjjb7kEDQRE2tQwEBAA
vjoi2fNKdu9/A1f67M7xoTYZYEkGeWXd2ISabVrHqXJeghdWQDi81v88oMWyAw0t
y2URlVPAgf4sEn5eJqOPd9ZNl+JBk2VIhw7LfniizhfKS8PVCrZRikqx4fzjiSK1
w6IPQWJ7bzhFK7SofrgaeHvAldWthXooxHRwnREEf5Cz199K80ulio8o4QYMZwqY
537O1fgAjh6U1xQRjzMVVj8YC9j3pn5b4cho/Eixb58XafOOVS9F9sMzFhb7KIAK
U3pdwL8jexQM8exuN9a4p9lKyc4Q0itFURKG8/3BAhwVKMdcB4d+ExD/ExXKftUr
PiZ3kC0HT10CLg1BAYR76gJLfbvAR4sgW2xnpKz1pvnpgeiYsF4zfagovbi7WR4X
ix6l/JpeqSud4peLxi48g+jSThRLvSG0/V6grfPqQoylnRFuM6nHeX3KWF6+z1XQ
xh8hdP8AXyrePbLCT8IG5c009U0qWrvRqnUu6jWGLz6rg0wepD+9sxDwDyJAAoZ5
5Ke0J3wDsa1w5TUPBRb54gFTg1rnvlUfA8+2ZXz0o/v10FC9tykHsTlDoPBkDjxt
+lNUod0drF3zKPqj6E15v/vDTvfbv4WXeYEESKxw0aYMNP4I6OwSmsIrRTsXwP/A
3rTxwSr38vmOMZqZveqeC8xMIBD70AmHeKXK9JQv1mcABAsP/2wy8WjRC5J0lglh
+3SlafHHe0FrUiRdojvQkCl2CCu7AVS4onlxWwhIE+Wx4ci+Kp+RkO+o8OBIWEEx
PhRXhejHcCdBY5QjSh/RexFPF/5AZZZKK/jG30CKNr22NNsaFHITifb/oyAqgqP1
i3RU/oX0usIn5ojcqAzO3C9mOGncWVVfk59j7DnwVKEGp2enjgGBRJzLe9qHwMS5
hkb97ixFmIEK7C8+F86GXPOgeyQvDPs3R6T4xASkiY6Or4LdGBcPOOWaxsN+L6v7
s19kXfN4sNKv2nE6MfaRcpNZmhubprOLOTsJO31Sfu8mgfIjmlFzrx6Q1/vSYoPM
z4MHPxyAiCnFKeETO4nP+EJeFbGx4s1ni4o+ayTiu0qvTTw9UMQWUqv1SHcd/+wh
aG8q9kBI8sPM3EsZv9Kg88fDjW6bwrs6L1AdPX7tCuBTZQLqbfQUtmRED2HLCce+
Swy+5XeDchlMaAsGazsg2a2EZvCTzB9ecJDBvZi8TfbCryDDGkI17hKN0oiOVeVi
MkcV6hfs+Do8duaF25FtxkGmAk6F+gZ0S1Y2+3/xzGOA/g4vmdX5Nwur19vyOly3
6P4M4ofMvhNqqj8A2x7Eu5nKZIeMzmGsem/As7d9fN7zlg4bC49pZdjzhYrsO7JD
0lBQ8aCjcIoYgKgXSx8/rWAr1v2viEkEGBECAAkFAkTa1DACGwwACgkQPFwF64Vq
qIrAWgCghOtFVBFL4I/LC6s0kH7WFUk8guUAoMHy+m/k6vw6VtHUCte9gGAPpLoD
=eTcH
-----END PGP PUBLIC KEY BLOCK-----

410
mail/spamassassin/MTX.pm Normal file
View File

@@ -0,0 +1,410 @@
# MTX plugin for SpamAssassin
# (c) Darxus@ChaosReigns.com, released under the GPL.
# http://www.chaosreigns.com/mtx/
#
# 2010-02-10 Initial release.
# 2010-02-12 Implemented blacklisting, switched to SA's DnsResolver
# 2010-02-12-01 Fixed failure to handle IP CNAME caused in previous.
# Reduce chances of exploiting flaws by more defaulting to
# "fail".
# 2010-02-12-01 Switched from last external to last untrusted relay.
# 2010-02-13 Rename of above.
# 2010-02-13-01 Don't "fail" on "last untrusted relay unavailable".
# 2010-02-14 Rename of above.
# 2010-02-14-01 Implemented policy record without delegation.
# 2010-02-14-02 Implemented policy record delegation.
# 2010-02-14-03 Fixed whining about "implicit split to @_".
# 2010-02-15 Rename of above.
# 2010-02-15-01 Don't check Policy of None has already been determined.
# 2010-02-15-02 Removed unnecessary variable $arraydepth.
# 2010-02-15-03 Minor tidying.
# 2010-02-15-04 Removed unnecessary variable $hostname.
# 2010-02-15-05 Further minor tidying.
# 2010-02-16 Rename of above.
# 2010-02-15-01 Switched back to Net::DNS::Resolver for SpamAssassin v3.3.0 compatability.
# All releases pass harness testing on SA v3.2.5 + perl
# v5.8.8 and v3.3.0 + perl v5.10.0 starting here.
# 2010-10-19 Throw a freaking error if the DNS lookup fails. Thanks to
# Patrick Domack for reporting.
# 2010-10-24 If DNS lookup on sending IP returns something but it
# contains nothing, also set mtx_none so check_polciy
# doesn't get run. Thanks to Patrick Domack for reporting.
# 2011-05-29 IPv6 support by Andreas Schulze.
#
# TODO
# * Switch to Mail::SpamAssassin::DnsResolver::bgsend ?
=head1 NAME
MTX - MTX
=head1 SYNOPSIS
# http://www.chaosreigns.com/mtx/
loadplugin Mail::SpamAssassin::Plugin::MTX
header MTX_PASS eval:check_mtx_pass()
score MTX_PASS -5
describe MTX_PASS MTX: Passed: http://www.chaosreigns.com/mtx/
header MTX_FAIL eval:check_mtx_fail()
score MTX_FAIL 1
describe MTX_FAIL MTX: Failed: http://www.chaosreigns.com/mtx/
header MTX_BLACKLIST eval:check_mtx_blacklist()
# Do not define a score, it's defined with mtx_blacklist.
describe MTX_BLACKLIST MTX: On your blacklist.
# Blacklist by the host name (PTR) of the sending IP (last untrusted relay).
# Second argument is the score to assign, use whatever you want.
mtx_blacklist *.example.com 5 # Known to send spam *and* nonspam, nullify PASS score.
mtx_blacklist *.example2.com 100 # Only sends spam, big penalty.
=head1 DESCRIPTION
Write the above lines in the synopsis to
C</etc/spamassassin/local.cf>.
=cut
use strict;
use warnings;
package Mail::SpamAssassin::Plugin::MTX;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger; # dbg()
# REVIEW: does including Net::IP introduce performance
# or massiv memory usage changes?
# REVIEW: may this trigger problems outside this plugin?
use Net::IP;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
#my $res = Mail::SpamAssassin::DnsResolver->new;
my $res = Net::DNS::Resolver->new;
sub new {
my $class = shift;
my $mailsaobject = shift;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsaobject);
bless ($self, $class);
$self->register_eval_rule ("check_mtx_pass");
$self->register_eval_rule ("check_mtx_fail");
$self->register_eval_rule ("check_mtx_none");
$self->register_eval_rule ("check_mtx_neutral");
$self->register_eval_rule ("check_mtx_softfail");
$self->register_eval_rule ("check_mtx_hardfail");
$self->register_eval_rule ("check_mtx_blacklist");
$self->set_config($mailsaobject->{conf});
return $self;
}
sub check_mtx_pass {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return $scanner->{mtx_pass};
}
sub check_mtx_fail {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return $scanner->{mtx_fail};
}
sub check_mtx_none {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return 0 if ( $scanner->{mtx_hardfail} );
&check_policy if ( $scanner->{mtx_fail} and ! $scanner->{policy_checked}
and ! $scanner->{mtx_none} );
return $scanner->{mtx_none};
}
sub check_mtx_neutral {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return 0 if ( $scanner->{mtx_hardfail} );
&check_policy if ( $scanner->{mtx_fail} and ! $scanner->{policy_checked}
and ! $scanner->{mtx_none} );
return $scanner->{mtx_neutral};
}
sub check_mtx_softfail {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return 0 if ( $scanner->{mtx_hardfail} );
&check_policy if ( $scanner->{mtx_fail} and ! $scanner->{policy_checked}
and ! $scanner->{mtx_none} );
return $scanner->{mtx_softfail};
}
sub check_mtx_hardfail {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
return 1 if ( $scanner->{mtx_hardfail} );
&check_policy if ( $scanner->{mtx_fail} and ! $scanner->{policy_checked}
and ! $scanner->{mtx_none} );
return $scanner->{mtx_hardfail};
}
sub check_policy {
my ($self,$scanner) = @_;
$scanner->{policy_checked} = 1;
my $participant = 0;
dbg("mtx: Checking MTX Policy.");
use Mail::SpamAssassin::Util::RegistrarBoundaries;
my $domain = Mail::SpamAssassin::Util::RegistrarBoundaries::trim_domain($scanner->{hostname});
my @hostname = split('\.', $scanner->{hostname});
my $policy_mindepth = scalar( @{[split('\.', $domain)]} );
my $policy_maxdepth = scalar( @hostname );
$policy_maxdepth = 20 if ($policy_maxdepth > 20);
dbg ("mtx: Policy mindepth: $policy_mindepth, maxdepth: $policy_maxdepth" );
for my $policy_depth ($policy_mindepth .. $policy_maxdepth) {
my $delegate = 0;
my $policyfound = 0;
$domain = join('.',reverse((reverse(@hostname))[0 .. $policy_depth -1]));
my $policy = "policy.mtx.$domain";
dbg("mtx: MTX Policy record name: $policy, depth: $policy_depth");
my $packet = $res->send($policy, 'A');
unless (defined $packet) {
dbg('mtx: DNS "A" record lookup failed. You appear to have a DNS problem: ', $res->errorstring);
return;
}
my @answer = $packet->answer;
unless (@answer) {
dbg("mtx: Failed to get policy record $policy.");
$scanner->{mtx_none}=1;
return;
}
for my $rr (@answer) {
if (${$rr}{type} eq 'A') {
my $address = ${$rr}{address};
unless (defined $address) {
dbg("mtx: A record exists but has no value. I don't think this is possible.");
$scanner->{mtx_none}=1;
return;
}
dbg("mtx: MTX Policy record value: $address.");
if ($address =~ m#^127\.\d{1,3}\.(0|1)\.(0|1|2)$#) { ##
$delegate = $1;
$participant = $2;
$policyfound = 1;
if ($delegate == 1) {
dbg("mtx: Delegated.");
} else {
dbg("mtx: Not delegated.");
}
if ($participant == 0) {
dbg("mtx: Found Neutral.");
$scanner->{mtx_neutral}=1;
$scanner->{mtx_softfail}=0;
$scanner->{mtx_hardfail}=0;
} elsif ($participant == 1) {
dbg("mtx: Found SoftFail.");
$scanner->{mtx_neutral}=0;
$scanner->{mtx_softfail}=1;
$scanner->{mtx_hardfail}=0;
} elsif ($participant == 2) {
dbg("mtx: Found HardFail.");
$scanner->{mtx_neutral}=0;
$scanner->{mtx_softfail}=0;
$scanner->{mtx_hardfail}=1;
}
} else {
dbg("mtx: Unknown policy record found, wildcard DNS record, or new version of MTX? Ignoring.");
}
last; # Protocol says only check first.
}
}
if ($policyfound != 1) {
dbg("mtx: No policy found at this depth.");
unless ( $scanner->{mtx_neutral} or $scanner->{mtx_softfail}
or $scanner->{mtx_hardfail} ) {
$scanner->{mtx_none}=1;
}
return;
}
last unless ($delegate == 1);
}
}
sub check_mtx {
my ($self,$scanner) = @_;
# Sane defaults. Reduce chance of exploitable flaws.
$scanner->{mtx_fail}=1;
$scanner->{mtx_pass}=0;
dbg("mtx: Doing the necessary DNS lookups.");
$scanner->{mtx_checked}=1;
$scanner->{lasthop} = $scanner->{relays_untrusted}->[0];
if (!defined $scanner->{lasthop}) {
dbg("mtx: Last untrusted relay not available, fix your SA config, skipping MTX.");
# The *only* failure that doesn't result in a "fail", because it's
# due to SA misconfiguration. Or all hops being trusted, or something.
$scanner->{mtx_fail}=0;
return;
}
my $ip = $scanner->{lasthop}->{ip};
dbg("mtx: Testing IP: $ip (last untrusted relay).");
my $mtx = '';
{
my $packet = $res->send($ip, 'PTR');
unless (defined $packet) {
dbg('mtx: DNS "PTR" record lookup failed. You appear to have a DNS problem: ', $res->errorstring);
return;
}
my @answer = $packet->answer;
unless (@answer) {
dbg("mtx: Failed to get PTR record for $ip.");
$scanner->{mtx_fail}=1;
$scanner->{mtx_none}=1;
return;
}
# Can't just use the first record because it could be a CNAME with
# a PTR in there somewhere.
for my $rr (@answer) {
if (${$rr}{type} eq 'PTR') {
$scanner->{hostname} = ${$rr}{ptrdname};
dbg("mtx: Host name ('PTR' record) is ". $scanner->{hostname} .".");
my $netip = new Net::IP ($ip);
my $reverseip = $netip->reverse_ip();
$reverseip =~ s/\.(in-addr|ip6)\.arpa\.//i;
unless (defined $reverseip and $reverseip =~ /\./) {
info("mtx: failed to reverse $ip");
# REVIEW: maybe $scanner->{mtx_foo} should be set ?????
return;
}
$mtx = $reverseip . '.mtx.' . $scanner->{hostname};
$scanner->{mtx_record}=$mtx;
dbg("mtx: Relevant MTX record is: $mtx");
last; # Protocol says use the first one.
}
}
if ($mtx eq '') {
dbg("mtx: Looking up DNS PTR record for sender returned a vailue which did not contain the answer.");
# Looking up the DNS record for the delivering IP returned an
# answer, but it contained nothing. That's pretty freaky.
# Need to call it a "fail" anyway so spammers don't explot it.
$scanner->{mtx_fail}=1;
$scanner->{mtx_none}=1;
return;
}
dbg("mtx: Checking blacklist.");
foreach my $black_addr (keys %{$scanner->{conf}->{mtx_blacklist}}) {
my $re = qr/$scanner->{conf}->{mtx_blacklist}->{$black_addr}{re}/i;
if ($mtx =~ $re) {
# How can I do this without an array?
my $bl_score = (@{$scanner->{conf}->{mtx_blacklist}->{$black_addr}{domain}})[0];
dbg("mtx: Blacklisted with score $bl_score and regex $re");
$scanner->{blacklist_score}=$bl_score;
last; # Use first matching blacklist entry.
}
}
}
{
my $packet = $res->send($mtx, 'A');
unless (defined $packet) {
dbg('mtx: DNS "A" record lookup failed. You appear to have a DNS problem: ', $res->errorstring);
return;
}
my @answer = $packet->answer;
unless (@answer) {
dbg("mtx: Failed to get A record for $mtx.");
$scanner->{mtx_fail}=1;
return;
}
for my $rr (@answer) {
if (${$rr}{type} eq 'A') {
my $address = ${$rr}{address};
unless (defined $address) {
dbg("mtx: A record exists but has no value. I don't think this is possible.");
# Make sure it doesn't get exploited, just in case.
$scanner->{mtx_fail}=1;
return;
}
dbg("mtx: MTX record value: $address.");
if ($address =~ m#^127\.\d{1,3}\.\d{1,3}\.(0|1)$#) { ##
my $mtxvalue = $1;
if ($mtxvalue == 1) {
dbg("mtx: MTX record value indicates legit server. That's the only pass.");
$scanner->{mtx_pass}=1;
$scanner->{mtx_fail}=0;
return;
} elsif ($mtxvalue == 0) {
dbg("mtx: MTX record value indicates non-legit server. That's a fail.");
$scanner->{mtx_fail}=1;
$scanner->{mtx_hardfail}=1;
return;
} else {
dbg("mtx: Somebody introduced a bug.");
die "mtx: Somebody introduced a bug.";
}
} else {
dbg("mtx: Unknown MTX record value found. Wildcard DNS record or new version of MTX? Ignoring.");
}
last; # Protocol says only check first.
}
}
dbg("mtx: No known MTX record value found, fail.");
$scanner->{mtx_fail}=1;
return;
}
}
sub set_config {
my ($self, $conf) = @_;
my @cmds;
push (@cmds, {
setting => 'mtx_blacklist',
code => sub {
my ($self, $key, $value, $line) = @_;
local ($1,$2);
unless (defined $value and $value !~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
# It is important to not accept negative scores on the blacklist,
# because these hostnames can effortlessly beforged by the IP owner.
unless (defined $value and $value =~ /^(\S+)\s+([\d\.]+)$/) {
return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
my $address = $1;
my $score = $2;
$self->{parser}->add_to_addrlist_rcvd('mtx_blacklist', $address, $score);
}
});
return($conf->{parser}->register_commands(\@cmds));
}
sub check_mtx_blacklist {
my ($self, $scanner) = @_;
&check_mtx unless $scanner->{mtx_checked};
my @cmds;
my $score = $scanner->{blacklist_score};
if($score) {
my $description = $scanner->{conf}->{descriptions}->{MTX_BLACKLIST};
$description .= " Score $score.";
$scanner->{conf}->{descriptions}->{MTX_BLACKLIST} = $description;
# Set the score.
$scanner->got_hit("MTX_BLACKLIST", "", score => $score);
for my $set (0..3) {
$scanner->{conf}->{scoreset}->[$set]->{"MTX_BLACKLIST"} = sprintf("%0.3f", $score);
}
}
return 0;
}
1;

View File

@@ -0,0 +1,44 @@
loadplugin PhishTag PhishTag.pm
trigger_ratio 0.01
#scripting related keywords from http://www.w3.org/TR/html401/interact/scripts.html
rawbody __HAS_SCRIPT /<SCRIPT|on((un)?load|(dbl)?click|mouse(down|up|over|move|out)|blur|key(press|down|up)|submit|reset|select|change)/i
#This trigger will match non-scripted emails that contain a URL in the
#URIBL blacklist, and also has a ip URL which has a non-ip anchor
#text; a common signature for phishing emails.
#If this trigger goes off, all the URLs will be rewritten to point to
#the consumer education page of the Anti-Phishing Working Group.
meta HARD_URL (( URIBL_BLACK && ! __HAS_SCRIPT) && (HTTPS_IP_MISMATCH))
score HARD_URL 0
trigger_target HARD_URL http://www.antiphishing.org/consumer_recs.html
#Another trick used by phishers is to use a subdomain name that
#corresponds to a legitimate company domain name, hence creates
#trust to the untrained eye.
#If this trigger goes off, all the URLs will be rewritten to point to
#the consumer education page of the Anti-Phishing Working Group.
meta EASY_URL (( URIBL_BLACK && ! __HAS_SCRIPT) && (SPOOF_COM2COM || SPOOF_NET2COM))
score EASY_URL 0
trigger_target EASY_URL http://www.antiphishing.org/consumer_recs.html
#URI_PH_SURBL
#HTTPS_IP_MISMATCH
#NORMAL_HTTP_TO_IP
#IP_LINK_PLUS
#SPOOF_COM2OTH
#SPOOF_COM2COM
#SPOOF_NET2COM
#FB_GAPPY_ADDRESS
#WEIRD_PORT
#HTTP_EXCESSIVE_ESCAPES
#NUMERIC_HTTP_ADDR
#HTTPS_HTTP_MISMATCH
#HIGH_CODEPAGE_URI
#HTTP_ESCAPED_HOST
#URI_HEX
#trigger_config PhishTag.config

View File

@@ -0,0 +1,256 @@
package PhishTag;
use strict;
use warnings;
use Mail::SpamAssassin;
use Mail::SpamAssassin::Logger;
use vars qw(@ISA);
our @ISA = qw(Mail::SpamAssassin::Plugin);
sub new{
my ($class, $mailsa)=@_;
$class=ref($class) ||$class;
my $self = $class->SUPER::new($mailsa);
bless($self,$class);
$self->set_config($mailsa->{conf});
return $self;
}
sub set_config{
my($self, $conf) = @_;
my @cmds = ();
push (@cmds, {
setting => 'trigger_target',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_HASH_KEY_VALUE,
is_admin => 1,
});
push (@cmds, {
setting => 'trigger_config',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
is_admin => 1,
default => '',
});
push (@cmds, {
setting => 'trigger_ratio',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
is_admin => 1,
default => 0,
});
$conf->{parser}->register_commands(\@cmds);
}
#prepare the plugin
sub check_start{
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
#initialize the PHISHTAG data structure for
#saving configuration information
$pms->{PHISHTAG} = {};
$pms->{PHISHTAG}->{triggers}={};
$pms->{PHISHTAG}->{targets}=[];
#read the configuration info
$self->read_configfile($params);
$self->read_settings($params);
}
sub read_settings{
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
my $triggers= $pms->{PHISHTAG}->{triggers};
my $targets= $pms->{PHISHTAG}->{targets};
while (my ($tname,$ttarget)=each %{$pms->{conf}->{trigger_target}}){
push @$targets, [$ttarget, $tname];
$$triggers{$tname}=0;
}
}
sub read_configfile{
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
#nothing interesting here if there is not a configuration file
return if($pms->{conf}->{trigger_config} !~/\S/);
my $triggers= $pms->{PHISHTAG}->{triggers};
my $targets= $pms->{PHISHTAG}->{targets};
my $target;
open(F, '<', $pms->{conf}->{trigger_config});
while(<F>){
#each entry is separated by blank lines
undef($target) if(!/\S/);
#lines that start with pound are comments
next if(/^\s*\#/);
#an entry starts with a URL line prefixed with the word "target"
if(/^target\s+(\S+)/){
$target=[$1];
push @$targets,$target;
}
#add the test to the list of listened triggers
#and to the triggers of the last target
elsif(defined $target){
s/\s+//g;
$$triggers{$_}=0;
push @$target, $_;
}
}
close(F);
}
sub hit_rule {
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
my $rulename = $params->{rulename};
#mark the rule as hit
if(defined($pms->{PHISHTAG}->{triggers}->{$rulename})){
$pms->{PHISHTAG}->{triggers}->{$rulename}=1;
dbg("PHISHTAG: $rulename has been caught\n");
}
}
sub check_post_learn {
my ($self, $params) = @_;
my $pms = $params->{permsgstatus};
#find out which targets have fulfilled their requirements
my $triggers= $pms->{PHISHTAG}->{triggers};
my $targets= $pms->{PHISHTAG}->{targets};
my @filled=();
foreach my $target(@$targets){
my $uri= $$target[0];
my $fulfilled=1;
#all the triggers of a target have to exist for it to be fulfilled
foreach my $i(1..$#$target){
if(! $triggers->{$$target[$i]}){
$fulfilled=0;
last;
}
}
if($fulfilled){
push @filled, $uri;
dbg("PHISHTAG: Fulfilled $uri\n");
}
}
if(scalar(@filled) &&
$pms->{conf}->{trigger_ratio} > rand(100)){
$pms->{PHISHTAG}->{letgo}=0;
$pms->{PHISHTAG}->{uri}=$filled[int(rand(scalar(@filled)))];
dbg("PHISHTAG: Decided to keep this email and point to ".
$pms->{PHISHTAG}->{uri});
#make sure that SpamAssassin does not remove this email
$pms->got_hit("PHISHTAG_TOSS",
"BODY: ",
score => -100);
}
else{
dbg("PHISHTAG: Will let this email to SpamAssassin's discretion\n");
$pms->{PHISHTAG}->{letgo}=1;
}
#nothing interesting here, if we will not rewrite the email
if($pms->{PHISHTAG}->{letgo}){
return;
}
my $pristine_body=\$pms->{msg}->{pristine_body};
#dbg("PRISTINE>>\n".$$pristine_body);
my $uris = $pms->get_uri_detail_list();
#rewrite the url
while (my($uri, $info) = each %{$uris}) {
if(defined ($info->{types}->{a})){
$$pristine_body=~s/$uri/$pms->{PHISHTAG}->{uri}/mg;
}
}
dbg("PRISTINE>>\n".$$pristine_body);
}
1;
__END__
=head1 NAME
PhishTag - SpamAssassin plugin for redirecting links in incoming emails.
=head1 SYNOPSIS
=over 4
loadplugin Mail::SpamAssassin::Plugin::PhishTag
trigger_ratio 0.1
trigger_target RULE_NAME http://www.antiphishing.org/consumer_recs.html
=back
=head1 DESCRIPTION
PhishTag enables administrators to rewrite links in emails that trigger
certain tests; preferrable blacklist tests. The plugin will inhibit the
blocking of a portion of the emails that trigger the test by SpamAssassin,
and let them pass to the users' inbox after the rewrite. It is useful in
providing training to email users about company policies and general email
usage.
=head1 OPTIONS
The following options can be set by modifying the configuration file.
=over 4
=item * trigger_ratio percentage_value
Sets the probability in percentage that a positive test will trigger the
email rewrite, e.g. 0.1 will rewrite on the average 1 in 1000 emails that
match the trigger.
=item * trigger_target RULE_NAME http_url
The name of the test which would trigger the email rewrite; all the URLs
will be replaced by http_url.
=back
=head1 DOWNLOAD
The source of this plugin is avaliable at:
http://umut.topkara.org/PhishTag/PhishTag.pm
a sample configuration file is also available:
http://umut.topkara.org/PhishTag/PhishTag.cf
=head1 SEE ALSO
Check the list of tests performed by SpamAssassin to modify the
configuration file to match your needs from
http://spamassassin.apache.org/tests.html
=head1 AUTHOR
Umut Topkara, 2008, E<lt>umut@topkara.orgE<gt>
http://umut.topkara.org
=head1 COPYRIGHT AND LICENSE
This plugin is free software; you can redistribute it and/or modify
it under the same terms as SpamAssassin itself, either version 3.2.4
or, at your option, any later version of SpamAssassin you may have
available.
=cut

View File

@@ -0,0 +1,120 @@
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>
# Author: Steve Freegard <steve.freegard@fsl.com>
package Mail::SpamAssassin::Plugin::SaveHits;
my $VERSION = 0.1;
use strict;
use Mail::SpamAssassin::Plugin;
use Digest::SHA1 qw{sha1_hex};
use File::Path;
use File::Basename;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
sub dbg {
Mail::SpamAssassin::Plugin::dbg ("SaveHits: @_");
}
sub new {
my ($class, $mailsa) = @_;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsa);
bless ($self, $class);
return $self;
}
sub parse_config {
my ($self,$params) = @_;
return 0 unless ($params->{key} =~ /^savehits_(\S+)$/i);
my $key = $1;
if($key =~ /^rule$/i) {
my(@split) = split(/\s+/,$params->{value});
foreach my $rule (@split) {
$self->{main}->{conf}->{'savehits_rules'}->{$rule} = 1 if $rule;
dbg("Added rule $rule to save list");
}
}
if($key =~ /^dir$/i) {
$self->{main}->{conf}->{$params->{key}} = $params->{value};
}
$self->inhibit_further_callbacks();
return 1;
}
sub check_end {
my ($self, $params) = @_;
my ($pms) = $params->{permsgstatus};
my ($saverules) = $self->{main}->{conf}->{'savehits_rules'};
my ($savedir) = $self->{main}->{conf}->{'savehits_dir'};
return 0 if not $savedir;
my ($msg) = $pms->get_message();
my ($pristine) = $msg->get_pristine();
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime;
my ($date) = sprintf("%d%02d%02d",$year+1900,$mon+1,$mday);
my(@hits) = split(',',($pms->get_names_of_tests_hit.','.$pms->get_names_of_subtests_hit));
my(%rules);
map { $rules{$_}++ } @hits;
foreach my $rule (keys %rules) {
if(defined($saverules->{$rule})) {
dbg("Got hit: $rule");
my $hash = sha1_hex($pristine);
my $filename = "$savedir/msgs/$date/$hash";
my $linkname = "$savedir/rules/$rule/$date/$hash";
# Untaint
$filename =~ /^([\/-\@\w.]+)$/;
$filename = $1;
$linkname =~ /^([\/-\@\w.]+)$/;
$linkname = $1;
eval { mkpath(dirname($filename)); };
if ($@) {
dbg("error: $@");
return 0;
}
eval { mkpath(dirname($linkname)); };
if ($@) {
dbg("error: $@");
return 0;
}
if (! -f $filename) {
dbg("Saving message to $filename");
eval { open(FILE, ">$filename"); };
if ($@) {
dbg("error: $@");
return 0;
}
print FILE $pristine;
close(FILE);
}
if ((! -f $linkname) && (-f $filename)) {
dbg("Creating symlink $linkname to $filename");
eval { symlink($filename, $linkname); };
if ($@) {
dbg("error: $@");
return 0;
}
}
}
}
return 1;
}
1;

View File

@@ -0,0 +1,25 @@
loadplugin Mail::SpamAssassin::Plugin::iXhash /etc/mail/spamassassin/iXhash.pm
# This makes DNS queries time out after 10 seconds (2x default)
ixhash_timeout 10
# This list uses iX Magazine's spam as datasource.
body IXHASH1 eval:ixhashtest('ix.dnsbl.manitu.net')
describe IXHASH1 This mail has been classified as spam @ iX Magazine, Germany
tflags IXHASH1 net
score IXHASH1 2.5
# This list comes in @ spamtraps run by former LogIn & Solutions AG, Germany
body IXHASH2 eval:ixhashtest('generic.ixhash.net')
describe IXHASH2 mail has been classified as spam @ former LogIn&Solutions AG, Germany
tflags IXHASH2 net
score IXHASH2 1.5
body IXHASH3 eval:ixhashtest('ctyme.ixhash.net')
describe IXHASH3 mail has been classified as spam @ JunkEmailFilter, Germany
tflags IXHASH3 net
score IXHASH3 1.0
body IXHASH4 eval:ixhashtest('hosteurope.ixhash.net')
describe IXHASH4 mail has been classified as spam @ HostEurope, Germany
tflags IXHASH4 net
score IXHASH4 1.0

View File

@@ -0,0 +1,67 @@
#*************************************************************************
# Bayes OCR Plugin, version 0.1
#*************************************************************************
# Copyright 2007 P.R.A. Group - D.I.E.E. - University of Cagliari (ITA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#*************************************************************************
loadplugin BayesOCR_PLG BayesOCR_PLG.pm
# Cerberus guarded the gate to Hades and ensured
# that spirits of the dead could enter...
# BayesOCR Plugin guards the inboxes and ensures
# that only legitimate images can enter,
# spam images are detected and eated..
# Rule: BayesOCR_check(thr)
# Categorisation of text embedded in images with TextCategorisation techniques.
# Require gocr, convert (imagemagick)
body BayesOCR_PLG40 eval:BayesOCR_check(0.40, 0.50)
body BayesOCR_PLG50 eval:BayesOCR_check(0.50, 0.60)
body BayesOCR_PLG60 eval:BayesOCR_check(0.60, 0.70)
body BayesOCR_PLG70 eval:BayesOCR_check(0.70, 0.80)
body BayesOCR_PLG80 eval:BayesOCR_check(0.80, 0.90)
body BayesOCR_PLG90 eval:BayesOCR_check(0.90, 0.95)
body BayesOCR_PLG95 eval:BayesOCR_check(0.95, 0.99)
body BayesOCR_PLG99 eval:BayesOCR_check(0.99, 1.00)
describe BayesOCR_PLG40 Bayesian ImageSpam probability is 40% to 50%
describe BayesOCR_PLG50 Bayesian ImageSpam probability is 50% to 60%
describe BayesOCR_PLG60 Bayesian ImageSpam probability is 60% to 70%
describe BayesOCR_PLG70 Bayesian ImageSpam probability is 70% to 80%
describe BayesOCR_PLG80 Bayesian ImageSpam probability is 80% to 90%
describe BayesOCR_PLG90 Bayesian ImageSpam probability is 90% to 95%
describe BayesOCR_PLG95 Bayesian ImageSpam probability is 95% to 99%
describe BayesOCR_PLG99 Bayesian ImageSpam probability is 99% to 100%
add_header all BayesOCR-OUT _PLGBAYESOCROUT_
priority BayesOCR_PLG40 1000
priority BayesOCR_PLG50 1000
priority BayesOCR_PLG60 1000
priority BayesOCR_PLG70 1000
priority BayesOCR_PLG80 1000
priority BayesOCR_PLG90 1000
priority BayesOCR_PLG95 1000
priority BayesOCR_PLG99 1000
score BayesOCR_PLG40 0 0 0.5 0.5
score BayesOCR_PLG50 0 0 1.0 1.0
score BayesOCR_PLG60 0 0 1.5 1.5
score BayesOCR_PLG70 0 0 2.0 2.0
score BayesOCR_PLG80 0 0 2.7 2.7
score BayesOCR_PLG90 0 0 3.5 3.5
score BayesOCR_PLG95 0 0 4.0 4.0
score BayesOCR_PLG99 0 0 4.5 4.5

View File

@@ -0,0 +1,400 @@
#*************************************************************************
# Bayes OCR Plugin, version 0.1
#*************************************************************************
# Copyright 2007 P.R.A. Group - D.I.E.E. - University of Cagliari (ITA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#*************************************************************************
package BayesOCR_PLG;
use strict;
use Mail::SpamAssassin;
use Mail::SpamAssassin::Util;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
our @ISA = qw (Mail::SpamAssassin::Plugin);
# constructor: register the eval rule
sub new {
my ( $class, $mailsa ) = @_;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsa);
bless( $self, $class );
dbg("PLG-BayesOCR:: new:: register_eval_rule");
$self->register_eval_rule("BayesOCR_check");
$self->{'imgTxt_classifierOut'} = -1;
$self->{'imgTxt_tagmsg'} = ""; #msg to be saved in e-mail tag when $self->{'imgTxt_classifierOut'} <= 0
return $self;
}
#===========================================================================
#===========================================================================
sub check_start{
# Called before eval rule
my ( $self, $pms ) = @_;
dbg("PLG-BayesOCR:: check_start:: init score");
#Init outNB_imgTxt
$self->{'imgTxt_classifierOut'} = -1;
$self->{'imgTxt_tagmsg'} = "";
}
sub isValidUser{
my ($pms) = @_;
my $username = $pms->{main}->{username};
dbg("PLG-BayesOCR:: isValidUser:: Username: $username");
return 1;
}
sub BayesOCR_check {
# BayesOCR_check(thr)
# Return an hit when (outNB > thr)
# The score is computed as (weigth * outNB)
#
my ($self, $pms, $unused, $thrL, $thrH) = @_;
my $plgRuleName = $pms->get_current_eval_rule_name();
#if( isValidUser($pms) == 0) { return 0; }
dbg("PLG-BayesOCR:: BayesOCR_check :: Rule: $plgRuleName");
dbg("PLG-BayesOCR:: BayesOCR_check :: thr: ($thrH, $thrL)");
if($self->{'imgTxt_classifierOut'} < 0)
{
#Output
if( $self->imageSpam_OCRTextProcessing($pms ) )
{
$self->{'imgTxt_tagmsg'} = $self->{'imgTxt_classifierOut'};
}
dbg("PLG-BayesOCR:: BayesOCR_check:: Write Mail Header\n\n");
$pms->set_tag ("PLGBAYESOCROUT", $self->{'imgTxt_tagmsg'} );
}
my $resHit = ($self->{'imgTxt_classifierOut'} > $thrL) && ($self->{'imgTxt_classifierOut'} <= $thrH );
return $resHit;
}
1;
#===========================================================================
sub imageSpam_OCRTextProcessing
# boolen $self->imageSpam_OCRTextProcessing($pms)
#
# imageSpam processing by image's text analisys with SA's NaiveBayes
# return 1 : (sucess) image's text has beeen extract and processed by NB
# return 0 : (failed) no images, no text, no NB.
{
my ( $self, $pms ) = @_;
# $self :: Obj Plugin
# $pms :: Obj Mail::SpamAssassin::PerMsgStatus
# $pms->{msg} :: message of class Mail::SpamAssassin::Message
#================================
# Init result
#================================
$self->{'imgTxt_classifierOut'} = 0;
#================================
# Check & Create Classifier
#================================
my $nbSA = $pms->{main}->{bayes_scanner};
#my $nbSA = new Mail::SpamAssassin::Bayes ($pms->{main});
if( $nbSA->is_scan_available() == 0)
{
dbg("PLG-BayesOCR:: imageTextClassifierOutEstimation: NB scan not available");
$self->{'imgTxt_tagmsg'} = "0.0 (NaiveBayes not available)";
return 0;
}
#================================
# Image extraction
#================================
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: Check for Attached Images");
my ($imgTextOcr, $numImages) = imageTextExtractionFromMSG($pms->{msg});
if($numImages == 0)
{
$self->{'imgTxt_tagmsg'} = "0.0 (No images found)";
return 0;
}
# Check extracted text
my $numWord = 0;
while($imgTextOcr =~ /[a-z]{3,}/gi)
{
$numWord++;
}
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: $numWord words (3+ chars) recognised");
if($numWord <= 3)
{
$self->{'imgTxt_tagmsg'} = "0.0 (No usefull text found)";
return 0;
}
#================================
# Classifier's output estimation
#================================
# creation of msg with image's text
my $mailraw = createMSGFromText($pms, $imgTextOcr);
my $msgTmp = $pms->{main}->parse($mailraw,1);
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: Compute score with trained NaiveBayes");
my $pmsTMP = new Mail::SpamAssassin::PerMsgStatus($pms->{main}, $msgTmp);
# Classification
my $outNB = $nbSA->scan($pmsTMP, $msgTmp);
$self->{'imgTxt_classifierOut'} = sprintf("%0.3f", $outNB);
dbg("PLG-BayesOCR:: imageSpam_OCRTextProcessing:: classifier's out = $self->{'imgTxt_classifierOut'}" );
return 1; # All OK
}
#===========================================================================
sub imageTextExtractionFromMSG
# ($imgTextOcr, $numImages) = imageTextExtractionFromMSG($msg)
# Extract the text from all attached images
# Return all text anche the number of attached images
{
my $msg = $_[0];
dbg("PLG-BayesOCR:: imageTextExtractionFromMSG:: Extract & Convert Images");
my @mimeStr = ("image/*", "img/*");
my @tmpImgFile;
my $num=0;
my $imgTextOcr = "";
foreach (@mimeStr)
{
# Search all attach with current MIME
my @img_parts = $msg->find_parts($_);
for (my $i=0; $i <= $#img_parts; $i++)
{
my $imagestream = $img_parts[$i]->decode(1048000); # ~ 1 MB
$imgTextOcr = join $imgTextOcr, imageTextExtractionByOCR($imagestream), "\n";
$num++;
}
}
dbg("PLG-BayesOCR:: imageTextExtractionFromMSG:: $num images extracted");
return ($imgTextOcr, $num);
}
#===========================================================================
sub imageTextExtractionByOCR
# $textOut = imageTextExtractionByOCR( $imagestream )
# Text extraction from imge file "" by OCR engine
{
my $imagestream = $_[0];
my $imagelen = length($imagestream) / 1024;
my $tmpDir = "/tmp"; #Get tmp dir
my $tmpFile = "$tmpDir/sa_bayesOCR_tmpImg.$$";
# Zooming small images could improve OCR accuracy
# Byte Check
# > 1000K => no OCR
# < 15K => OCR + zoom 4X
# else => Check resolution
# Check resolution
# res > 1400x1050 => no OCR
# 1024x768 <= res < 1400x1050 => OCR (no zoom)
# 800x600 <= res < 1024x768 => OCR + zoom 2X
# res < 800x600 => OCR + zoom 4X
if ($imagelen > 1000)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Skip, image size = $imagelen");
return "";
}
open (FILE, ">$tmpFile.tmp") or return "";
print FILE "$imagestream \n";
close FILE;
my $convertOPT = "";
my $imageIdentifyTxt = "";
if($imagelen < 20 )
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 4X");
$convertOPT = "-sample 400% -density 280";
}
else
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Check image dim");
# check WxH
open EXEFH, "identify -quiet -ping $tmpFile.tmp |";
$imageIdentifyTxt = join "", <EXEFH>;
close EXEFH;
if( $imageIdentifyTxt =~ s/\s(\d*)x(\d*)\s//i )
{
my $size1 = $1;
my $size2 = $2;
if($size1 * $size2 > 1400*1050 && $size1 > 1280 && $size2 > 1024)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Skip, image dim = $size1 x $size2");
unlink "$tmpFile.tmp";
return "";
}
if( $size1 * $size2 < 800*600)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 4X");
$convertOPT = "-sample 400% -density 280";
}
elsif( $size1 * $size2 < 1024*768)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Enable zoom 2X");
$convertOPT = "-sample 200% -density 280";
}
}
}
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Convert & OCR");
# -append :: concatenate image i layers
# -flatten :: fuse layers
# -density :: set dpi
my $exstatus = system("convert $tmpFile.tmp -append -flatten $convertOPT $tmpFile.pnm");
if($exstatus != 0)
{
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: Convert ERROR!!");
#Catturo SDOUT e STERR
open EXEFH, "identify -verbose -strip $tmpFile.tmp 2>&1 |";
$imageIdentifyTxt = join "", <EXEFH>;
close EXEFH;
my $msg = "Stream size (kb): $imagelen\nIdentify output: \n$imageIdentifyTxt\n";
saveLogMsg($tmpDir, "Convert Error", $msg);
unlink "$tmpFile.tmp";
return "";
}
# GOCR call with timeout (thanks to B. Austin for the usefull suggestions)
my $textOut = "";
eval {
local $SIG{ALRM} = sub { die "GOCR_TIMEOUT\n" };
alarm 10;
# Retrieve gocr output
open EXEFH, "gocr $tmpFile.pnm |";
$textOut = join "", <EXEFH>;
close EXEFH;
alarm 0;
};
if ($@) {
die unless $@ eq "GOCR_TIMEOUT\n"; # propagate unexpected errors
# timed out
dbg("PLG-BayesOCR:: imageTextExtractionByOCR:: OCR timeout!!");
# Extract the list of all child of this process
open PSFH, "ps -o pid,cmd --ppid $$ |";
my $psOut = join "", <PSFH>;
close PSFH;
#Get the PID of gocr child
if( $psOut =~ s/(\d*) gocr//i)
{
kill 9, $1;
}
my $msg = "Stream size (kb): $imagelen\nPS out:\n $psOut\n";
saveLogMsg($tmpDir, "OCR timeout", $msg);
$textOut = "";
}
unlink "$tmpFile.tmp";
unlink "$tmpFile.pnm";
return $textOut;
}
#===========================================================================
sub createMSGFromText
# msg = createMSGFromText(@img_ocrText)
{
my ($pms, $ocrText) = @_;
dbg("PLG-BayesOCR: createMSGFromText:: Make temp email with OCR's text");
my $subject = "";
my $date = $pms->{msg}->get_pristine_header("Date");
my $from = ""; #$pms->{msg}->get_pristine_header("From");
my $to = ""; #$pms->{msg}->get_pristine_header("To");
my $mailraw = "From: $from\nTo: $to\nSubject: $subject\nDate: $date\nContent-Type: text/plain;\n charset=\"us-ascii\"\nContent-Disposition: inline\n\n$ocrText\n";
return $mailraw
}
#===========================================================================
#===========================================================================
sub saveLogMsg()
{
my ($tmpDir, $title, $msg) = @_;
my $timenow = localtime time;
open (FILE, ">>$tmpDir/sa_bayesOCR.log");
print FILE "#--------------------------------\n";
print FILE " $timenow\n";
print FILE " $title\n";
print FILE "#--------------------------------\n";
print FILE "$msg\n";
close FILE;
}
#===========================================================================

View File

@@ -0,0 +1,273 @@
# Adds DNSWL.org to recipients of spamassassin --report.
#
# In a SpamAssassin config file, add the lines:
#
# loadplugin Mail::SpamAssassin::Plugin::DNSWLh
# dnswl_address user@example.com
# dnswl_password yourpassword
#
# The last two must be from an account created via
# http://www.dnswl.org/registerreporter.pl
#
#
# 2010-02-26-23 Initial release.
# 2010-02-27-11 Also call report successful on unlisted IPs.
# 2010-02-28-20 State when reported email has trust level "Unlisted".
# 2010-03-02-10 Report the IP DNSWL thought was interesting.
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>
=head1 NAME
Mail::SpamAssassin::Plugin::DNSWL - perform DNSWL reporting of messages
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::DNSWL
=head1 DESCRIPTION
DNSWL is a service which lists known legitimate mail servers.
This module enables automatic reporting of spam to DNSWL, to improve
the accuracy of their database.
Note that spam reports sent by this plugin to DNSWL each include the
entire spam message.
See http://www.dnswl.org/ for more information about DNSWL.
=cut
package Mail::SpamAssassin::Plugin::DNSWLh;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
use IO::Socket;
use strict;
use warnings;
use bytes;
use re 'taint';
use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; };
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
sub new {
my $class = shift;
my $mailsaobject = shift;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsaobject);
bless ($self, $class);
# are network tests enabled?
if (!$mailsaobject->{local_tests_only} && HAS_LWP_USERAGENT) {
$self->{dnswl_available} = 1;
dbg("DNSWL: network tests on, attempting DNSWL");
}
else {
$self->{dnswl_available} = 0;
dbg("DNSWL: local tests only, disabling DNSWL");
}
$self->set_config($mailsaobject->{conf});
return $self;
}
sub set_config {
my($self, $conf) = @_;
my @cmds;
=head1 USER OPTIONS
=over 4
=cut
push (@cmds, {
setting => 'dnswl_address',
default => 'spamassassin-submit@spam.dnswl.chaosreigns.com',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^([^<\s]+\@[^>\s]+)$/) {
$self->{dnswl_address} = $1;
}
elsif ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
else {
return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
},
});
push (@cmds, {
setting => 'dnswl_password',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^(\S+)$/) {
$self->{dnswl_password} = $1;
}
elsif ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
else {
return $Mail::SpamAssassin::Conf::INVALID_VALUE;
}
},
});
=item dnswl_max_report_size (default: 50)
Messages larger than this size (in kilobytes) will be truncated in
report messages sent to DNSWL. The default setting is the maximum
size that DNSWL will accept at the time of release.
=cut
push (@cmds, {
setting => 'dnswl_max_report_size',
default => 50,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
});
$conf->{parser}->register_commands(\@cmds);
}
sub plugin_report {
my ($self, $options) = @_;
return unless $self->{dnswl_available};
#dbg("DNSWL: address/pass: " . $options->{report}->{conf}->{dnswl_address}
# .' '. $options->{report}->{conf}->{dnswl_password} );
if (!$options->{report}->{options}->{dont_report_to_dnswl}) {
if ($options->{report}->{conf}->{dnswl_address} and
$options->{report}->{conf}->{dnswl_password}) {
if ($self->dnswl_report($options)) {
$options->{report}->{report_available} = 1;
info("DNSWL: spam reported to DNSWL");
$options->{report}->{report_return} = 1;
} else {
info("DNSWL: could not report spam to DNSWL");
}
} else {
dbg("DNSWL: dnswl_address and/or dnswl_password not defined.");
}
}
}
sub dnswl_report {
my ($self, $options) = @_;
# original text
my $original = ${$options->{text}};
# check date
my $header = $original;
$header =~ s/\r?\n\r?\n.*//s;
my $date = Mail::SpamAssassin::Util::receive_date($header);
if ($date && $date < time - 2*86400) {
warn("DNSWL: Message older than 2 days, not reporting\n");
return 0;
}
# message variables
my $description = "spam report via " . Mail::SpamAssassin::Version();
my $trusted = $options->{msg}->{metadata}->{relays_trusted_str};
my $untrusted = $options->{msg}->{metadata}->{relays_untrusted_str};
# message data
# truncate message
if (length($original) > $self->{main}->{conf}->{dnswl_max_report_size} * 1024) {
substr($original, ($self->{main}->{conf}->{dnswl_max_report_size} * 1024)) =
"\n[truncated by SpamAssassin]\n";
}
my $body = <<"EOM";
Content-Description: $description
X-Spam-Relays-Trusted: $trusted
X-Spam-Relays-Untrusted: $untrusted
$original
EOM
# compose message
my $message;
$message = $body;
# send message
my %form = (
'action', 'save',
'abuseReport',$message,
);
my $ua = LWP::UserAgent->new;
my $netloc = 'www.dnswl.org:80';
my $realm = 'dnswl.org Abuse Reporting';
$ua->credentials( $netloc, $realm, $options->{report}->{conf}->{dnswl_address}, $options->{report}->{conf}->{dnswl_password} );
my $response = $ua->post('http://www.dnswl.org/abuse/report.pl', \%form);
# my $response = $ua->post('http://www.dnswl.org/abuse/report.test.pl', \%form);
# open OUT, ">/tmp/dnswlbody.".time.".txt";
# print OUT $form{'abuseReport'};
# close OUT;
if ($response->is_success) {
#if ( $response->content =~ m#Thank you for your report# ) {
if ( $response->content =~ m#IP ([\d\.]+) matches with DNSWL# ) {
my $reportedip = $1;
dbg("DNSWL: Successfully reported $reportedip.");
print "Successfully reported to DNSWL $reportedip.\n";
return 1;
#} elsif ( $response->content =~ m#No matching entry found for#) {
} elsif ( $response->content =~ m#No matching entry found for IP ([\d\.]+)#) {
my $reportedip = $1;
dbg("DNSWL: Successfully reported $reportedip. Current trust level is: Unlisted.");
print "Successfully reported to DNSWL $reportedip. Current trust level is: Unlisted.\n";
return 1;
} else {
dbg("DNSWL: Failed to report, acknowledgement not received.");
print "Failed to report to DNSWL, acknowledgement not received.\n";
# open OUT, ">/tmp/dnswlerr.".time.".txt";
# print OUT $response->content;
# close OUT;
return 0;
}
} else {
dbg("DNSWL: Failed to report: ". $response->status_line);
print "Failed to report to DNSWL, HTTP error: ". $response->status_line ."\n";
return 0;
}
dbg("DNSWL: Error: This isn't possible.");
return 0;
}
1;
=back
=cut

View File

@@ -0,0 +1,604 @@
loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs /etc/mail/spamassassin/DecodeShortURLs.pm
body HAS_SHORT_URL eval:short_url_tests()
describe HAS_SHORT_URL Message contains one or more shortened URLs
score HAS_SHORT_URL 0.01
body SHORT_URL_CHAINED eval:short_url_tests()
describe SHORT_URL_CHAINED Message has shortened URL chained to other shorteners
score SHORT_URL_CHAINED 3.0
body SHORT_URL_MAXCHAIN eval:short_url_tests()
describe SHORT_URL_MAXCHAIN Message has shortened URL that causes more than 10 redirections
score SHORT_URL_MAXCHAIN 5.0
body SHORT_URL_LOOP eval:short_url_tests()
describe SHORT_URL_LOOP Message has short URL that loops back to itself
score SHORT_URL_LOOP 0.01
body SHORT_URL_404 eval:short_url_tests()
describe SHORT_URL_404 Message has short URL that returns 404
score SHORT_URL_404 1.0
uri URI_BITLY_BLOCKED /^http:\/\/bit\.ly\/a\/warning/i
describe URI_BITLY_BLOCKED Message contains a bit.ly URL that has been disabled due to abuse
score URI_BITLY_BLOCKED 10.0
uri URI_SIMURL_BLOCKED /^http:\/\/simurl\.com\/redirect_black\.php/i
describe URI_SIMURL_BLOCKED Message contains a simurl URL that has been disabled due to abuse
score URI_SIMURL_BLOCKED 10.0
uri URI_MIGRE_BLOCKED /^http:\/\/migre\.me\/bloqueado/i
describe URI_MIGRE_BLOCKED Message contains a migre.me URL that has been disabled due to abuse
score URI_MIGRE_BLOCKED 10.0
meta SHORT_URIBL HAS_SHORT_URL && (URIBL_BLACK || URIBL_AB_SURBL || URIBL_WS_SURBL || URIBL_JP_SURBL || URIBL_SC_SURBL || URIBL_RHS_DOB || URIBL_DBL_SPAM || URIBL_SBL)
describe SHORT_URIBL Message contains shortened URL(s) and also hits a URIDNSBL
score SHORT_URIBL 0.01
url_shortener_log /tmp/DecodeShortURLs.txt
url_shortener_cache /tmp/DecodeShortURLs.sq3
#url_shortener_syslog 1
url_shortener 0rz.tw
url_shortener 1l2.us
url_shortener 1u.ro
url_shortener 1url.com
url_shortener 2.gp
url_shortener 2.ly
url_shortener 2chap.it
url_shortener 2pl.us
url_shortener 2su.de
url_shortener 2tu.us
url_shortener 2ze.us
url_shortener 3.ly
url_shortener 301.to
url_shortener 301url.com
url_shortener 307.to
# url_shortener 4sq.com
url_shortener 6url.com
url_shortener 7.ly
url_shortener 9mp.com
url_shortener a.gd
url_shortener a.gg
url_shortener a.nf
url_shortener a2a.me
url_shortener a2n.eu
url_shortener abbr.com
url_shortener abe5.com
url_shortener access.im
url_shortener ad.vu
url_shortener adf.ly
url_shortener adjix.com
url_shortener alturl.com
url_shortener amzn.com
url_shortener amzn.to
url_shortener arm.in
url_shortener asso.in
url_shortener atu.ca
url_shortener aurls.info
url_shortener awe.sm
url_shortener ayl.lv
url_shortener azqq.com
url_shortener b23.ru
url_shortener b65.com
url_shortener b65.us
url_shortener bacn.me
url_shortener beam.to
url_shortener bgl.me
url_shortener bit.ly
url_shortener bkite.com
url_shortener blippr.com
url_shortener bloat.me
url_shortener blu.cc
url_shortener bon.no
url_shortener bt.io
url_shortener budurl.com
url_shortener buk.me
url_shortener burnurl.com
url_shortener c-o.in
url_shortener c.shamekh.ws
url_shortener canurl.com
url_shortener cd4.me
url_shortener chilp.it
url_shortener chopd.it
url_shortener chpt.me
url_shortener chs.mx
url_shortener chzb.gr
url_shortener clck.ru
url_shortener cli.gs
url_shortener cliccami.info
url_shortener clickthru.ca
url_shortener clipurl.us
url_shortener clk.my
url_shortener clop.in
url_shortener clp.ly
url_shortener coge.la
url_shortener cokeurl.com
url_shortener cort.as
url_shortener cot.ag
url_shortener crum.pl
url_shortener curio.us
url_shortener cuthut.com
url_shortener cuturl.com
url_shortener cuturls.com
url_shortener dealspl.us
url_shortener decenturl.com
url_shortener df9.net
url_shortener digbig.com
url_shortener digg.com
url_shortener digipills.com
url_shortener digs.by
url_shortener dld.bz
url_shortener dlvr.it
url_shortener dn.vc
url_shortener doi.org
url_shortener doiop.com
url_shortener dr.tl
url_shortener durl.me
url_shortener durl.us
url_shortener dvlr.it
url_shortener dwarfurl.com
url_shortener easyurl.net
url_shortener eca.sh
url_shortener eclurl.com
url_shortener eepurl.com
url_shortener eezurl.com
url_shortener ewerl.com
url_shortener ezurl.eu
url_shortener fa.by
url_shortener faceto.us
url_shortener fav.me
url_shortener fb.me
url_shortener ff.im
url_shortener fff.to
url_shortener fhurl.com
url_shortener flic.kr
url_shortener flingk.com
url_shortener flq.us
url_shortener fly2.ws
url_shortener fon.gs
url_shortener foxyurl.com
url_shortener fuseurl.com
url_shortener fwd4.me
url_shortener fwdurl.net
url_shortener fwib.net
url_shortener g8l.us
url_shortener get-shorty.com
url_shortener get-url.com
url_shortener get.sh
url_shortener gi.vc
url_shortener gkurl.us
url_shortener gl.am
url_shortener go.9nl.com
url_shortener go.to
url_shortener go2.me
url_shortener golmao.com
url_shortener goo.gl
url_shortener good.ly
url_shortener goshrink.com
url_shortener gri.ms
url_shortener gurl.es
url_shortener hao.jp
url_shortener hellotxt.com
url_shortener hex.io
url_shortener hiderefer.com
url_shortener hop.im
url_shortener hotredirect.com
url_shortener hotshorturl.com
url_shortener href.in
url_shortener ht.ly
url_shortener htxt.it
url_shortener hugeurl.com
url_shortener hurl.it
url_shortener hurl.no
url_shortener hurl.ws
url_shortener icanhaz.com
url_shortener icio.us
url_shortener idek.net
url_shortener ikr.me
url_shortener ir.pe
url_shortener irt.me
url_shortener is.gd
url_shortener iscool.net
url_shortener it2.in
url_shortener ito.mx
url_shortener j.mp
url_shortener j2j.de
url_shortener jdem.cz
url_shortener jijr.com
url_shortener just.as
url_shortener k.vu
url_shortener ketkp.in
url_shortener kisa.ch
url_shortener kissa.be
url_shortener kl.am
url_shortener klck.me
url_shortener kore.us
url_shortener korta.nu
url_shortener kots.nu
url_shortener krz.ch
url_shortener ktzr.us
url_shortener kxk.me
url_shortener l.pr
url_shortener l9k.net
url_shortener liip.to
url_shortener liltext.com
url_shortener lin.cr
url_shortener lin.io
url_shortener linkbee.com
url_shortener linkee.com
url_shortener linkgap.com
url_shortener linkslice.com
url_shortener linxfix.de
url_shortener liteurl.net
url_shortener liurl.cn
url_shortener livesi.de
url_shortener lix.in
url_shortener lk.ht
url_shortener ln-s.net
url_shortener ln-s.ru
url_shortener lnk.by
url_shortener lnk.in
url_shortener lnk.ly
url_shortener lnk.ms
url_shortener lnk.sk
url_shortener lnkurl.com
url_shortener loopt.us
url_shortener lost.in
url_shortener lru.jp
url_shortener lt.tl
url_shortener lu.to
url_shortener lurl.no
url_shortener mavrev.com
url_shortener memurl.com
url_shortener merky.de
url_shortener metamark.net
url_shortener migre.me
url_shortener min2.me
url_shortener minilien.com
url_shortener minilink.org
url_shortener miniurl.com
url_shortener minurl.fr
url_shortener moby.to
url_shortener moourl.com
url_shortener msg.sg
url_shortener murl.kz
url_shortener mv2.me
url_shortener mysp.in
url_shortener myurl.in
url_shortener myurl.si
url_shortener nanoref.com
url_shortener nanourl.se
url_shortener nbx.ch
url_shortener ncane.com
url_shortener ndurl.com
url_shortener ne1.net
url_shortener netnet.me
url_shortener netshortcut.com
url_shortener ni.to
url_shortener nig.gr
url_shortener nm.ly
url_shortener nn.nf
url_shortener notlong.com
url_shortener nutshellurl.com
url_shortener nyti.ms
url_shortener o-x.fr
url_shortener o.ly
url_shortener oboeyasui.com
url_shortener offur.com
url_shortener ofl.me
url_shortener om.ly
url_shortener omf.gd
url_shortener onecent.us
url_shortener onion.com
url_shortener onsaas.info
url_shortener ooqx.com
url_shortener oreil.ly
url_shortener ow.ly
url_shortener oxyz.info
url_shortener p.ly
url_shortener p8g.tw
url_shortener parv.us
url_shortener paulding.net
url_shortener pduda.mobi
url_shortener peaurl.com
url_shortener pendek.in
url_shortener pep.si
url_shortener pic.gd
url_shortener piko.me
url_shortener ping.fm
url_shortener piurl.com
url_shortener plumurl.com
url_shortener plurl.me
url_shortener pnt.me
url_shortener poll.fm
url_shortener pop.ly
url_shortener poprl.com
url_shortener post.ly
url_shortener posted.at
url_shortener pt2.me
url_shortener ptiturl.com
url_shortener puke.it
url_shortener pysper.com
url_shortener qik.li
url_shortener qlnk.net
url_shortener qoiob.com
url_shortener qr.cx
url_shortener quickurl.co.uk
url_shortener qurl.com
url_shortener qurlyq.com
url_shortener quu.nu
url_shortener qux.in
url_shortener r.im
url_shortener rb6.me
url_shortener rde.me
url_shortener readthis.ca
url_shortener reallytinyurl.com
url_shortener redir.ec
url_shortener redirects.ca
url_shortener redirx.com
url_shortener relyt.us
url_shortener retwt.me
url_shortener ri.ms
url_shortener rickroll.it
url_shortener rivva.de
url_shortener rly.cc
url_shortener rnk.me
url_shortener rsmonkey.com
url_shortener rt.nu
url_shortener rubyurl.com
url_shortener rurl.org
url_shortener s.gnoss.us
url_shortener s3nt.com
url_shortener s4c.in
url_shortener s7y.us
url_shortener safe.mn
url_shortener safelinks.ru
url_shortener sai.ly
url_shortener SameURL.com
url_shortener sfu.ca
url_shortener shadyurl.com
url_shortener shar.es
url_shortener shim.net
url_shortener shink.de
url_shortener shorl.com
url_shortener short.ie
url_shortener short.to
url_shortener shorten.ws
url_shortener shortenurl.com
url_shortener shorterlink.com
url_shortener shortio.com
url_shortener shortlinks.co.uk
url_shortener shortn.me
url_shortener shortna.me
url_shortener shortr.me
url_shortener shorturl.com
url_shortener shortz.me
url_shortener shoturl.us
url_shortener shredu
url_shortener shredurl.com
url_shortener shrinkify.com
url_shortener shrinkr.com
url_shortener shrinkster.com
url_shortener shrinkurl.us
url_shortener shrt.fr
url_shortener shrt.ws
url_shortener shrtl.com
url_shortener shrtn.com
url_shortener shrtnd.com
url_shortener shurl.net
url_shortener shw.me
url_shortener simurl.com
url_shortener simurl.net
url_shortener simurl.org
url_shortener simurl.us
url_shortener sitelutions.com
url_shortener siteo.us
url_shortener sl.ly
url_shortener slidesha.re
url_shortener slki.ru
url_shortener smallr.com
url_shortener smallr.net
url_shortener smfu.in
url_shortener smsh.me
url_shortener smurl.com
url_shortener sn.im
url_shortener sn.vc
url_shortener snadr.it
url_shortener snipie.com
url_shortener snipr.com
url_shortener snipurl.com
url_shortener snkr.me
url_shortener snurl.com
url_shortener song.ly
url_shortener sp2.ro
url_shortener spedr.com
url_shortener sqze.it
url_shortener srnk.net
url_shortener srs.li
url_shortener starturl.com
url_shortener stickurl.com
url_shortener stpmvt.com
url_shortener sturly.com
url_shortener su.pr
url_shortener surl.co.uk
url_shortener surl.it
url_shortener t.co
url_shortener t.lh.com
url_shortener ta.gd
url_shortener takemyfile.com
url_shortener tcrn.ch
url_shortener tgr.me
url_shortener th8.us
url_shortener thecow.me
url_shortener thrdl.es
url_shortener tighturl.com
url_shortener timesurl.at
url_shortener tini.us
url_shortener tiniuri.com
url_shortener tiny.cc
url_shortener tiny.pl
url_shortener tinyarro.ws
url_shortener tinylink.com
url_shortener tinypl.us
url_shortener tinysong.com
url_shortener tinytw.it
url_shortener tinyurl.com
url_shortener tl.gd
url_shortener tllg.net
url_shortener tncr.ws
url_shortener tnw.to
url_shortener to.je
url_shortener to.ly
url_shortener to.vg
url_shortener togoto.us
url_shortener tr.im
url_shortener tr.my
url_shortener tra.kz
url_shortener traceurl.com
url_shortener trcb.me
url_shortener trg.li
url_shortener trick.ly
url_shortener trii.us
url_shortener trim.li
url_shortener trumpink.lt
url_shortener trunc.it
url_shortener truncurl.com
url_shortener tsort.us
url_shortener tubeurl.com
# url_shortener tumblr.com
url_shortener turo.us
url_shortener tw0.us
url_shortener tw1.us
url_shortener tw2.us
url_shortener tw5.us
url_shortener tw6.us
url_shortener tw8.us
url_shortener tw9.us
url_shortener twa.lk
url_shortener tweet.me
url_shortener tweetburner.com
url_shortener tweetl.com
url_shortener twi.gy
url_shortener twip.us
url_shortener twirl.at
url_shortener twit.ac
url_shortener twitclicks.com
url_shortener twitterurl.net
url_shortener twitthis.com
url_shortener twittu.ms
url_shortener twiturl.de
url_shortener twitzap.com
url_shortener twlv.net
url_shortener twtr.us
url_shortener twurl.cc
url_shortener twurl.nl
url_shortener u.mavrev.com
url_shortener u.nu
url_shortener u76.org
url_shortener ub0.cc
url_shortener uiop.me
url_shortener ulimit.com
url_shortener ulu.lu
url_shortener unfaker.it
url_shortener updating.me
url_shortener ur.ly
url_shortener ur1.ca
url_shortener urizy.com
url_shortener url.ag
url_shortener url.az
url_shortener url.co.uk
url_shortener url.go.it
url_shortener url.ie
url_shortener url.inc-x.eu
url_shortener url.lotpatrol.com
# url_shortener url4.eu
url_shortener urlao.com
url_shortener urlbee.com
url_shortener urlborg.com
url_shortener urlbrief.com
url_shortener urlcorta.es
url_shortener urlcut.com
url_shortener urlcutter.com
url_shortener urlg.info
url_shortener urlhawk.com
url_shortener urli.nl
url_shortener urlkiss.com
url_shortener urloo.com
url_shortener urlpire.com
url_shortener urltea.com
url_shortener urlu.ms
url_shortener urlvi.b
url_shortener urlvi.be
url_shortener urlx.ie
url_shortener urlz.at
url_shortener urlzen.com
url_shortener usat.ly
url_shortener uservoice.com
url_shortener ustre.am
url_shortener vado.it
url_shortener vb.ly
url_shortener vdirect.com
url_shortener vi.ly
url_shortener viigo.im
url_shortener virl.com
url_shortener vl.am
url_shortener voizle.com
url_shortener vtc.es
url_shortener w0r.me
url_shortener w33.us
url_shortener w34.us
url_shortener w3t.org
url_shortener wa9.la
url_shortener wapurl.co.uk
url_shortener webalias.com
url_shortener welcome.to
url_shortener wh.gov
url_shortener wipi.es
url_shortener wkrg.com
url_shortener woo.ly
url_shortener wp.me
url_shortener x.hypem.com
url_shortener x.se
url_shortener x.vu
url_shortener xeeurl.com
url_shortener xil.in
url_shortener xlurl.de
url_shortener xr.com
url_shortener xrl.in
url_shortener xrl.us
url_shortener xrt.me
url_shortener xurl.jp
url_shortener xxsurl.de
url_shortener xzb.cc
url_shortener yatuc.com
url_shortener ye-s.com
url_shortener yep.it
# url_shortener youtu.be
url_shortener z.pe
url_shortener zapt.in
url_shortener zi.ma
url_shortener zi.me
url_shortener zi.pe
url_shortener zip.li
url_shortener zipmyurl.com
url_shortener zootit.com
url_shortener zud.me
url_shortener zurl.ws
url_shortener zz.gd
url_shortener zzang.kr
url_shortener xn--cwg.ws
url_shortener xn--fwg.ws
url_shortener xn--bih.ws
url_shortener xn--l3h.ws
url_shortener xn--1ci.ws
url_shortener xn--odi.ws
url_shortener xn--rei.ws
url_shortener xn--3fi.ws
url_shortener xn--egi.ws
url_shortener xn--hgi.ws
url_shortener xn--ogi.ws
url_shortener xn--vgi.ws
url_shortener xn--5gi.ws
url_shortener xn--9gi.ws

View File

@@ -0,0 +1,564 @@
# <@LICENSE>
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to you under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# </@LICENSE>
# Author: Steve Freegard <steve.freegard@fsl.com>
=head1 NAME
DecodeShortURLs - Expand shortened URLs
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::DecodeShortURLs
url_shortener bit.ly
url_shortener go.to
...
=head1 DESCRIPTION
This plugin looks for URLs shortened by a list of URL shortening services and
upon finding a matching URL will connect using to the shortening service and
do an HTTP HEAD lookup and retrieve the location header which points to the
actual shortened URL, it then adds this URL to the list of URIs extracted by
SpamAssassin which can then be accessed by other plug-ins, such as URIDNSBL.
This plugin also sets the rule HAS_SHORT_URL if any matching short URLs are
found.
Regular 'uri' rules can be used to detect and score links disabled by the
shortening service for abuse and URL_BITLY_BLOCKED is supplied as an example.
It should be safe to score this rule highly on a match as experience shows
that bit.ly only blocks access to a URL if it has seen consistent abuse and
problem reports.
As of version 0.3 this plug-in will follow 'chained' shorteners e.g.
short URL -> short URL -> short URL -> real URL
If this form of chaining is found, then the rule 'SHORT_URL_CHAINED' will be
fired. If a loop is detected then 'SHORT_URL_LOOP' will be fired.
This plug-in limits the number of chained shorteners to a maximim of 10 at
which point it will fire the rule 'SHORT_URL_MAXCHAIN' and go no further.
If a shortener returns a '404 Not Found' result for the short URL then the
rule 'SHORT_URL_404' will be fired.
=head1 NOTES
This plugin runs the parsed_metadata hook with a priority of -1 so that
it may modify the parsed URI list prior to the URIDNSBL plugin which
runs as priority 0.
Currently the plugin queries a maximum of 10 distinct shortened URLs with
a maximum timeout of 5 seconds per lookup. It does not recurse and follow
'chained' shortening as the author has no examples of this happening.
=head1 ACKNOWLEDGEMENTS
A lot of this plugin has been hacked together by using other plugins as
examples. The author would particularly like to tip his hat to Karsten
Bräckelmann for the _add_uri_detail_list() function that he stole from
GUDO.pm for which this plugin would not be possible due to the SpamAssassin
API making no provision for adding to the base list of extracted URIs and
the author not knowing enough about Perl to be able to achieve this without
a good example from someone that does ;-)
=cut
package Mail::SpamAssassin::Plugin::DecodeShortURLs;
my $VERSION = 0.6;
use Mail::SpamAssassin::Plugin;
use strict;
use warnings;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
use constant HAS_LWP_USERAGENT => eval { local $SIG{'__DIE__'}; require LWP::UserAgent; };
use constant HAS_SQLITE => eval { local $SIG{'__DIE__'}; require DBD::SQLite; };
use Fcntl qw(:flock SEEK_END);
use Sys::Syslog qw(:DEFAULT setlogsock);
sub dbg {
my $msg = shift;
return Mail::SpamAssassin::Logger::dbg("DecodeShortURLs: $msg");
}
sub new {
my $class = shift;
my $mailsaobject = shift;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsaobject);
bless ($self, $class);
if ($mailsaobject->{local_tests_only} || !HAS_LWP_USERAGENT) {
$self->{disabled} = 1;
} else {
$self->{disabled} = 0;
}
unless ($self->{disabled}) {
$self->{ua} = new LWP::UserAgent;
$self->{ua}->{max_redirect} = 0;
$self->{ua}->{timeout} = 5;
$self->{ua}->env_proxy;
$self->{logging} = 0;
$self->{caching} = 0;
$self->{syslog} = 0;
}
$self->set_config($mailsaobject->{conf});
$self->register_method_priority ('parsed_metadata', -1);
$self->register_eval_rule('short_url_tests');
return $self;
}
sub set_config {
my($self, $conf) = @_;
my @cmds = ();
push (@cmds, {
setting => 'url_shortener',
default => {},
code => sub {
my ($self, $key, $value, $line) = @_;
if ($value =~ /^$/) {
return $Mail::SpamAssassin::Conf::MISSING_REQUIRED_VALUE;
}
foreach my $domain (split(/\s+/, $value)) {
$self->{url_shorteners}->{lc $domain} = 1;
}
}
});
=cut
=head1 PRIVILEGED SETTINGS
=over 4
=item url_shortener_log (default: none)
A path to a log file to be written to. The file will be created if it does
not already exist and must be writable by the user running spamassassin.
For each short URL found the following will be written to the log file:
[unix_epoch_time] <short url> => <decoded url>
=cut
push (@cmds, {
setting => 'url_shortener_log',
default => '',
is_priv => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
});
=item url_shortener_cache (default: none)
The full path to a database file to write cache entries to. The database will
be created automatically if is does not already exist but the supplied path
and file must be read/writable by the user running spamassassin or spamd.
NOTE: you will need the DBD::SQLite module installed to use this feature.
Example:
url_shortener_cache /tmp/DecodeShortURLs.sq3
=cut
push (@cmds, {
setting => 'url_shortener_cache',
default => '',
is_priv => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
});
=item url_shortener_cache_ttl (default: 86400)
The length of time a cache entry will be valid for in seconds.
Default is 86400 (1 day).
NOTE: you will also need to run the following via cron to actually remove the
records from the database:
echo "DELETE FROM short_url_cache WHERE modified < strftime('%s',now) - <ttl>; | sqlite3 /path/to/database"
NOTE: replace <ttl> above with the same value you use for this option
=cut
push (@cmds, {
setting => 'url_shortener_cache_ttl',
is_admin => 1,
default => 86400,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
});
=item url_shortener_syslog (default: 0 (off))
If this option is enabled (set to 1), then short URLs and the decoded URLs will be logged to syslog (mail.info).
=cut
push (@cmds, {
setting => 'url_shortener_syslog',
is_admin => 1,
default => 0,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL
});
$conf->{parser}->register_commands(\@cmds);
}
sub parsed_metadata {
my ($self, $opts) = @_;
my $pms = $opts->{permsgstatus};
my $msg = $opts->{msg};
return if $self->{disabled};
dbg ('warn: get_uri_detail_list() has been called already')
if exists $pms->{uri_detail_list};
# don't keep dereferencing these
$self->{url_shorteners} = $pms->{main}->{conf}->{url_shorteners};
($self->{url_shortener_log}) = ($pms->{main}->{conf}->{url_shortener_log} =~ /^(.*)$/g);
($self->{url_shortener_cache}) = ($pms->{main}->{conf}->{url_shortener_cache} =~ /^(.*)$/g);
$self->{url_shortener_cache_ttl} = $pms->{main}->{conf}->{url_shortener_cache_ttl};
$self->{url_shortener_syslog} = $pms->{main}->{conf}->{url_shortener_syslog};
# Sort short URLs into hash to de-dup them
my %short_urls;
my $uris = $pms->get_uri_detail_list();
while (my($uri, $info) = each %{$uris}) {
next unless ($info->{domains});
foreach ( keys %{ $info->{domains} } ) {
if (exists $self->{url_shorteners}->{lc $_}) {
# NOTE: $info->{domains} appears to contain all the domains parsed
# from the single input URI with no way to work out what the base
# domain is. So to prevent someone from stuffing the URI with a
# shortener to force this plug-in to follow a link that *isn't* on
# the list of shorteners; we enforce that the shortener must be the
# base URI and that a path must be present.
if ($uri !~ /^http:\/\/(?:www\.)?$_\/.+$/) {
dbg("Discarding URI: $uri");
next;
}
$short_urls{$uri} = 1;
next;
}
}
}
# Make sure we have some work to do
# Before we open any log files etc.
my $count = scalar keys %short_urls;
return undef unless $count gt 0;
# Initialise logging if enabled
if ($self->{url_shortener_log}) {
eval {
local $SIG{'__DIE__'};
open($self->{logfh}, '>>'.$self->{url_shortener_log}) or die $!;
};
if ($@) {
dbg("warn: $@");
} else {
$self->{logging} = 1;
}
}
# Initialise syslog if enabled
if ($self->{url_shortener_syslog}) {
eval {
local $SIG{'__DIE__'};
openlog('DecodeShortURLs','ndelay,pid','mail');
};
if ($@) {
dbg("warn: $@");
} else {
$self->{syslog} = 1;
}
}
# Initialise cache if enabled
if ($self->{url_shortener_cache} && HAS_SQLITE) {
eval {
local $SIG{'__DIE__'};
$self->{dbh} = DBI->connect_cached("dbi:SQLite:dbname=".$self->{url_shortener_cache},"","", {RaiseError => 1, PrintError => 0, InactiveDestroy => 1}) or die $!;
};
if ($@) {
dbg("warn: $@");
} else {
$self->{caching} = 1;
# Create database if needed
eval {
local $SIG{'__DIE__'};
$self->{dbh}->do("
CREATE TABLE IF NOT EXISTS short_url_cache (
short_url TEXT PRIMARY KEY NOT NULL,
decoded_url TEXT NOT NULL,
hits INTEGER NOT NULL DEFAULT 1,
created INTEGER NOT NULL DEFAULT (strftime('%s','now')),
modified INTEGER NOT NULL DEFAULT (strftime('%s','now'))
)
");
$self->{dbh}->do("
CREATE INDEX IF NOT EXISTS short_url_by_modified
ON short_url_cache(short_url, modified)
");
$self->{dbh}->do("
CREATE INDEX IF NOT EXISTS short_url_modified
ON short_url_cache(modified)
");
};
if ($@) {
dbg("warn: $@");
$self->{caching} = 0;
}
}
}
my $max_short_urls = 10;
foreach my $short_url (keys %short_urls) {
next if ($max_short_urls le 0);
my $location = $self->recursive_lookup($short_url, $pms);
$max_short_urls--;
}
# Close log
eval {
local $SIG{'__DIE__'};
close($self->{logfh}) or die $!;
} if $self->{logging};
# Close syslog
eval {
local $SIG{'__DIE__'};
closelog() or die $!;
} if $self->{syslog};
# Don't disconnect cached database handle
# eval { $self->{dbh}->disconnect() or die $!; } if $self->{caching};
}
sub recursive_lookup {
my ($self, $short_url, $pms, %been_here) = @_;
my $count = scalar keys %been_here;
dbg("Redirection count $count") if $count gt 0;
if ($count ge 10) {
dbg("Error: more than 10 shortener redirections");
# Fire test
$pms->got_hit('SHORT_URL_MAXCHAIN');
return undef;
}
my $location;
if ($self->{caching} && ($location = $self->cache_get($short_url))) {
dbg("Found cached $short_url => $location");
eval {
local $SIG{'__DIE__'};
$self->log_to_file("$short_url => $location")
} if $self->{logging};
syslog('info',"Found cached $short_url => $location") if $self->{syslog};
} else {
# Not cached; do lookup
my $response = $self->{ua}->head($short_url);
if (!$response->is_redirect) {
dbg("Skipping URL as not redirect: $short_url = ".$response->status_line);
$pms->got_hit('SHORT_URL_404') if($response->code == '404');
return undef;
}
$location = $response->headers->{location};
# Bail out if $short_url redirects to itself
return undef if ($short_url eq $location);
$self->cache_add($short_url, $location) if $self->{caching};
dbg("Found $short_url => $location");
eval {
local $SIG{'__DIE__'};
$self->log_to_file("$short_url => $location")
} if $self->{logging};
syslog('info',"Found $short_url => $location") if $self->{syslog};
}
# At this point we have a new URL in $response
$pms->got_hit('HAS_SHORT_URL');
_add_uri_detail_list($pms, $location);
# Set chained here otherwise we might mark a disabled page or
# redirect back to the same host as chaining incorrectly.
$pms->got_hit('SHORT_URL_CHAINED') if ($count gt 0);
# Check if we are being redirected to a local page
# Don't recurse in this case...
if($location !~ /^https?:/) {
my($host) = ($short_url =~ /^(https?:\/\/\S+)\//);
$location = "$host/$location";
dbg("Looks like a local redirection: $short_url => $location");
_add_uri_detail_list($pms, $location);
return $location;
}
# Check for recursion
if ((my ($domain) = ($location =~ /^https?:\/\/(\S+)\//))) {
if (exists $been_here{$location}) {
# Loop detected
dbg("Error: loop detected");
$pms->got_hit('SHORT_URL_LOOP');
return $location;
} else {
if (exists $self->{url_shorteners}->{$domain}) {
$been_here{$location} = 1;
# Recurse...
return $self->recursive_lookup($location, $pms, %been_here);
}
}
}
# No recursion; just return the final location...
return $location;
}
sub short_url_tests {
# Set by parsed_metadata
return 0;
}
# Beware. Code copied from PerMsgStatus get_uri_detail_list().
# Stolen from GUDO.pm
sub _add_uri_detail_list {
my ($pms, $uri) = @_;
my $info;
# Cache of text parsed URIs, as previously used by get_uri_detail_list().
push @{$pms->{parsed_uri_list}}, $uri;
$info->{types}->{parsed} = 1;
$info->{cleaned} =
[Mail::SpamAssassin::Util::uri_list_canonify (undef, $uri)];
foreach (@{$info->{cleaned}}) {
my $dom = Mail::SpamAssassin::Util::uri_to_domain($_);
if ($dom && !$info->{domains}->{$dom}) {
$info->{domains}->{$dom} = 1;
$pms->{uri_domain_count}++;
}
}
$pms->{uri_detail_list}->{$uri} = $info;
# And of course, copied code from PerMsgStatus get_uri_list(). *sigh*
dbg ('warn: PMS::get_uri_list() appears to have been harvested'),
push @{$pms->{uri_list}}, @{$info->{cleaned}}
if exists $pms->{uri_list};
}
sub log_to_file {
my ($self, $msg) = @_;
return undef if not $self->{logging};
my $fh = $self->{logfh};
eval {
flock($fh, LOCK_EX) or die $!;
seek($fh, 0, SEEK_END) or die $!;
print $fh '['.time.'] '.$msg."\n";
flock($fh, LOCK_UN) or die $!;
};
}
sub cache_add {
my ($self, $short_url, $decoded_url) = @_;
return undef if not $self->{caching};
eval {
$self->{sth_insert} = $self->{dbh}->prepare_cached("
INSERT INTO short_url_cache (short_url, decoded_url)
VALUES (?,?)
");
};
if ($@) {
dbg("warn: $@");
return undef;
};
$self->{sth_insert}->execute($short_url, $decoded_url);
return undef;
}
sub cache_get {
my ($self, $key) = @_;
return undef if not $self->{caching};
eval {
$self->{sth_select} = $self->{dbh}->prepare_cached("
SELECT decoded_url FROM short_url_cache
WHERE short_url = ? AND modified > (strftime('%s','now') - ?)
");
};
if ($@) {
dbg("warn: $@");
return undef;
}
eval {
$self->{sth_update} = $self->{dbh}->prepare_cached("
UPDATE short_url_cache
SET modified=strftime('%s','now'), hits=hits+1
WHERE short_url = ?
");
};
if ($@) {
dbg("warn: $@");
return undef;
}
$self->{sth_select}->execute($key, $self->{url_shortener_cache_ttl});
my $row = $self->{sth_select}->fetchrow_array();
if($row) {
# Found cache entry; touch it to prevent expiry
$self->{sth_update}->execute($key);
$self->{sth_select}->finish();
$self->{sth_update}->finish();
return $row;
}
$self->{sth_select}->finish();
$self->{sth_update}->finish();
return undef;
}
1;

View File

@@ -0,0 +1,3 @@
loadplugin Mail::SpamAssassin::Plugin::DNSWLh
dnswl_address bogdan@vrem.ro
dnswl_password 7llfxe

View File

@@ -0,0 +1,404 @@
=head1 NAME
Mail::SpamAssassin::Plugin::iXhash - compute fuzzy checksums from mail bodies and compare to known spam ones via DNS
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::iXhash /path/to/iXhash.pm
# Timeout in seconds - default is 10 seconds
ixhash_timeout 10
# Should we add the hashes to the messages' metadata for later re-use
# Default is not to cache hashes (i.e. re-compute them for every check)
use_ixhash_cache 0
# wether to only use perl (ixhash_pureperl = 1) or the system's 'tr' and 'md5sum'
# Default is to use Perl only
ixhash_pureperl 1
# If you should have 'tr' and/or 'md5sum' in some weird place (e.g on a Windows server)
# or you want to specify which version to use you can specifiy the exact paths here
# Default is to have SpamAssassin find the executables
ixhash_tr_path "/usr/bin/tr"
ixhash_md5sum_path "/usr/bin/md5sum"
# The actual rule
body IXHASH eval:ixhashtest('ix.dnsbl.manitu.net')
describe IXHASH This mail has been classified as spam @ iX Magazine, Germany
tflags IXHASH net
score IXHASH 1.5
=head1 DESCRIPTION
iXhash.pm is a plugin for SpamAssassin 3.0.0 and up. It takes the body of a mail, strips parts from it and then computes a hash value
from the rest. These values will then be looked up via DNS to see if the hashes have already been categorized as spam by others.
This plugin is based on parts of the procmail-based project 'NiX Spam', developed by Bert Ungerer.(un@ix.de)
For more information see http://www.heise.de/ix/nixspam/. The procmail code producing the hashes only can be found here:
ftp://ftp.ix.de/pub/ix/ix_listings/2004/05/checksums
To see which DNS zones are currently available see http://www.ixhash.net
=cut
package Mail::SpamAssassin::Plugin::iXhash;
use strict;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
use Mail::SpamAssassin::Timeout;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use Net::DNS;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
my $VERSION = "1.5.5";
sub new {
my ($class, $mailsa, $server) = @_;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsa);
bless ($self, $class);
# Are network tests enabled?
if ($mailsa->{local_tests_only}) {
dbg("IXHASH: local tests only, not using iXhash plugin");
$self->{iXhash_available} = 0;
}
else {
dbg("IXHASH: Using iXhash plugin $VERSION");
$self->{iXhash_available} = 1;
}
$self->set_config($mailsa->{conf});
$self->register_eval_rule ("ixhashtest");
return $self;
}
sub set_config {
my ($self, $conf) = @_;
my @cmds = ();
# implements iXhash_timeout config option - by dallase@uribl.com
push(@cmds, {
setting => 'ixhash_timeout',
default => 10,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'use_ixhash_cache',
default => 0,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'ixhash_pureperl',
default => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'ixhash_tr_path',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
}
);
push(@cmds, {
setting => 'ixhash_md5sum_path',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
}
);
$conf->{parser}->register_commands(\@cmds);
}
sub ixhashtest {
my ($self, $permsgstatus,$full,$dnszone) = @_;
dbg("IXHASH: IxHash querying $dnszone");
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 0){
# Return subito if we are do not find the tools we need
# Only relevant if we are those tools in the 1st way
return 0 unless $self->is_md5sum_available();
return 0 unless $self->is_tr_available();
}
my ($answer,$ixdigest) = "";
# Changed to use get_pristine_body returning a scalar
my $body = $permsgstatus->{msg}->get_pristine_body();
my $resolver = Net::DNS::Resolver->new;
my $body_copy = "";
my $rr;
my $tmpfile = '';
my $tmpfh = undef;
my $hits = 0;
my $digest = 0;
# alarm the dns query - dallase@uribl.com
# --------------------------------------------------------------------------
# here we implement proper alarms, ala Pyzor, Razor2 plugins.
# keep the alarm as $oldalarm, so we dont loose the timeout-child alarm
# see http://issues.apache.org/SpamAssassin/show_bug.cgi?id=3828#c123
my $oldalarm = 0;
my $timer = Mail::SpamAssassin::Timeout->new({ secs => $permsgstatus->{main}->{conf}->{'ixhash_timeout'}});
my $time_err = $timer->run_and_catch(sub {
# create a temporary file unless we are to use only Perl code and we don't find a hash value in metadata
# If we use the system's 'tr' and 'md5sum' utilities we need this.
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 0){
unless ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-1') or $permsgstatus->{msg}->get_metadata('X-iXhash-hash-2') or $permsgstatus->{msg}->get_metadata('X-iXhash-hash-3')) {
($tmpfile, $tmpfh) = Mail::SpamAssassin::Util::secure_tmpfile();
$body_copy = $body;
$body_copy =~ s/\r\n/\n/g;
print $tmpfh $body_copy;
close $tmpfh;
dbg ("IXHASH: Writing body to temporary file $tmpfile");
}
else {
dbg ("IXHASH: Not writing body to temporary file - reusing stored hashes");
}
}
my $digest = compute1sthash($permsgstatus,$body, $tmpfile);
if ($digest){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
}
}
}
# Only go ahead if $hits ist still 0 - i.e hash #1 didn't score a hit
if ($hits == 0 ){
$digest = compute2ndhash($permsgstatus,$body, $tmpfile);
if ($digest){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
} # end foreach
} # end if $answer
} # end if $digest
} # end if $hits
if ( $hits == 0 ){
$digest = compute3rdhash($permsgstatus,$body, $tmpfile);
if (length($digest) == 32){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
} # foreach $answer
} # end if $anser
} # end if $digest
} # end if $hits
} # end of sub{
); # end of timer->run_and_catch
if ($timer->timed_out()) {
dbg("IXHASH: ".$permsgstatus->{main}->{conf}->{'ixhash_timeout'}." second timeout exceeded while checking ".$digest.".".$dnszone."!");
}
elsif ($time_err) {
chomp $time_err;
dbg("IXHASH: iXhash lookup failed: $time_err");
}
unlink $tmpfile;
return $hits;
}
sub compute1sthash {
my ($permsgstatus, $body, $tmpfile) = @_;
my $body_copy = '';
my $digest = '';
# Creation of hash # 1 if following conditions are met:
# - mail contains 20 spaces or tabs or more - changed follwoing a suggestion by Karsten Br<42>ckelmann
# - mail consists of at least 2 lines
# This should generate the most hits (according to Bert Ungerer about 70%)
# This also is where you can tweak your plugin if you have problems with short mails FP'ing -
# simply raise that barrier here.
# We'll try to find the required hash in this message's metadata first.
# This might be the case if another zone has been queried already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1 ) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-1'))) {
dbg ("IXHASH: Hash value for method #1 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-1');
}
else
{
if (($body =~ /(?>\s.+?){20}/g) || ( $body =~ /\n.*\n/ ) ){
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1 ){
# All space class chars just one time
# Do this in two steps to avoid Perl segfaults
# if there are more than x identical chars to be replaced
# Thanks to Martin Blapp for finding that out and suggesting this workaround concerning spaces only
# Thanks to Karsten Br<42>ckelmann for pointing out this would also be the case with _any_ characater, not only spaces
$body_copy = $body;
$body_copy =~ s/\r\n/\n/g;
# Step One
$body_copy =~ s/([[:space:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:space:]])(?:\1+)/$1/g;
# remove graph class chars and some specials
$body_copy =~ s/[[:graph:]]+//go;
# Create actual digest
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value ".$digest." via method 1, using perl exclusively");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-1', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
} else {
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:space:]' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:graph:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 1, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-1', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #1 not computed, requirements not met");
}
}
return $digest;
}
sub compute2ndhash{
my ($permsgstatus, $body, $tmpfile) = @_;
my $body_copy = '';
my $digest = '';
# See if this hash has been computed already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-2'))) {
dbg ("IXHASH: Hash value for method #2 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-2');
}
else
{
# Creation of hash # 2 if mail contains at least 3 of the following characters:
# '[<>()|@*'!?,]' or the combination of ':/'
# (To match something like "Already seen? http:/host.domain.tld/")
if ($body =~ /((([<>\(\)\|@\*'!?,])|(:\/)).*?){3,}/m ) {
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1 ){
$body_copy = $body;
# remove redundant stuff
$body_copy =~ s/[[:cntrl:][:alnum:]%&#;=]+//g;
# replace '_' with '.'
$body_copy =~ tr/_/./;
# replace duplicate chars. This too suffers from a bug in perl
# so we do it in two steps
# Step One
$body_copy =~ s/([[:print:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:print:]])(?:\1+)/$1/g;
# Computing hash...
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value $digest via method 2, using perl exclusively");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-2', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
else {
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:cntrl:][:alnum:]%&#;=' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} '_' '.' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:print:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 2, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-2', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #2 not computed, requirements not met");
}
}
return $digest;
}
sub compute3rdhash{
my ($permsgstatus, $body, $tmpfile ) = @_;
my $body_copy = '';
my $digest = '';
# See if this hash has been computed already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-3'))) {
dbg ("IXHASH: Hash value for method #3 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-3');
}
else
{
# Compute hash # 3 if
# - there are at least 8 non-space characters in the body and
# - neither hash #1 nor hash #2 have been computed
# (which means $digest is still empty, in any case < 32)
if (($body =~ /[\S]{8}/) && (length($digest) < 32)) {
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1){
$body_copy = $body;
$body_copy =~ s/[[:cntrl:][:space:]=]+//g;
# replace duplicate chars. This too suffers from a bug in perl
# so we do it in two steps
# Step One
$body_copy =~ s/([[:print:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:graph:]])(?:\1+)/$1/g;
# Computing actual hash
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value $digest via method 3");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-3', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
else {
# shellcode
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:cntrl:][:space:]=' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:graph:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 3, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-3', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #3 not computed, requirements not met");
}
}
return $digest;
}
sub is_tr_available {
# Find out where your 'tr' lives
# shamelessly stolen from the Pyzor plugin code
my ($self) = @_;
my $tr = $self->{main}->{conf}->{ixhash_tr_path} || '';
unless ($tr) {
$tr = Mail::SpamAssassin::Util::find_executable_in_env_path('tr');
}
unless ($tr && -x $tr) {
dbg("IXHASH: tr is not available: no tr executable found");
return 0;
}
# remember any found tr
$self->{main}->{conf}->{ixhash_tr_path} = $tr;
dbg("IXHASH: tr is available: " . $self->{main}->{conf}->{ixhash_tr_path});
return 1;
}
sub is_md5sum_available {
# Find out where your 'md5sum' lives
# again shamelessly stolen from the Pyzor plugin code
my ($self) = @_;
my $md5sum = $self->{main}->{conf}->{ixhash_md5sum_path} || '';
unless ($md5sum) {
$md5sum = Mail::SpamAssassin::Util::find_executable_in_env_path('md5sum');
}
unless ($md5sum && -x $md5sum) {
dbg("IXHASH: md5sum is not available: no md5sum executable found");
return 0;
}
# remember any found md5sum
$self->{main}->{conf}->{ixhash_md5sum_path} = $md5sum;
dbg("IXHASH: md5sum is available: " . $self->{main}->{conf}->{ixhash_md5sum_path});
return 1;
}
1;

View File

@@ -0,0 +1,8 @@
#!/usr/bin/python
import os
# set umask
os.umask(0077)
import pyzor.client
pyzor.client.run()

View File

@@ -0,0 +1,105 @@
#!/usr/bin/python
import os
import os.path
import sys
import getopt
import pyzor
import pyzor.server
import ConfigParser
_author__ = pyzor.__author__
__version__ = pyzor.__version__
__revision__ = "$Id: pyzord,v 1.23 2002-10-09 00:33:44 ftobin Exp $"
progname = 'pyzord'
default_anonymous_allows = map(pyzor.Opname, ['check', 'report', 'ping',
'info'])
def usage():
sys.stderr.write("usage: %s [-d] [--homedir dir]\n" % progname)
sys.exit(1)
def load_access_file(access_fn, server):
server.acl = pyzor.server.ACL()
if os.path.exists(access_fn):
pyzor.server.AccessFile(open(access_fn)).feed_into(server.acl)
else:
output.warn("%s does not exist; using default ACL: allowing anonymous to do %s"
% (access_fn, default_anonymous_allows))
for op in default_anonymous_allows:
server.acl.add_entry(pyzor.server.ACLEntry((pyzor.anonymous_user,
op,
True)))
def load_passwd_file(access_fn, server):
server.passwd = pyzor.server.Passwd()
if os.path.exists(passwd_fn):
for user, key in pyzor.server.PasswdFile(open(passwd_fn)):
server.passwd[user] = key
########################################################################
# functions above, run below
# set umask
os.umask(0077)
debug = 0
(options, args) = getopt.getopt(sys.argv[1:], 'dh:', ['homedir='])
if len(args) != 0:
usage()
specified_homedir = None
for (o, v) in options:
if o == '-d':
debug = 1
elif o == '-h':
usage()
elif o == '--homedir':
specified_homedir = v
homedir = pyzor.get_homedir(specified_homedir)
if not os.path.exists(homedir):
os.mkdir(homedir)
defaults = {'port': '24441',
'listenaddress': '0.0.0.0',
'digestdb': 'pyzord.db',
'passwdfile': 'pyzord.passwd',
'accessfile': 'pyzord.access',
'CleanupAge': "%d" % pyzor.server.DBHandle.max_age,
}
config = pyzor.Config(homedir)
config.add_section('server')
for k, v in defaults.items():
config.set('server', k, v)
config.read(os.path.join(homedir, 'config'))
port = config.getint('server', 'port')
listen_adr = config.get('server', 'ListenAddress')
dbfile = config.get_filename('server', 'DigestDB')
passwd_fn = config.get_filename('server', 'passwdfile')
access_fn = config.get_filename('server', 'accessfile')
pyzor.server.DBHandle.max_age = config.getint('server', 'CleanupAge')
output = pyzor.Output(debug=debug)
pyzor.server.DBHandle.initialize(dbfile, 'c')
server = pyzor.server.Server((listen_adr, port),
pyzor.server.Log(sys.stdout))
load_passwd_file(passwd_fn, server)
load_access_file(access_fn, server)
server.serve_forever()

View File

@@ -0,0 +1,86 @@
# http://wiki.apache.org/spamassassin/RuleUpdates
CHANNELURL=updates.spamassassin.org
KEYID=5244EC45
# Ignore everything below.
return 0
This is the GPG key that updates are signed with (currently,
as of Wed Dec 21 19:31:38 PST 2005. Please contact <dev /at/
spamassassin.apache.org> with any questions.
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.2 (SunOS)
mQILBEOnbDQBEADBfda+hU8cGXD/2WYrIHsZ5CmvC2eCYKgQ87W706tzwmxoZWQS
JfnRpkZnBqS5WDhXhNBOhk9CgF5/e9yHnDQCusNYfRstKd+t0XTFvq30/tacrJNe
67zgq+DtWqIK9C7akfElc+2M5NkX6mF4cjaMXZoW17ltPy0XSSeirf584nvK3pXf
oEFLYQ/0AUV9EBpo9+i2DkMUd8d5tz7A6O5foB3ijYPzIcVtVJ1eyCg6gO1I4cIA
YbIZCH0WIVx5MQjydfKyCR4D7VFPpZgwcZ1PmyZSsy3lrigGVvYEoUS2fWTt2jUO
pB3wg5pgzuu9hN5CpChZGvq65t4PGtAeShnBkddIH4l+iDC6sAc6W06KidSaUCW1
BKvNMa39lyEkO4bfLblZRjoZbj7Tjq3wQV/PLpPyKDa8ZZ88GfWaeRDUNRgZG6Qq
e6UKlFGfrw2RXOImUje7Sjy/eG4Ud/BOeGkV913yWBm9CHsPNtaVDK+iQI6vkAWS
3QkiPjBkXGTZFHsUx9/i3k5Iga6d4Gq2cBIVBur3sDxjKuuSazLwA9OAybpzQe2s
PvTzbGc/f1P7plT++HBFlBHwFtl/v68Q8pkbMWlEc5M9nYJ6yXHATHZzFfThxBwt
OYfF25XGaclUMkOMX++RiRkmjaEaT7Whv5aPbeb3+H3v6Omjvnebge24lQAGKbQ/
dXBkYXRlcy5zcGFtYXNzYXNzaW4ub3JnIFNpZ25pbmcgS2V5IDxyZWxlYXNlQHNw
YW1hc3Nhc3Npbi5vcmc+iQI2BBMBAgAgBQJDp2w0AhsDBgsJCAcDAgQVAggDBBYC
AwECHgECF4AACgkQQFamGlJE7EVkfg//ZjBQ6UXDizX9UPsEmogWXIqbBsyP5DJH
uToaFa6OzCbOJqcYnXNfOjovYdDOTje+x3ZEkwbx+y6MSfhmDuHPDPqBU7hXenxx
oRktC68mJasKo0wXym2YfyWFnhSZMlXXFQ9We48zNGcVRckzaxLzM67BFJuRUfOM
EV6Lf3HxMvoUK3/Xzq9YPEq2sqFO1Eu+qPC3nq726Tj/aYBBFHgHmbjDrZTaQNyV
fHvEjDzPcDRjlJI+vZw1UEuXG+BKATPpiT7U7I1OGLDa2ExDIxh0+eJnsmA3YyHG
VweE7nDN2GmkXMVfa5vXHH49Ae9Ee8jIIRipfgMgZWnkZ0XYDvLj2ueH0Ixu4o9R
D2zJIwqzRh1sytG+1YOfHrOMUCplImJaY/ARgOM324ZdBvhkgIi1XvT7Sy/ZmGWd
DKFo+GjX0r2cujR8Pd4i7VlKsF9wRypk+n/aupXiaz5GY44EIVbnweyS5IlCNrwn
4UtqcB9/9uk1tmUNIcC5xjbq5ud/Y+iMIqCKCH0C9WUwSNSdsg+K+9xoZuvlaXY0
JeXWNcDdq+tMir+x+/o0U4ENVYBkSFesnotmHwN6jZj4lSMRmvcFHPBljXqLqzM+
y5wZxnCo1N7T+erZaI7BUrpJYm8JxcJ2VCWV0JFoO1Ec//B6XYB0pckbRuSTX/Zw
pKEkNqOdmjm5AgsEQ6dsigEQAKvdggbwqJgfDbRE2Lcy2gsn4j7haqu3IVBbyUDn
kGuuDuEtSeoRjCZXEb5DaKibIpEy5vzvRGvCFFkrBs4KXk/uamkgCpGnQZFnoz/S
rNZ8U7+e1pecEePpIkhQyafUKox9+p43UVoq4UybdPRDvE9SmQ1qaNUhyQY2FP9S
WT1a63u5GA73aH4puGO0BuZ9R3MNaDYZe/MOlRRjmlAsbY4oqWOudlNVaZ71EV3O
FFmOH4pnpxdO0X0l6sF6nvqvO5/gdZ3dI5iqrJjUneVgVOmPkREq7tQ5qHS/2pny
rDrH8NZCDNT5TXciBxBrt53bxxL/V/HWaolmtJi8gK82uXt8YlmT6zuEsofufDmu
P/HMDZ+BhGI+ggNzY2AVwERTRD6ecHDOI3iIuCP4Ck26YNHRCLyocL3CSlIpjQPu
tb3qfdAcqKLJ/fVyLtGkXr24crel6IeJY7/AGjYBrfh47DWnK7Xds8bAqJ8VCjOc
/q1usFTHgGkYocvtv0gmcjbu8YypzuG8HxOg9Yk9qRLQgg1fNhzXE2lqEPyMlBfj
eLmMNRvKP70fH8CK8adinPIegaRrS6gZ/iIdv8+YV+1rlEt28qzzGJxnmzUEmW6X
Xj44u91umg9WOsLxTOCQWdjGHonytHqj/xIsf45N2JIGLhU0lF04hYfEo5p65AyM
PpYhAAYpiQIfBBgBAgAJBQJDp2yKAhsCAAoJEEBWphpSROxFungP/iWKe7o8szOz
VmXkj89xDVFZ69nthVKkbgSYIZYQC+QLF8P1MWRnNWO/8TY+XsaCT3SrqxDFQ/R/
9mlAPGUM1ySVihOPmP/DPiOlWLCsc0mb6OzYF2olcOR33s05MqvJlqXSmIrdB+hI
KkC7G5byZ+XZwPXVj4XlxIEOzs18+0YJqy0IPZPXTiMet4k2KyWyWkJpJYUCb19G
R6QC8hZQD97EYTbkbr5Ss26jjY/9AqLofW5F1/98pLDo+ron7pI2k8Ymn5DngEsa
XoGsQuyvPfTAjS4p9q/XwExJcX3gvQesdw18mpoSaGAOgDISolBPRqpHpy7v7vuw
3UMnsefKOX3F0Rossevw+c2/JCulnGmJDlgz6nHSR6FhHsbrDKF8oBeYPfGW/Kjw
NvzB1i9yubAMrsTQVu1Q8e5LsnL/MNYKb6oEJbBywdeHxBkehGWFXVdSoFvVSih/
VNqX9f7jlybpLZW/n8cQ2r1ax19v7FleO/xSGvkYm7B1+4BW0mjy6A5dta5+e5WG
D5R06Uya3/xRAPGdmV6t4Mw8fFsuyCvs+vC73PR3+eS1UvCYsDpcQD8KpVBnsHaA
duWRKKhjuFL0vdOWAr25tFOTKAj5Ywas47PBukO0isov2WBCA1rVqOr6FUvdP76y
mqHv/0E6/vnTLxFoNsu4Ce42nAQ/A/jRiQQ+BBgBAgAJAhsCBQJHhbheAinBXSAE
GQECAAYFAkOnbIoACgkQbFU5eCT0NM68MQ/8DvYqxRm3vP0Gwnr+63kzET8S+6vf
gxOghnU+eMlqUeUu/ajqnVDMzoAIRDw9QgQc9ZZoklOSJQwOuloAbdpL4TwQ2XfJ
MLU60JkZWnEOXJwClb0qG1GqtcBPbMEUPfZcQfphdRL3jpWZlaexFiJRSD+A0riw
7q3NZKPDt4FrF7F3GY9krFy+P0nRt5f462DeDhCYZgguBQH+oGtjc5Hx+kOVWDsS
txo5xkt4/0DG50ZklPkTlCohmJwRLACy+NswdQ9q83eWAhzKOPgkal7xF6a+LyE+
ytVYy2EgEU74r2gVw5iizy92FDj//Z2QAUyf/c4BMuAhvfwVIHd8n2DPHvpMP15L
6fwoymh0OjzmhwK94Z2u1YqNC1CK27/hfB6okQ/Tct7/Ik61dBjtiYdUC9tTA5Ze
W8X5ouSmttS1QFixx+Z4hiXV7Qj12lgVKuJohjrVshfcbVzTHljjAo3YkOZIHIoA
IJTUMRNzTIx9k4hrPVbxbVQhKjKTwFNtBuxvmptGTcLEIv9THpqlq8jkcStJ2Zrd
hhofPCWRT/Kzo+WE+Kgefv88T5Li7Ku12U/UpiK85+6nRspXj3rnkfDOUbLZjGM+
1NET0xQTPuyxN6CXF7MMxfGCpszCudYxMANDQqNXu9brcPN/+EIxGRjqin4E7q+h
kYUaY7Ki8mXtJ8cJEEBWphpSROxFktcQALWQv996bFq1iFcGuQ0ITxNDlOWCsses
bgEM5zR10DH+6s2bXEO8xyDHQJtrvdCPetRDosnuOToBMnGMXTYVytnWzwwAzwq1
YM+bGAeTHaIX+2UmxwFyX4GMOdqsNB+xDZ8pmRKjamJSgUQt6e18YpZlg1Y4QkxS
Vptq7OZBjiKeLUhLhGJ6GWgEIedLcoCtFzKCfz3zwn0Oxl+1EnVu8yqN+quWTf8P
7EZn+0ztqZY059BrcK2jmOyXvtOZBcAHXCUknh/uPHwAJV2WFWSNid2kNiLOrV+J
3eLTs5sF9wNhxWRhl6/10cwTzjy0Onv5cJh2tjdwksigMRMwz4c839zXORni/tnY
+IY22kNTKu84gB8rBuqUq8MQXNdS3bbROwwNUzpC0D1C1z1fBvyXDL1EwJdz70Wc
2m/Sw6tIid5g98+XMW+Ibt43Jk2XbK71JLhbVbePbAcHVh/UXEtnjhRfX7oyWlwS
a+lkKMiJd/6CQ6bvYsgklE7uEzTpRskpkkOcCk1O+8jfl+DsDwKrvVaNu8tpx45k
TtV4JDA6iEHKakD/zZdVTR79W2CFqBvRfRikc5INOl1OfMQ4ODmjkMl3yI9wrHwS
SQQxdq2XsS7xbU9HDFBEguQDu0rfzILZ9DuKIVHyr/CsRoJ5joj+JvKaUQC81ywQ
aB8EKy5bg4U6
=IbYW
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -0,0 +1,3 @@
loadplugin Mail::SpamAssassin::Plugin::DNSWLh
dnswl_address bogdan@vrem.ro
dnswl_password 7llfxe

404
mail/spamassassin/iXhash.pm Normal file
View File

@@ -0,0 +1,404 @@
=head1 NAME
Mail::SpamAssassin::Plugin::iXhash - compute fuzzy checksums from mail bodies and compare to known spam ones via DNS
=head1 SYNOPSIS
loadplugin Mail::SpamAssassin::Plugin::iXhash /path/to/iXhash.pm
# Timeout in seconds - default is 10 seconds
ixhash_timeout 10
# Should we add the hashes to the messages' metadata for later re-use
# Default is not to cache hashes (i.e. re-compute them for every check)
use_ixhash_cache 0
# wether to only use perl (ixhash_pureperl = 1) or the system's 'tr' and 'md5sum'
# Default is to use Perl only
ixhash_pureperl 1
# If you should have 'tr' and/or 'md5sum' in some weird place (e.g on a Windows server)
# or you want to specify which version to use you can specifiy the exact paths here
# Default is to have SpamAssassin find the executables
ixhash_tr_path "/usr/bin/tr"
ixhash_md5sum_path "/usr/bin/md5sum"
# The actual rule
body IXHASH eval:ixhashtest('ix.dnsbl.manitu.net')
describe IXHASH This mail has been classified as spam @ iX Magazine, Germany
tflags IXHASH net
score IXHASH 1.5
=head1 DESCRIPTION
iXhash.pm is a plugin for SpamAssassin 3.0.0 and up. It takes the body of a mail, strips parts from it and then computes a hash value
from the rest. These values will then be looked up via DNS to see if the hashes have already been categorized as spam by others.
This plugin is based on parts of the procmail-based project 'NiX Spam', developed by Bert Ungerer.(un@ix.de)
For more information see http://www.heise.de/ix/nixspam/. The procmail code producing the hashes only can be found here:
ftp://ftp.ix.de/pub/ix/ix_listings/2004/05/checksums
To see which DNS zones are currently available see http://www.ixhash.net
=cut
package Mail::SpamAssassin::Plugin::iXhash;
use strict;
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Logger;
use Mail::SpamAssassin::Timeout;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use Net::DNS;
use vars qw(@ISA);
@ISA = qw(Mail::SpamAssassin::Plugin);
my $VERSION = "1.5.5";
sub new {
my ($class, $mailsa, $server) = @_;
$class = ref($class) || $class;
my $self = $class->SUPER::new($mailsa);
bless ($self, $class);
# Are network tests enabled?
if ($mailsa->{local_tests_only}) {
dbg("IXHASH: local tests only, not using iXhash plugin");
$self->{iXhash_available} = 0;
}
else {
dbg("IXHASH: Using iXhash plugin $VERSION");
$self->{iXhash_available} = 1;
}
$self->set_config($mailsa->{conf});
$self->register_eval_rule ("ixhashtest");
return $self;
}
sub set_config {
my ($self, $conf) = @_;
my @cmds = ();
# implements iXhash_timeout config option - by dallase@uribl.com
push(@cmds, {
setting => 'ixhash_timeout',
default => 10,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'use_ixhash_cache',
default => 0,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'ixhash_pureperl',
default => 1,
type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC,
}
);
push(@cmds, {
setting => 'ixhash_tr_path',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
}
);
push(@cmds, {
setting => 'ixhash_md5sum_path',
type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING,
}
);
$conf->{parser}->register_commands(\@cmds);
}
sub ixhashtest {
my ($self, $permsgstatus,$full,$dnszone) = @_;
dbg("IXHASH: IxHash querying $dnszone");
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 0){
# Return subito if we are do not find the tools we need
# Only relevant if we are those tools in the 1st way
return 0 unless $self->is_md5sum_available();
return 0 unless $self->is_tr_available();
}
my ($answer,$ixdigest) = "";
# Changed to use get_pristine_body returning a scalar
my $body = $permsgstatus->{msg}->get_pristine_body();
my $resolver = Net::DNS::Resolver->new;
my $body_copy = "";
my $rr;
my $tmpfile = '';
my $tmpfh = undef;
my $hits = 0;
my $digest = 0;
# alarm the dns query - dallase@uribl.com
# --------------------------------------------------------------------------
# here we implement proper alarms, ala Pyzor, Razor2 plugins.
# keep the alarm as $oldalarm, so we dont loose the timeout-child alarm
# see http://issues.apache.org/SpamAssassin/show_bug.cgi?id=3828#c123
my $oldalarm = 0;
my $timer = Mail::SpamAssassin::Timeout->new({ secs => $permsgstatus->{main}->{conf}->{'ixhash_timeout'}});
my $time_err = $timer->run_and_catch(sub {
# create a temporary file unless we are to use only Perl code and we don't find a hash value in metadata
# If we use the system's 'tr' and 'md5sum' utilities we need this.
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 0){
unless ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-1') or $permsgstatus->{msg}->get_metadata('X-iXhash-hash-2') or $permsgstatus->{msg}->get_metadata('X-iXhash-hash-3')) {
($tmpfile, $tmpfh) = Mail::SpamAssassin::Util::secure_tmpfile();
$body_copy = $body;
$body_copy =~ s/\r\n/\n/g;
print $tmpfh $body_copy;
close $tmpfh;
dbg ("IXHASH: Writing body to temporary file $tmpfile");
}
else {
dbg ("IXHASH: Not writing body to temporary file - reusing stored hashes");
}
}
my $digest = compute1sthash($permsgstatus,$body, $tmpfile);
if ($digest){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
}
}
}
# Only go ahead if $hits ist still 0 - i.e hash #1 didn't score a hit
if ($hits == 0 ){
$digest = compute2ndhash($permsgstatus,$body, $tmpfile);
if ($digest){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
} # end foreach
} # end if $answer
} # end if $digest
} # end if $hits
if ( $hits == 0 ){
$digest = compute3rdhash($permsgstatus,$body, $tmpfile);
if (length($digest) == 32){
dbg ("IXHASH: Now checking $digest.$dnszone");
# Now check via DNS query
$answer = $resolver->search($digest.'.'.$dnszone, "A", "IN");
if ($answer) {
foreach $rr ($answer->answer) {
next unless $rr->type eq "A";
dbg ("IXHASH: Received reply from $dnszone:". $rr->address);
$hits = 1 if $rr->address =~ /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}/;
} # foreach $answer
} # end if $anser
} # end if $digest
} # end if $hits
} # end of sub{
); # end of timer->run_and_catch
if ($timer->timed_out()) {
dbg("IXHASH: ".$permsgstatus->{main}->{conf}->{'ixhash_timeout'}." second timeout exceeded while checking ".$digest.".".$dnszone."!");
}
elsif ($time_err) {
chomp $time_err;
dbg("IXHASH: iXhash lookup failed: $time_err");
}
unlink $tmpfile;
return $hits;
}
sub compute1sthash {
my ($permsgstatus, $body, $tmpfile) = @_;
my $body_copy = '';
my $digest = '';
# Creation of hash # 1 if following conditions are met:
# - mail contains 20 spaces or tabs or more - changed follwoing a suggestion by Karsten Br<42>ckelmann
# - mail consists of at least 2 lines
# This should generate the most hits (according to Bert Ungerer about 70%)
# This also is where you can tweak your plugin if you have problems with short mails FP'ing -
# simply raise that barrier here.
# We'll try to find the required hash in this message's metadata first.
# This might be the case if another zone has been queried already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1 ) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-1'))) {
dbg ("IXHASH: Hash value for method #1 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-1');
}
else
{
if (($body =~ /(?>\s.+?){20}/g) || ( $body =~ /\n.*\n/ ) ){
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1 ){
# All space class chars just one time
# Do this in two steps to avoid Perl segfaults
# if there are more than x identical chars to be replaced
# Thanks to Martin Blapp for finding that out and suggesting this workaround concerning spaces only
# Thanks to Karsten Br<42>ckelmann for pointing out this would also be the case with _any_ characater, not only spaces
$body_copy = $body;
$body_copy =~ s/\r\n/\n/g;
# Step One
$body_copy =~ s/([[:space:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:space:]])(?:\1+)/$1/g;
# remove graph class chars and some specials
$body_copy =~ s/[[:graph:]]+//go;
# Create actual digest
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value ".$digest." via method 1, using perl exclusively");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-1', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
} else {
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:space:]' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:graph:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 1, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-1', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #1 not computed, requirements not met");
}
}
return $digest;
}
sub compute2ndhash{
my ($permsgstatus, $body, $tmpfile) = @_;
my $body_copy = '';
my $digest = '';
# See if this hash has been computed already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-2'))) {
dbg ("IXHASH: Hash value for method #2 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-2');
}
else
{
# Creation of hash # 2 if mail contains at least 3 of the following characters:
# '[<>()|@*'!?,]' or the combination of ':/'
# (To match something like "Already seen? http:/host.domain.tld/")
if ($body =~ /((([<>\(\)\|@\*'!?,])|(:\/)).*?){3,}/m ) {
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1 ){
$body_copy = $body;
# remove redundant stuff
$body_copy =~ s/[[:cntrl:][:alnum:]%&#;=]+//g;
# replace '_' with '.'
$body_copy =~ tr/_/./;
# replace duplicate chars. This too suffers from a bug in perl
# so we do it in two steps
# Step One
$body_copy =~ s/([[:print:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:print:]])(?:\1+)/$1/g;
# Computing hash...
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value $digest via method 2, using perl exclusively");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-2', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
else {
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:cntrl:][:alnum:]%&#;=' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} '_' '.' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:print:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 2, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-2', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #2 not computed, requirements not met");
}
}
return $digest;
}
sub compute3rdhash{
my ($permsgstatus, $body, $tmpfile ) = @_;
my $body_copy = '';
my $digest = '';
# See if this hash has been computed already
if (($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) && ($permsgstatus->{msg}->get_metadata('X-iXhash-hash-3'))) {
dbg ("IXHASH: Hash value for method #3 found in metadata, re-using that one");
$digest = $permsgstatus->{msg}->get_metadata('X-iXhash-hash-3');
}
else
{
# Compute hash # 3 if
# - there are at least 8 non-space characters in the body and
# - neither hash #1 nor hash #2 have been computed
# (which means $digest is still empty, in any case < 32)
if (($body =~ /[\S]{8}/) && (length($digest) < 32)) {
if ($permsgstatus->{main}->{conf}->{'ixhash_pureperl'} == 1){
$body_copy = $body;
$body_copy =~ s/[[:cntrl:][:space:]=]+//g;
# replace duplicate chars. This too suffers from a bug in perl
# so we do it in two steps
# Step One
$body_copy =~ s/([[:print:]]{100})(?:\1+)/$1/g;
# Step Two
$body_copy =~ s/([[:graph:]])(?:\1+)/$1/g;
# Computing actual hash
$digest = md5_hex($body_copy);
dbg ("IXHASH: Computed hash-value $digest via method 3");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-3', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
else {
# shellcode
$digest = `cat $tmpfile | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d '[:cntrl:][:space:]=' | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -s '[:graph:]' | $permsgstatus->{main}->{conf}->{ixhash_md5sum_path} | $permsgstatus->{main}->{conf}->{ixhash_tr_path} -d ' -'`;
chop($digest);
dbg ("IXHASH: Computed hash-value ".$digest." via method 3, using system utilities");
$permsgstatus->{msg}->put_metadata('X-iXhash-hash-3', $digest) if ($permsgstatus->{main}->{conf}->{'use_ixhash_cache'} == 1) ;
}
}
else
{
dbg ("IXHASH: Hash value #3 not computed, requirements not met");
}
}
return $digest;
}
sub is_tr_available {
# Find out where your 'tr' lives
# shamelessly stolen from the Pyzor plugin code
my ($self) = @_;
my $tr = $self->{main}->{conf}->{ixhash_tr_path} || '';
unless ($tr) {
$tr = Mail::SpamAssassin::Util::find_executable_in_env_path('tr');
}
unless ($tr && -x $tr) {
dbg("IXHASH: tr is not available: no tr executable found");
return 0;
}
# remember any found tr
$self->{main}->{conf}->{ixhash_tr_path} = $tr;
dbg("IXHASH: tr is available: " . $self->{main}->{conf}->{ixhash_tr_path});
return 1;
}
sub is_md5sum_available {
# Find out where your 'md5sum' lives
# again shamelessly stolen from the Pyzor plugin code
my ($self) = @_;
my $md5sum = $self->{main}->{conf}->{ixhash_md5sum_path} || '';
unless ($md5sum) {
$md5sum = Mail::SpamAssassin::Util::find_executable_in_env_path('md5sum');
}
unless ($md5sum && -x $md5sum) {
dbg("IXHASH: md5sum is not available: no md5sum executable found");
return 0;
}
# remember any found md5sum
$self->{main}->{conf}->{ixhash_md5sum_path} = $md5sum;
dbg("IXHASH: md5sum is available: " . $self->{main}->{conf}->{ixhash_md5sum_path});
return 1;
}
1;

View File

@@ -0,0 +1,36 @@
# This is the right place to customize your installation of SpamAssassin.
#
# See 'perldoc Mail::SpamAssassin::Conf' for details of what can be
# tweaked.
#
# This file contains plugin activation commands for plugins included
# in SpamAssassin 3.0.x releases. It will not be installed if you
# already have a file in place called "init.pre".
#
# There are now multiple files read to enable plugins in the
# /etc/mail/spamassassin directory; previously only one, "init.pre" was
# read. Now both "init.pre", "v310.pre", and any other files ending in
# ".pre" will be read. As future releases are made, new plugins will be
# added to new files, named according to the release they're added in.
###########################################################################
# RelayCountry - add metadata for Bayes learning, marking the countries
# a message was relayed through
#
# Note: This requires the IP::Country::Fast Perl module
#
# loadplugin Mail::SpamAssassin::Plugin::RelayCountry
# URIDNSBL - look up URLs found in the message against several DNS
# blocklists.
#
loadplugin Mail::SpamAssassin::Plugin::URIDNSBL
# Hashcash - perform hashcash verification.
#
loadplugin Mail::SpamAssassin::Plugin::Hashcash
# SPF - perform SPF verification.
#
loadplugin Mail::SpamAssassin::Plugin::SPF

View File

@@ -0,0 +1,29 @@
# This is the right place to customize your installation of SpamAssassin.
#
# See 'perldoc Mail::SpamAssassin::Conf' for details of what can be
# tweaked.
#
# This file contains plugin activation commands for plugins included
# in SpamAssassin 3.0.x releases. It will not be installed if you
# already have a file in place called "init.pre".
#
# There are now multiple files read to enable plugins in the
# /etc/mail/spamassassin directory; previously only one, "init.pre" was
# read. Now both "init.pre", "v310.pre", and any other files ending in
# ".pre" will be read. As future releases are made, new plugins will be
# added to new files, named according to the release they're added in.
###########################################################################
# URIDNSBL - look up URLs found in the message against several DNS
# blocklists.
#
loadplugin Mail::SpamAssassin::Plugin::URIDNSBL
# Hashcash - perform hashcash verification.
#
loadplugin Mail::SpamAssassin::Plugin::Hashcash
# SPF - perform SPF verification.
#
loadplugin Mail::SpamAssassin::Plugin::SPF

View File

@@ -0,0 +1,3 @@
#!/bin/sh
/bin/nice /usr/bin/sa-learn --spam -u exim --dir /var/spool/exim/spam > /dev/null
rm -f /var/spool/exim4/spam/* > /dev/null

View File

@@ -0,0 +1,518 @@
"""networked spam-signature detection"""
__author__ = "Frank J. Tobin, ftobin@neverending.org"
__version__ = "0.5.0"
__revision__ = "$Id: __init__.py,v 1.43 2002-09-17 15:12:58 ftobin Exp $"
import os
import os.path
import re
import sys
import sha
import tempfile
import random
import ConfigParser
import rfc822
import cStringIO
import time
proto_name = 'pyzor'
proto_version = 2.0
class CommError(Exception):
"""Something in general went wrong with the transaction"""
pass
class ProtocolError(CommError):
"""Something is wrong with talking the protocol"""
pass
class TimeoutError(CommError):
pass
class IncompleteMessageError(ProtocolError):
pass
class UnsupportedVersionError(ProtocolError):
pass
class SignatureError(CommError):
"""unknown user, signature on msg invalid,
or not within allowed time range"""
pass
class Singleton(object):
__slots__ = []
def __new__(cls, *args, **kwds):
it = cls.__dict__.get('__it__')
if it is None:
cls.__it__ = object.__new__(cls)
return cls.__it__
class BasicIterator(object):
def __iter__(self):
return self
def next(self):
raise NotImplementedError
class Username(str):
user_pattern = re.compile(r'^[-\.\w]+$')
def __init__(self, s):
self.validate()
def validate(self):
if not self.user_pattern.match(self):
raise ValueError, "%s is an invalid username" % self
class Opname(str):
op_pattern = re.compile(r'^[-\.\w]+$')
def __init__(self, s):
self.validate()
def validate(self):
if not self.op_pattern.match(self):
raise ValueError, "%s is an invalid username" % self
class Output(Singleton):
do_debug = False
quiet = False
def __init__(self, quiet=None, debug=None):
if quiet is not None: self.quiet = quiet
if debug is not None: self.do_debug = debug
def data(self, msg):
print msg
def warn(self, msg):
if not self.quiet: sys.stderr.write('%s\n' % msg)
def debug(self, msg):
if self.do_debug: sys.stderr.write('%s\n' % msg)
class DataDigest(str):
# hex output doubles digest size
value_size = sha.digest_size * 2
def __init__(self, value):
if len(value) != self.value_size:
raise ValueError, "invalid digest value size"
class DataDigestSpec(list):
"""a list of tuples, (perc_offset, length)"""
def validate(self):
for t in self:
self.validate_tuple(t)
def validate_tuple(t):
(perc_offset, length) = t
if not (0 <= perc_offset < 100):
raise ValueError, "offset percentage out of bounds"
if not length > 0:
raise ValueError, "piece lengths must be positive"
validate_tuple = staticmethod(validate_tuple)
def netstring(self):
# flattened, commified
return ','.join(map(str, reduce(lambda x, y: x + y, self, ())))
def from_netstring(self, s):
new_spec = apply(self)
expanded_list = s.split(',')
if len(extended_list) % 2 != 0:
raise ValueError, "invalid list parity"
for i in range(0, len(expanded_list), 2):
perc_offset = int(expanded_list[i])
length = int(expanded_list[i+1])
self.validate_tuple(perc_offset, length)
new_spec.append((perc_offset, length))
return new_spec
from_netstring = classmethod(from_netstring)
class Message(rfc822.Message, object):
def __init__(self, fp=None):
if fp is None:
fp = cStringIO.StringIO()
super(Message, self).__init__(fp)
self.setup()
def setup(self):
"""called after __init__, designed to be extended"""
pass
def init_for_sending(self):
if __debug__:
self.ensure_complete()
def __str__(self):
s = ''.join(self.headers)
s += '\n'
self.rewindbody()
# okay to slurp since we're dealing with UDP
s += self.fp.read()
return s
def __nonzero__(self):
# just to make sure some old code doesn't try to use this
raise NotImplementedError
def ensure_complete(self):
pass
class ThreadedMessage(Message):
def init_for_sending(self):
if not self.has_key('Thread'):
self.set_thread(ThreadId.generate())
assert self.has_key('Thread')
self.setdefault('PV', str(proto_version))
super(ThreadedMessage, self).init_for_sending()
def ensure_complete(self):
if not (self.has_key('PV') and self.has_key('Thread')):
raise IncompleteMessageError, \
"doesn't have fields for a ThreadedMessage"
super(ThreadedMessage, self).ensure_complete()
def get_protocol_version(self):
return float(self['PV'])
def get_thread(self):
return ThreadId(self['Thread'])
def set_thread(self, i):
typecheck(i, ThreadId)
self['Thread'] = str(i)
class MacEnvelope(Message):
ts_diff_max = 300
def ensure_complete(self):
if not (self.has_key('User')
and self.has_key('Time')
and self.has_key('Sig')):
raise IncompleteMessageError, \
"doesn't have fields for a MacEnvelope"
super(MacEnvelope, self).ensure_complete()
def get_submsg(self, factory=ThreadedMessage):
self.rewindbody()
return apply(factory, (self.fp,))
def verify_sig(self, user_key):
typecheck(user_key, long)
user = Username(self['User'])
ts = int(self['Time'])
said_sig = self['Sig']
hashed_user_key = self.hash_key(user_key, user)
if abs(time.time() - ts) > self.ts_diff_max:
raise SignatureError, "timestamp not within allowed range"
msg = self.get_submsg()
calc_sig = self.sign_msg(hashed_user_key, ts, msg)
if not (calc_sig == said_sig):
raise SignatureError, "invalid signature"
def wrap(self, user, key, msg):
"""This should be used to create a MacEnvelope"""
typecheck(user, str)
typecheck(msg, Message)
typecheck(key, long)
env = apply(self)
ts = int(time.time())
env['User'] = user
env['Time'] = str(ts)
env['Sig'] = self.sign_msg(self.hash_key(key, user),
ts, msg)
env.fp.write(str(msg))
return env
wrap = classmethod(wrap)
def hash_msg(msg):
"""returns a digest object"""
typecheck(msg, Message)
return sha.new(str(msg))
hash_msg = staticmethod(hash_msg)
def hash_key(key, user):
"""returns lower(H(U + ':' + lower(hex(K))))"""
typecheck(key, long)
typecheck(user, Username)
return sha.new("%s:%x" % (Username, key)).hexdigest().lower()
hash_key = staticmethod(hash_key)
def sign_msg(self, hashed_key, ts, msg):
"""ts is timestamp for message (epoch seconds)
S = H(H(M) + ':' T + ':' + K)
M is message
T is decimal epoch timestamp
K is hashed_key
returns a digest object"""
typecheck(ts, int)
typecheck(msg, Message)
typecheck(hashed_key, str)
h_msg = self.hash_msg(msg)
return sha.new("%s:%d:%s" % (h_msg.digest(), ts, hashed_key)).hexdigest().lower()
sign_msg = classmethod(sign_msg)
class Response(ThreadedMessage):
ok_code = 200
def ensure_complete(self):
if not(self.has_key('Code') and self.has_key('Diag')):
raise IncompleteMessageError, \
"doesn't have fields for a Response"
super(Response, self).ensure_complete()
def is_ok(self):
return self.get_code() == self.ok_code
def get_code(self):
return int(self['Code'])
def get_diag(self):
return self['Diag']
def head_tuple(self):
return (self.get_code(), self.get_diag())
class Request(ThreadedMessage):
"""this is class that should be used to read in Requests of any type.
subclasses are responsible for setting 'Op' if they are generating
a message"""
def get_op(self):
return self['Op']
def ensure_complete(self):
if not self.has_key('Op'):
raise IncompleteMessageError, \
"doesn't have fields for a Request"
super(Request, self).ensure_complete()
class ClientSideRequest(Request):
def setup(self):
super(Request, self).setup()
self.setdefault('Op', self.op)
class PingRequest(ClientSideRequest):
op = Opname('ping')
class ShutdownRequest(ClientSideRequest):
op = Opname('shutdown')
class SimpleDigestBasedRequest(ClientSideRequest):
def __init__(self, digest):
typecheck(digest, str)
super(SimpleDigestBasedRequest, self).__init__()
self.setdefault('Op-Digest', digest)
class CheckRequest(SimpleDigestBasedRequest):
op = Opname('check')
class InfoRequest(SimpleDigestBasedRequest):
op = Opname('info')
class SimpleDigestSpecBasedRequest(SimpleDigestBasedRequest):
def __init__(self, digest, spec):
typecheck(digest, str)
typecheck(spec, DataDigestSpec)
super(SimpleDigestSpecBasedRequest, self).__init__(digest)
self.setdefault('Op-Spec', spec.netstring())
class ReportRequest(SimpleDigestSpecBasedRequest):
op = Opname('report')
class WhitelistRequest(SimpleDigestSpecBasedRequest):
op = Opname('whitelist')
class ErrorResponse(Response):
def __init__(self, code, s):
typecheck(code, int)
typecheck(s, str)
super(ErrorResponse, self).__init__()
self.setdefault('Code', str(code))
self.setdefault('Diag', s)
class ThreadId(int):
# (0, 1024) is reserved
full_range = (0, 2**16)
ok_range = (1024, full_range[1])
error_value = 0
def __init__(self, i):
super(ThreadId, self).__init__(i)
if not (self.full_range[0] <= self < self.full_range[1]):
raise ValueError, "value outside of range"
def generate(self):
return apply(self, (apply(random.randrange, self.ok_range),))
generate = classmethod(generate)
def in_ok_range(self):
return (self >= self.ok_range[0] and self < self.ok_range[1])
class Address(tuple):
def __init__(self, *varargs, **kwargs):
self.validate()
def validate(self):
typecheck(self[0], str)
typecheck(self[1], int)
if len(self) != 2:
raise ValueError, "invalid address: %s" % str(self)
def __str__(self):
return (self[0] + ':' + str(self[1]))
def from_str(self, s):
fields = s.split(':')
fields[1] = int(fields[1])
return self(fields)
from_str = classmethod(from_str)
class Config(ConfigParser.ConfigParser, object):
def __init__(self, homedir):
assert isinstance(homedir, str)
self.homedir = homedir
super(Config, self).__init__()
def get_filename(self, section, option):
fn = os.path.expanduser(self.get(section, option))
if not os.path.isabs(fn):
fn = os.path.join(self.homedir, fn)
return fn
def get_homedir(specified):
homedir = os.path.join('/etc', 'pyzor')
if specified is not None:
homedir = specified
else:
userhome = os.getenv('HOME')
if userhome is not None:
homedir = os.path.join(userhome, '.pyzor')
return homedir
def typecheck(inst, type_):
if not isinstance(inst, type_):
raise TypeError
def modglobal_apply(globs, repl, obj, varargs=(), kwargs=None):
"""temporarily modify globals during a call.
globs is the globals to modify (e.g., the return from globals())
repl is a dictionary of name: value replacements for the global
dict."""
if kwargs is None:
kwargs = {}
saved = {}
for (k, v) in repl.items():
saved[k] = globs[k]
globs[k] = v
try:
r = apply(obj, varargs, kwargs)
finally:
globs.update(saved)
return r
anonymous_user = Username('anonymous')

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,626 @@
"""networked spam-signature detection server"""
from __future__ import division
import os
import sys
import SocketServer
import time
import gdbm
import cStringIO
import traceback
import threading
import pyzor
from pyzor import *
__author__ = pyzor.__author__
__version__ = pyzor.__version__
__revision__ = "$Id: server.py,v 1.29 2002-10-09 00:45:45 ftobin Exp $"
class AuthorizationError(pyzor.CommError):
"""signature was valid, but not permitted to
do the requested action"""
pass
class ACL(object):
__slots__ = ['entries']
default_allow = False
def __init__(self):
self.entries = []
def add_entry(self, entry):
typecheck(entry, ACLEntry)
self.entries.append(entry)
def allows(self, user, op):
typecheck(user, Username)
typecheck(op, Opname)
for entry in self.entries:
if entry.allows(user, op):
return True
if entry.denies(user, op):
return False
return self.default_allow
class ACLEntry(tuple):
all_keyword = 'all'.lower()
def __init__(self, v):
(user, op, allow) = v
typecheck(user, Username)
typecheck(op, Opname)
assert bool(allow) == allow
def user(self):
return self[0]
user = property(user)
def op(self):
return self[1]
op = property(op)
def allow(self):
return self[2]
allow = property(allow)
def allows(self, user, op):
return self._says(user, op, True)
def denies(self, user, op):
return self._says(user, op, False)
def _says(self, user, op, allow):
"""If allow is True, we return true if and only if we allow user to do op.
If allow is False, we return true if and only if we deny user to do op
"""
typecheck(user, Username)
typecheck(op, Opname)
assert bool(allow) == allow
return (self.allow == allow
and (self.user == user
or self.user.lower() == self.all_keyword)
and (self.op == op
or self.op.lower() == self.all_keyword))
class AccessFile(object):
# I started doing an iterator protocol for this, but it just
# got too complicated keeping track of everything on the line
__slots__ = ['file', 'output', 'lineno']
allow_keyword = 'allow'
deny_keyword = 'deny'
def __init__(self, f):
self.output = Output()
self.file = f
self.lineno = 0
def feed_into(self, acl):
typecheck(acl, ACL)
for orig_line in self.file:
self.lineno += 1
line = orig_line.strip()
if not line or line.startswith('#'):
continue
parts = line.split(':')
if len(parts) != 3:
self.output.warn("access file: invalid number of parts in line %d"
% self.lineno)
continue
(ops_str, users_str, allow_str) = parts
ops = []
for op_str in ops_str.split():
try:
op = Opname(op_str)
except ValueError, e:
self.output.warn("access file: invalid opname %s line %d: %s"
% (repr(op_str), self.lineno, e))
else:
ops.append(op)
users = []
for u in users_str.split():
try:
user = Username(u)
except ValueError, e:
self.output.warn("access file: invalid username %s line %d: %s"
% (repr(u), self.lineno, e))
else:
users.append(user)
allow_str = allow_str.strip()
if allow_str.lower() == self.allow_keyword:
allow = True
elif allow_str.lower() == self.deny_keyword:
allow = False
else:
self.output.warn("access file: invalid allow/deny keyword %s line %d"
% (repr(allow_str), self.lineno))
continue
for op in ops:
for user in users:
acl.add_entry(ACLEntry((user, op, allow)))
class Passwd(dict):
def __setitem__(self, k, v):
typecheck(k, pyzor.Username)
typecheck(v, long)
super(Passwd, self).__setitem__(k, v)
class PasswdFile(BasicIterator):
"""Iteration gives (Username, long) objects
Format of file is:
user : key
"""
__slots__ = ['file', 'output', 'lineno']
def __init__(self, f):
self.file = f
self.output = Output()
self.lineno = 0
def next(self):
while True:
orig_line = self.file.readline()
self.lineno += 1
if not orig_line:
raise StopIteration
line = orig_line.strip()
if not line or line.startswith('#'):
continue
fields = line.split(':')
fields = map(lambda x: x.strip(), fields)
if len(fields) != 2:
self.output.warn("passwd line %d is invalid (wrong number of parts)"
% self.lineno)
continue
try:
return (Username(fields[0]), long(fields[1], 16))
except ValueError, e:
self.output.warn("invalid passwd entry line %d: %s"
% (self.lineno, e))
class Log(object):
__slots__ = ['fp']
def __init__(self, fp=None):
self.fp = fp
def log(self, address, user=None, command=None, arg=None, code=None):
# we don't use defaults because we want to be able
# to pass in None
if user is None: user = ''
if command is None: command = ''
if arg is None: arg = ''
if code is None: code = -1
# We duplicate the time field merely so that
# humans can peruse through the entries without processing
ts = int(time.time())
if self.fp is not None:
self.fp.write("%s\n" %
','.join((("%d" % ts),
time.ctime(ts),
user,
address[0],
command,
repr(arg),
("%d" % code)
)))
self.fp.flush()
class Record(object):
"""Prefix conventions used in this class:
r = report (spam)
wl = whitelist
"""
__slots__ = ['r_count', 'r_entered', 'r_updated',
'wl_count', 'wl_entered', 'wl_updated',
]
fields = ('r_count', 'r_entered', 'r_updated',
'wl_count', 'wl_entered', 'wl_updated',
)
this_version = '1'
# epoch seconds
never = -1
def __init__(self, r_count=0, wl_count=0):
self.r_count = r_count
self.wl_count = wl_count
self.r_entered = self.never
self.r_updated = self.never
self.wl_entered = self.never
self.wl_updated = self.never
def wl_increment(self):
# overflow prevention
if self.wl_count < sys.maxint:
self.wl_count += 1
if self.wl_entered == self.never:
self.wl_entered = int(time.time())
self.wl_update()
def r_increment(self):
# overflow prevention
if self.r_count < sys.maxint:
self.r_count += 1
if self.r_entered == self.never:
self.r_entered = int(time.time())
self.r_update()
def r_update(self):
self.r_updated = int(time.time())
def wl_update(self):
self.wl_updated = int(time.time())
def __str__(self):
return "%s,%d,%d,%d,%d,%d,%d" \
% ((self.this_version,)
+ tuple(map(lambda x: getattr(self, x), self.fields)))
def from_str(self, s):
parts = s.split(',')
dispatch = None
version = parts[0]
if len(parts) == 3:
dispatch = self.from_str_0
elif version == '1':
dispatch = self.from_str_1
else:
raise StandardError, ("don't know how to handle db value %s"
% repr(s))
return apply(dispatch, (s,))
from_str = classmethod(from_str)
def from_str_0(self, s):
r = Record()
parts = s.split(',')
fields = ('r_count', 'r_entered', 'r_updated')
assert len(parts) == len(fields)
for i in range(len(parts)):
setattr(r, fields[i], int(parts[i]))
return r
from_str_0 = classmethod(from_str_0)
def from_str_1(self, s):
r = Record()
parts = s.split(',')[1:]
assert len(parts) == len(self.fields)
for i in range(len(parts)):
setattr(r, self.fields[i], int(parts[i]))
return r
from_str_1 = classmethod(from_str_1)
class DBHandle(Singleton):
__slots__ = ['output', 'initialized']
db_lock = threading.Lock()
max_age = 3600*24*30*4 # 3 months
db = None
sync_period = 60
reorganize_period = 3600*24 # 1 day
def __init__(self):
assert self.db is not None, "database was not initialized"
def initialize(self, fn, mode):
self.output = Output()
self.db = gdbm.open(fn, mode)
self.start_reorganizing()
self.start_syncing()
initialize = classmethod(initialize)
def apply_locking_method(self, method, varargs=(), kwargs={}):
# just so we don't carry around a mutable kwargs
if kwargs == {}:
kwargs = {}
self.output.debug("acquiring lock")
self.db_lock.acquire()
self.output.debug("acquired lock")
try:
result = apply(method, varargs, kwargs)
finally:
self.output.debug("releasing lock")
self.db_lock.release()
self.output.debug("released lock")
return result
apply_locking_method = classmethod(apply_locking_method)
def __getitem__(self, key):
return self.apply_locking_method(self._really_getitem, (key,))
def _really_getitem(self, key):
return self.db[key]
def __setitem__(self, key, value):
self.apply_locking_method(self._really_setitem, (key, value))
def _really_setitem(self, key, value):
self.db[key] = value
def start_syncing(self):
self.apply_locking_method(self._really_sync)
self.sync_timer = threading.Timer(self.sync_period,
self.start_syncing)
self.sync_timer.start()
start_syncing = classmethod(start_syncing)
def _really_sync(self):
self.db.sync()
_really_sync = classmethod(_really_sync)
def start_reorganizing(self):
self.apply_locking_method(self._really_reorganize)
self.reorganize_timer = threading.Timer(self.reorganize_period,
self.start_reorganizing)
self.reorganize_timer.start()
start_reorganizing = classmethod(start_reorganizing)
def _really_reorganize(self):
self.output.debug("reorganizing the database")
key = self.db.firstkey()
breakpoint = time.time() - self.max_age
while key is not None:
rec = Record.from_str(self.db[key])
delkey = None
if rec.r_updated < breakpoint:
self.output.debug("deleting key %s" % key)
delkey = key
key = self.db.nextkey(key)
if delkey:
del self.db[delkey]
self.db.reorganize()
_really_reorganize = classmethod(_really_reorganize)
class Server(SocketServer.ThreadingUDPServer, object):
max_packet_size = 8192
time_diff_allowance = 180
def __init__(self, address, log):
typecheck(log, Log)
self.output = Output()
RequestHandler.output = self.output
RequestHandler.log = log
self.output.debug('listening on %s' % str(address))
super(Server, self).__init__(address, RequestHandler)
def serve_forever(self):
self.pid = os.getpid()
super(Server, self).serve_forever()
def replace_log(self, newlog):
typecheck(newlog, Log)
RequestHandler.log = newlog
self.output.debug("changing logfile")
class RequestHandler(SocketServer.DatagramRequestHandler, object):
def setup(self):
super(RequestHandler, self).setup()
# This is to work around a bug in current versions
# of Python. The bug has been reported, and fixed
# in Python's CVS.
self.wfile = cStringIO.StringIO()
self.client_address = Address(self.client_address)
self.out_msg = Response()
self.user = None
self.op = None
self.op_arg = None
self.out_code = None
self.msg_thread = None
def handle(self):
try:
self._really_handle()
except UnsupportedVersionError, e:
self.handle_error(505, "Version Not Supported: %s" % e)
except NotImplementedError, e:
self.handle_error(501, "Not implemented: %s" % e)
except (ProtocolError, KeyError), e:
# We assume that KeyErrors are due to not
# finding a key in the RFC822 message
self.handle_error(400, "Bad request: %s" % e)
except AuthorizationError, e:
self.handle_error(401, "Unauthorized: %s" % e)
except SignatureError, e:
self.handle_error(401, "Unauthorized, Signature Error: %s" % e)
except Exception, e:
self.handle_error(500, "Internal Server Error: %s" % e)
traceback.print_exc()
self.out_msg.setdefault('Code', str(self.out_msg.ok_code))
self.out_msg.setdefault('Diag', 'OK')
self.out_msg.init_for_sending()
self.log.log(self.client_address, self.user, self.op, self.op_arg,
int(self.out_msg['Code']))
msg_str = str(self.out_msg)
self.output.debug("sending: %s" % repr(msg_str))
self.wfile.write(msg_str)
def _really_handle(self):
"""handle() without the exception handling"""
self.output.debug("received: %s" % repr(self.packet))
signed_msg = MacEnvelope(self.rfile)
self.user = Username(signed_msg['User'])
if self.user != pyzor.anonymous_user:
if self.server.passwd.has_key(self.user):
signed_msg.verify_sig(self.server.passwd[self.user])
else:
raise SignatureError, "unknown user"
self.in_msg = signed_msg.get_submsg(pyzor.Request)
self.msg_thread = self.in_msg.get_thread()
# We take the int() of the proto versions because
# if the int()'s are the same, then they should be compatible
if int(self.in_msg.get_protocol_version()) != int(proto_version):
raise UnsupportedVersionError
self.out_msg.set_thread(self.msg_thread)
self.op = Opname(self.in_msg.get_op())
if not self.server.acl.allows(self.user, self.op):
raise AuthorizationError, "user is unauthorized to request the operation"
self.output.debug("got a %s command from %s" %
(self.op, self.client_address))
if not self.dispatches.has_key(self.op):
raise NotImplementedError, "requested operation is not implemented"
dispatch = self.dispatches[self.op]
if dispatch is not None:
apply(dispatch, (self,))
def handle_error(self, code, s):
self.out_msg = ErrorResponse(code, s)
if self.msg_thread is None:
self.out_msg.set_thread(ThreadId(0))
else:
self.out_msg.set_thread(self.msg_thread)
def handle_check(self):
digest = self.in_msg['Op-Digest']
self.op_arg = digest
self.output.debug("request to check digest %s" % digest)
db = DBHandle()
try:
rec = Record.from_str(db[digest])
r_count = rec.r_count
wl_count = rec.wl_count
except KeyError:
r_count = 0
wl_count = 0
self.out_msg['Count'] = "%d" % r_count
self.out_msg['WL-Count'] = "%d" % wl_count
def handle_report(self):
digest = self.in_msg['Op-Digest']
self.op_arg = digest
self.output.debug("request to report digest %s" % digest)
db = DBHandle()
try:
rec = Record.from_str(db[digest])
except KeyError:
rec = Record()
rec.r_increment()
db[digest] = str(rec)
def handle_whitelist(self):
digest = self.in_msg['Op-Digest']
self.op_arg = digest
self.output.debug("request to whitelist digest %s" % digest)
db = DBHandle()
try:
rec = Record.from_str(db[digest])
except KeyError:
rec = Record()
rec.wl_increment()
db[digest] = str(rec)
def handle_info(self):
digest = self.in_msg['Op-Digest']
self.op_arg = digest
self.output.debug("request to check digest %s" % digest)
db = DBHandle()
try:
record = Record.from_str(db[digest])
except KeyError:
record = Record()
r_count = record.r_count
wl_count = record.wl_count
self.out_msg['Entered'] = "%d" % record.r_entered
self.out_msg['Updated'] = "%d" % record.r_updated
self.out_msg['WL-Entered'] = "%d" % record.wl_entered
self.out_msg['WL-Updated'] = "%d" % record.wl_updated
self.out_msg['Count'] = "%d" % r_count
self.out_msg['WL-Count'] = "%d" % wl_count
dispatches = { 'check': handle_check,
'report': handle_report,
'ping': None,
'info': handle_info,
'whitelist': handle_whitelist,
}

154
mail/spamassassin/local.cf Normal file
View File

@@ -0,0 +1,154 @@
# mysql configuration settings
dns_available yes
skip_rbl_checks 0
# #rewrite_header Subject [SPAM]
rewrite_header Subject ***[SPAM](_SCORE_)***
# system report
report_safe 0
# internal & trusted networks
internal_networks 192.168.1.0/25
trusted_networks 192.168.1.0/25 89.121.131.74/32
# set file-locking method (flock is not safe over NFS, but is faster)
lock_method flock
# default required score
required_hits 5.0
required_score 5
#DCC
use_dcc 1
dcc_home /var/dcc
dcc_path /usr/local/bin/dccproc
dcc_timeout 10
score DCC_CHECK 4
add_header all DCC _DCCB_: _DCCR_
# bayesian system
use_bayes 1
use_bayes_rules 1
bayes_auto_learn 1
bayes_auto_learn_threshold_nonspam 0.1
bayes_auto_learn_threshold_spam 8.9
bayes_learn_during_report 1
use_auto_whitelist 1
bayes_use_hapaxes 1
bayes_expiry_max_db_size 400000
bayes_auto_expire 1
#lock_method flock
bayes_min_ham_num 50
bayes_min_spam_num 50
bayes_path /var/spool/amavisd/.spamassassin/bayes
# razor
use_razor2 1
razor_config /var/spool/amavisd/.razor/razor-agent.conf
# pyzor
use_pyzor 1
pyzor_path /usr/bin/pyzor
pyzor_options --homedir /etc/mail/spamassassin
#other
bayes_ignore_header X-Spam-Flag
bayes_ignore_header X-Spam-Status
bayes_ignore_header X-Spam-Report
score RAZOR2_CHECK 2.500
score PYZOR_CHECK 2.500
score DCC_CHECK 4.000
score ALL_TRUSTED -10.000
score URIBL_DBL_SPAM 3.4
spf_timeout 5
#locales
ok_locales all
score DEAR_SOMETHING 0.5
score SUBJ_ALL_CAPS 0.5
score UPPERCASE_75_100 1
score MIME_BASE64_TEXT 1
score URIBL_BLACK 5
score URIBL_SBL 3
score FORGED_YAHOO_RCVD 4
score SARE_RECV_IP_FROMIP3 2
score RDNS_NONE 0.5
score BAYES_99 3.5
score SARE_SUB_GOOD_DAY 1.2
score SARE_RECV_VIRTUACOMBR 1.6
score SARE_GIF_ATTACH 0.1
header LOCAL_RCVD Received =~ /\S+\.section6.net\s+\(.*\[.*\]\)/
score LOCAL_RCVD -50
# ## Optional Score Increases
score DCC_CHECK 4.000
score RAZOR2_CHECK 2.500
score PYZOR_CHECK 2.500
score BAYES_99 5.300
score BAYES_90 4.500
score BAYES_80 4.000
# # For scores have a look at /usr/local/share/spamassassin/50_scores.cf
# # file.
score HTML_FONT_INVISIBLE 3
score HTML_FONTCOLOR_UNKNOWN 2
score ORDER_NOW 1.5
score CLICK_BELOW 1
score LIMITED_TIME_ONLY 1
# # This rule might be extreme but html only spams get through too easy.
# # In other words, if you can't take the time to write something and are
# # posting an image only, then you're 86'd!
score HTML_IMAGE_ONLY_02 2
score HTML_IMAGE_ONLY_04 2
score OFFERS_ETC 2
score HTML_LINK_CLICK_HERE 1
score LINES_OF_YELLING 1
### Shortcircuit plugin ###
ifplugin Mail::SpamAssassin::Plugin::Shortcircuit
shortcircuit USER_IN_WHITELIST on
shortcircuit USER_IN_DEF_WHITELIST on
shortcircuit USER_IN_ALL_SPAM_TO on
shortcircuit SUBJECT_IN_WHITELIST on
shortcircuit USER_IN_BLACKLIST on
shortcircuit USER_IN_BLACKLIST_TO on
shortcircuit SUBJECT_IN_BLACKLIST on
shortcircuit ALL_TRUSTED on
shortcircuit BAYES_99 spam
shortcircuit BAYES_00 ham
endif
#### Shortcircuit plugin ###
ifplugin Mail::SpamAssassin::Plugin::TxRep
header TXREP eval:check_senders_reputation()
describe TXREP Score normalizing based on sender's reputation
tflags TXREP userconf noautolearn
priority TXREP 1000
endif
#MTX Blacklist
include /etc/spamassassin/mtx.cf
### MANUAL WHITELIST ###
whitelist_from *@escorte.pro
whitelist_to *@escorte.pro
whitelist_from *@vrem.ro
whitelist_to *@vrem.ro
whitelist_from *@newsletter.emag.ro
whitelist_from *@emag.ro
whitelist_from *@librabank.ro
whitelist_from *@simpatie.ro
### MANUAL BLACKLIST ###
blacklist_from comenzi@besttoner.ro
blacklist_from info@timisoaraazi.ro
blacklist_from hello@emailvision.ro
blacklist_from info@targetmail.ro
blacklist_from compact.sv@gmail.com
#MANUAL WHITELIST+BLACKLIST
include /etc/spamassassin/manual.cf

View File

@@ -0,0 +1,47 @@
### MANUAL WHITELIST ###
whitelist_from root@zira.898.ro
whitelist_from *@vrem.ro
whitelist_from *@paypal.com
whitelist_from *@intl.paypal.com
whitelist_from *@videosecrets.com
whitelist_from *@pagoplateste.ro
whitelist_from *@amazon.com
whitelist_from *@email.fitbit.com
whitelist_from *@*.pagoplateste.ro
whitelist_from *@itratos.org
### MANUAL BLACKLIST ###
blacklist_from comenzi@besttoner.ro
blacklist_from info@timisoaraazi.ro
blacklist_from info@aaloans.org
blacklist_from news@targetmail.ro
blacklist_from commercial@euro-contact.org
blacklist_from email@vineyani.ro
blacklist_from oferta@besttoner.ro
blacklist_from terri_loszynski@acmmt.com
blacklist_from no-reply@uggoriginal10.com
blacklist_from sales17@sykpcb.com
blacklist_from sales08@sykpcb.com
blacklist_from info@online-trading-academy.com
blacklist_from jurassic@jamiecatto.com
blacklist_from email@dezvoltambreaza.ro
blacklist_from contact@globaltreat.ro
blacklist_from *@e.email.zolucky.com
blacklist_from *@zolucky.com
#BODY BLACKLIST
body SPAM_1 /Draga prieten!/i
score SPAM_1 12
body SPAM_2 /Folosim anuare libere ale posturilor de munca din Uniunea Europeana; daca acest mesaj nu va intereseaza, va rugam ignorati-l/i
score SPAM_2 12
body SPAM_3 /Va rugam trimiteti-ne prin e-Cmail scrisoarea/i
score SPAM_3 12
body SPAM_SUGANORM /\bsuganorm\b/i # SugaNorm
score SPAM_SUGANORM 12
body SPAM_STARTRANSA /\bstartransa[\x{20}\x{20}][\x{2D}\x{2D}][\x{20}\x{20}]transaction\b/i # StarTransa - Transaction
score SPAM_STARTRANSA 12

30
mail/spamassassin/mtx.cf Normal file
View File

@@ -0,0 +1,30 @@
loadplugin Mail::SpamAssassin::Plugin::MTX MTX.pm
header MTX_PASS eval:check_mtx_pass()
header MTX_FAIL eval:check_mtx_fail()
header MTX_NONE eval:check_mtx_none()
header MTX_NEUTRAL eval:check_mtx_neutral()
header MTX_SOFTFAIL eval:check_mtx_softfail()
header MTX_HARDFAIL eval:check_mtx_hardfail()
header MTX_BLACKLIST eval:check_mtx_blacklist()
score MTX_PASS -2 # Bonus for Pass.
score MTX_FAIL 0.001 # Using NONE/NEUTRAL/SOFTFAIL/HARDFAIL instead.
score MTX_NONE 0.001 # No penalty for not using MTX, until it's
# more widely used.
score MTX_NEUTRAL 0.001 # Same lack of penalty for people using MTX prefering
# minimum penalty for IPs without an MTX record.
score MTX_SOFTFAIL 1 # More penalty for those who want it.
score MTX_HARDFAIL 100 # Major penalty for those who want it.
# MTX_BLACKLIST score defined per domain.
describe MTX_PASS MTX: Passed: http://www.chaosreigns.com/mtx/
describe MTX_FAIL MTX: Failed: http://www.chaosreigns.com/mtx/
describe MTX_NONE MTX: Not defined: http://www.chaosreigns.com/mtx/
describe MTX_NEUTRAL MTX: Neutral: http://www.chaosreigns.com/mtx/
describe MTX_SOFTFAIL MTX: SoftFail: http://www.chaosreigns.com/mtx/
describe MTX_HARDFAIL MTX: HardFail: http://www.chaosreigns.com/mtx/
describe MTX_BLACKLIST MTX: On your blacklist.
# MTX Blacklist file
include /etc/mail/spamassassin/mtx_blacklist.cf

View File

Binary file not shown.

View File

@@ -0,0 +1,46 @@
#!/usr/bin/perl
# (c) Darxus@ChaosReigns.com, released under the GPL.
# Update MTX blacklist.
# http://www.chaosreigns.com/mtx/
#
# 2010-02-14 Initial release.
use warnings;
use strict;
use LWP::Simple;
use IO::Uncompress::Gunzip qw(gunzip $GunzipError) ;
my %score;
$score{'somespam'} = 4;
$score{'allspam'} = $score{'somespam'} + 100;
my $gzfile = "/etc/mail/spamassassin/mtx_blacklist.gz";
my $safile = "/etc/mail/spamassassin/mtx_blacklist.cf";
my $url = "http://www.mtxbl.chaosreigns.com/mtxbl.gz";
my %typename = (
1, 'somespam',
2, 'allspam',
);
$url = $ARGV[0] if ( defined $ARGV[0] );
my $rc = mirror($url, $gzfile);
if (is_error($rc)) {
die("Download failed.");
}
unless ($rc == RC_NOT_MODIFIED) {
my $z = new IO::Uncompress::Gunzip $gzfile or die "gunzip failed: $GunzipError";
open OUT, ">$safile" or die "Couldn't write to $safile: $!";
while (my $line = $z->getline()) {
chomp $line;
if ($line =~ /(1|2) (\S+)\s*(?:$|#)/) {
my $type = $1;
my $domain = $2;
print OUT "mtx_blacklist *.$domain $score{$typename{$type}}\n";
}
}
close OUT;
}
exit; ###############################################

18
mail/spamassassin/pub.gpg Normal file
View File

@@ -0,0 +1,18 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.1 (GNU/Linux)
mQGiBETpq0cRBAD4H/0KXKzr7rrTzOJGpLYm/qNW6GBC+Jqzn6fYSyE5XKFZpaY6
SzIL51Bcbhri7zzagMVWo7qNSGbhY6IG+drr99vURgDfxW4PC5DtqPsz/mU7RYIU
noaLL/p2XgceM6P/xxe/Gm/3CLgHvPvZkGw5y7scnNJLgYRxncX8eF5B3wCggB9O
DKfecaYGz/8pp/3P+/NnS3UD+wdRbxk8pA/vN3OnhLL1+7LK6upI0qJ+VHBeiK+v
dVJ0DJ7VqMYKVCpvy39g1Hw1H45r6evZ8F5B7sRRJO27MU8zq2yX+WOo9sWU0FUL
SLJDb3RL957Q1aXz9t1NTVmjpbRNikGQ8RYSAFIqhRzasZGyHzQVM0Ylt4EIWcSk
5BPKBACoROiRhXNmE4cTPNuAyMKigHYhncr7MofH8QkPqrySyooBo6CnyZuiIlhH
JrjTM2Wov8P6HiLBijquoZO//3eItevdy4iuth0M6q/r5gHU8jZ3YaF/wS6tdRwM
Kk8T7rMQvQ2/YmlfnZpJGz0UOuOQigdOwuF8qOiEdRsy04/zTbRfT3BlbmNvbXB1
dGluZyBUZWNobm9sb2dpZXMgKEtleSB0byBzaWduIGFsbCBmaWxlcyBmcm9tIG9w
ZW5wcm90ZWN0LmNvbSkgPGVtYWlsQG9wZW5wcm90ZWN0LmNvbT6IXgQTEQIAHgUC
ROmrRwIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRAljNs6vencED6OAJwJxSxB
E17IYlu6NUDH+cv9jXxFcACfSH6QWe0MBuOpEYmJwfcNpVFguvU=
=vRh7
-----END PGP PUBLIC KEY BLOCK-----

Binary file not shown.

View File

@@ -0,0 +1,3 @@
ChangeLog
build
dist

View File

@@ -0,0 +1,340 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.
59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Library General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) 19yy <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) 19yy name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Library General
Public License instead of this License.

View File

@@ -0,0 +1,45 @@
Pyzor requires at least Python 2.6
To install this distribution, simply run the following:
python setup.py build
python setup.py install
Note that your system might install the modules and scripts
with non-world-readable permissions.
Correct this with a command such as:
chmod -R a+rX /usr/share/doc/pyzor \
/usr/lib/python2.2/site-packages/pyzor \
/usr/bin/pyzor /usr/bin/pyzord
To use the server, the Python gdbm module or MySQLdb module is required.
You can generally check if you have the gdbm module by executing:
python -c 'import gdbm' && echo 'gdbm found'
You can generally check if you have the MySQLdb module by executing:
python -c 'import MySQLdb' && echo 'MySQLdb found'
The gdbm module is available at:
Debian GNU/Linux:
http://packages.debian.org/stable/interpreters/python-gdbm.html
Gentoo Linux:
Will be built with Python if the gdbm library is found.
If it isn't there with your Python, try stealing
the FreeBSD setup.py patchfile in their ports to
install just the gdbm module, or simply re-install
Python.
FreeBSD:
ports/databases/py-gdbm
tar.gz:
included in the Python distribution
(not sure of the precise procedure for simply installing
the gdbm module; try stealing the FreeBSD setup.py patchfile
in their ports).
Pyzor also works with Python3.3. The code will be automatically refactored with
2to3 during the setup:
python3.3 setup.py install
Note that the MySQLdb library does not currently support Python3.
See docs/usage.html for usage documentation, and if you are upgrading
from another version of Pyzor, please read the UPGRADING file.

View File

@@ -0,0 +1,13 @@
include COPYING
include INSTALL
include NEWS
include README.txt
include THANKS
include UPGRADING
include requirements.txt
include config/*
include pyzor/*
include pyzor/engines/*
include pyzor/hacks/*
include scripts/*

View File

@@ -0,0 +1,250 @@
Noteworthy changes in 0.7.0
-----------------------------------------------------------------
Bug fixes:
* Fix decoding bug when messages are badly formed
* Pyzor now correctly creates the specified homedir, not the user's one
New features:
* Logging is now disabled by default
* Automatically run 2to3 during installation (if required)
New pyzord features:
* Added ability to disable expiry
* New redis engine support has been added
* New option to enable gevent
* Added the ability to reload accounts and access files using USR1 signal
* Added the ability to safely stop the daemon with TERM signal
* Split the usage-log and normal log in two separate files
* Pyzord daemon can now daemonize and detach itself
Noteworthy changes in 0.6.0
-----------------------------------------------------------------
* pyzor and pyzord will now work with Python3.3 (if
the the 2to3-3.3 is previously ran)
* pyzord and pyzor now supports IPv6
* Improved handling of multi-threading (signals where
again removed) for the mysql engine
* Introduced multi-processing capabilities
* Improved HTML parsing
* Introduced self document sample configurations
* Introduced ability to set custom report/whitelist thresholds
for the pyzor client
* Greatly improved tests coverage
Noteworthy changes in 0.5.0
-----------------------------------------------------------------
Note that the majority of changes in this release were contributed back
from the Debian pyzor package.
* Man pages for pyzor and pyzord.
* Changing back to signals for database locking,
rather than threads. It is likely that signals
will be removed again in the future, but the
existing threading changes caused problems.
* Basic checks on the results of "discover".
* Extended mbox support throughout the library.
* Better handling on unknown encodings.
* Added a --log option to log to a file.
* Better handling of command-line options.
* Improved error handling.
Noteworthy changes in 0.4.x
-----------------------------------------------------------------
* pyzor client now more gracefully handles base64 and
multipart decoding errors, so that it can be used
over an mbox.
* pyzor client has new config file option in the [client]
section, Timeout, which specifies a timeout in seconds
for queries to come back to the client.
* pyzord no longer daemonizes itself, and now writes
it logging to standard output.
* The following server config options no longer have effect:
PidFile, LogFile.
* Upped the allowed signed timestamp difference to be up to
5 minutes (up from 3 minutes).
* Removed the 'shutdown' command; implementation of
'meta' commands need to be re-thought.
* Rewrite of threads locking to access the database.
* pyzord no longer handles USR1 signals; instead, it now
automatically reorganizes and cleans-up the database daily.
* Client code now uses threading to catch timeouts,
rather than an alarm signal.
Noteworthy changes in 0.4.0
-----------------------------------------------------------------
* Messages are now decoded if they are encoded,
and subparts that are not encoded text/* are ignored.
Currently base64, quoted-printable, and uuencode
is supported.
* Message normalization now removes HTML tags
(irregardless of Content-Type).
* Message lines with less than 8 chars after normalization
are now not included in digests.
* Messages having less than or equal to 4 lines are entirely
digested.
* Implemented 'digest' command, which simply prints
out the digest(s) of the messages encountered.
* Implemented 'predigest' command which prints out
the data that is actually digested in a message.
* If HOME is unspecified and no --homedir is given,
The the config directory is /etc/pyzor
* If the pyzord process receives a HUP signal, it re-opens
the logfile.
Noteworthy changes in 0.3.1
-----------------------------------------------------------------
* Fixed bug where if pyzor would send reports or
whitelists to each server N times, where N
is the number of servers.
* Server now keeps database file open, instead of re-opening
it on each request.
* pyzord.log now includes response code.
Noteworthy changes in 0.3.0
-----------------------------------------------------------------
* Pyzor now requires Python 2.2.1.
* The protocol is not backwards compatible, so please
remove old ~/.pyzor/servers files, and they will
be refreshened to point to new servers.
* The pyzor system now has accounts, access controls on
users. anonymous users by default can do
['check', 'report', 'ping', 'info'].
For more information on this, please refer to the
documentation.
* Documentation has moved from the source files
(e.g., 'pydoc pyzor') into a separate XHTML document,
located in docs/usage.html, and normally installed
into a location such as /usr/share/doc/pyzor
* Messages are authenticated using digest-signing, similar
to HTTP-digest authentication. This is a
is a shared-secret scheme, but the secret is very
hard to recover from what is passed in the signature.
* Whitelisting messages is now possible.
* An 'info' command is no implemented
This returns extra info about any digest, such as
when it was first entered and last updated.
* a 'genkey' command has been implemented for the client;
this is used to create a (salt, key) string
used for authentication.
* a 'shutdown' commmand has been implemented, which can
be used to shutdown a server.
* Expiring of digests using a USR1 signal has been removed
for now. In the future a client/server message
will be likely be implemented for this functionality.
* pyzrod logfile now contains a human-readable timestamp
field in addition to the epoch-seconds field.
Noteworthy changes in 0.2.1
-----------------------------------------------------------------
* Fixed major bug where the incorrect exit
code is given.
Noteworthy changes in 0.2.0
-----------------------------------------------------------------
* Protocol break. Old clients will not work
with new servers, and vice versa.
* ~/.pyzor is now a directory, with ~/.pyzor/config
containing configuration directives.
~/.pyzor/servers contains a list of servers.
* pyzord's command-line interface has changed,
now being primarily configured in ~/.pyzor/config.
* pyzord now does logging (~/.pyzor/pyzord.log) and has
a pidfile (~/.pyzor/pyzord.pid).
* Debugging for client and server improved.
* Client now contacts each server listed
when doing a check/report/ping.
* Can now be used with ReadyExec,
http://readyexec.sourceforge.net/
Documentation on how to use ReadyExec is
in the pyzor documentation.
Noteworthy changes in 0.1.1
-----------------------------------------------------------------
* Fixed problem when trying to report messages
in non-unix mailbox format.
* Added --mbox option for 'pyzor report' for when
reporting entire mailboxes.
* No changes were made in the server portion.
Noteworthy changes in 0.1.0
-----------------------------------------------------------------
* Initial release.

View File

@@ -0,0 +1,7 @@
Pyzor is a Python implementation of a spam-blocking
networked system that use spam signatures to identify them.
http://pyzor.org/
See INSTALL for install documentation.
See https://sourceforge.net/apps/trac/pyzor/wiki/0.7/usage for usage documentation.

View File

@@ -0,0 +1,14 @@
Pyzor was originally written by Frank Tobin. Other people
contributed by reporting problems, suggesting various improvements or
submitting actual code. Here is a list of those people. Help me keep
it complete and free of errors.
Frank Tobin ftobin@neverending.org
Rick Macdougall rickm@nougen.com
Colin Smith colin@archeus.plus.com
Bobby Rose brose@med.wayne.edu
Roman Suzi rnd@onego.ru
Robert Schiele schiele@users.sourceforge.net
Tobias Klauser tux_edo@users.sourceforge.net
Tony Meyer tony.meyer@gmail.com
Alexandru Chirila chirila.s.alexandru@gmail.com

View File

@@ -0,0 +1,5 @@
If you are upgrading to Pyzor 0.3.x or newer from an older version,
please remove your old ~/.pyzor/servers file, as the protocol
is not backwards compatible. The Pyzor client will refreshen
the servers file to point to the the public server handling
the new protocol.

View File

@@ -0,0 +1,4 @@
## This file should contain a list of `host : port : username : salt,key`
## each on a new line. The salt and key can be generated with genkey command
## in the pyzor client. Example:
# 127.0.0.1 : 24441 : alice : d28f86151e80a9accba4a4eba81c460532384cd6,fc7f1cad729b5f3862b2ef192e2d9e0d0d4bd515

View File

@@ -0,0 +1,96 @@
## Note that the options that require a file name, must not contain absolute
## paths. They are relative to the specified --homedir, which defaults to
## ~/.pyzor
## All of these options are overridable from the respective command-line
## arguments.
## The client section only affects the pyzor client.
[client]
## The `ServersFile` must contain a newline-separated list of server
## addresses to report/whitelist/check with.
# ServersFile = servers
## The `AccountsFile` file containing information about accounts on servers.
# AccountsFile = accounts
## This option specifies the name of the log file.
# LogFile =
## This options specifies the number of seconds that the pyzor client should
## wait for a response from the server before timing out.
# Timeout = 5
## This options specifies the input style of the pyzor client. Current options
## are:
## - msg (individual RFC5321 message)
## - mbox (mbox file of messages)
## - digests (Pyzor digests, one per line)
# Style = msg
## Thes options specify the threshold for number of reports/whitelists.
## According to these thresholds the pyzor client exit code will differ.
# ReportThreshold = 0
# WhitelistThreshold = 0
## The server section only affects the pyzord server.
[server]
## Specifes the port and interface to listen on.
# Port = 24441
# ListenAddress = 0.0.0.0
## This option specifies the name of the log file.
# LogFile =
## This option specifies the name of the usage log file.
# UsageLogFile =
## This file will contain the PID of the pyzord daemon, when the it's
## started with the --detach options. The file is removed when the daemon is
## closed
# PidFile = pyzord.pid
## This file must contain the username and their keys
# PasswdFile = pyzord.passwd
## This file defines the ACL for the users
# AccessFile = pyzord.access
## These settings define the storage engine that the pyzord server should use.
## Example for gdbm (default):
# Engine = gdbm
# DigestDB = pyzord.db
## Example for mysql:
# Engine = mysql
# DigestDB = localhost,user,passwd,pyzor_db,pyzor_table
## Example for redis:
# Engine = redis
# DigestDB = localhost,6379,,0
## Or if a password is required
# DigestDB = localhost,6379,passwd,0
## The maximum age of an record, after which it will be removed.
## To disable this set this to 0.
# CleanupAge = 10368000 # aprox 4 months
## These setting define how and if the pyzord server should use concurrency
## For multi-threading:
# Threads = False
# MaxThreads = 0 # unlimited
# DBConnections = 0 # new connection for each request
## For multi-processing:
# Processes = False
# MaxProcesses = 40

View File

@@ -0,0 +1,5 @@
## This defines the ACL for each user, by default if a user is not specified
## here he is denied all access ( this includes anonymous users). Examples:
# check report ping pong info whitelist : alice : allow
# ALL : anonymous : allow
# whitelist : anonymous : deny

View File

@@ -0,0 +1,4 @@
## This file must contain the username and their keys, so that the recieving
## server can verify the user's signature. Example
# alice : fc7f1cad729b5f3862b2ef192e2d9e0d0d4bd515
# bob : cf88277c5d4abdc0a3f56f416011966d04a3f462

View File

@@ -0,0 +1,3 @@
## This file should contain a list of pyzor servers to which to direct the
## requests. Each address:port on a new line.
public.pyzor.org:24441

View File

@@ -0,0 +1 @@
*.pyc

View File

@@ -0,0 +1,60 @@
"""Networked spam-signature detection."""
__author__ = "Frank J. Tobin, ftobin@neverending.org"
__credits__ = "Tony Meyer, Dreas von Donselaar, all the Pyzor contributors."
__version__ = "0.7.0"
import hashlib
proto_name = 'pyzor'
proto_version = 2.1
anonymous_user = 'anonymous'
# We would like to use sha512, but that would mean that all the digests
# changed, so for now, we stick with sha1 (which is the same as the old
# sha module).
sha = hashlib.sha1
# This is the maximum time between a client signing a Pyzor request and the
# server checking the signature.
MAX_TIMESTAMP_DIFFERENCE = 300 # seconds
class CommError(Exception):
"""Something in general went wrong with the transaction."""
pass
class ProtocolError(CommError):
"""Something is wrong with talking the protocol."""
pass
class TimeoutError(CommError):
"""The connection timed out."""
pass
class IncompleteMessageError(ProtocolError):
"""A complete requested was not received."""
pass
class UnsupportedVersionError(ProtocolError):
"""Client is using an unsupported protocol version."""
pass
class SignatureError(CommError):
"""Unknown user, signature on msg invalid, or not within allowed time
range."""
pass
class AuthorizationError(CommError):
"""The signature was valid, but the user is not permitted to do the
requested action."""
pass

View File

@@ -0,0 +1,75 @@
"""A collection of utilities that facilitate working with Pyzor accounts.
Note that accounts are not necessary (on the client or server), as an
"anonymous" account always exists."""
import time
import hashlib
import pyzor
def sign_msg(hashed_key, timestamp, msg, hash_=hashlib.sha1):
"""Converts the key, timestamp (epoch seconds), and msg into a digest.
lower(H(H(M) + ':' T + ':' + K))
M is message
T is integer epoch timestamp
K is hashed_key
H is the hash function (currently SHA1)
"""
M = msg.as_string().strip().encode("utf8")
digest = hash_()
digest.update(hash_(M).digest())
digest.update((":%d:%s" % (timestamp, hashed_key)).encode("utf8"))
return digest.hexdigest().lower()
def hash_key(key, user, hash_=hashlib.sha1):
"""Returns the hash key for this username and password.
lower(H(U + ':' + lower(K)))
K is key (hex string)
U is username
H is the hash function (currently SHA1)
"""
S = ("%s:%s" % (user, key.lower())).encode("utf8")
return hash_(S).hexdigest().lower()
def verify_signature(msg, user_key):
"""Verify that the provided message is correctly signed.
The message must have "User", "Time", and "Sig" headers.
If the signature is valid, then the function returns normally.
If the signature is not valid, then a pyzor.SignatureError() exception
is raised."""
timestamp = int(msg["Time"])
user = msg["User"]
provided_signature = msg["Sig"]
# Check that this signature is not too old.
if abs(time.time() - timestamp) > pyzor.MAX_TIMESTAMP_DIFFERENCE:
raise pyzor.SignatureError("Timestamp not within allowed range.")
# Calculate what the correct signature is.
hashed_user_key = hash_key(user_key, user)
# The signature is not part of the message that is signed.
del msg["Sig"]
correct_signature = sign_msg(hashed_user_key, timestamp, msg)
if correct_signature != provided_signature:
raise pyzor.SignatureError("Invalid signature.")
class Account(object):
def __init__(self, username, salt, key):
self.username = username
self.salt = salt
self.key = key
def key_from_hexstr(s):
try:
salt, key = s.split(",")
except ValueError:
raise ValueError("Invalid number of parts for key; perhaps you "
"forgot the comma at the beginning for the "
"salt divider?")
return salt, key
AnonymousAccount = Account(pyzor.anonymous_user, None, "")

View File

@@ -0,0 +1,255 @@
"""Networked spam-signature detection client.
>>> import pyzor
>>> import pyzor.client
>>> import pyzor.digest
>>> import pyzor.config
To load the accounts file:
>>> accounts = pyzor.config.load_accounts(filename)
To create a client (to then issue commands):
>>> client = pyzor.client.Client(accounts)
To create a client, using the anonymous user:
>>> client = pyzor.client.Client()
To get a digest (of an email.message.Message object, or similar):
>>> digest = pyzor.digest.get_digest(msg)
To query a server (where address is a (host, port) pair):
>>> client.ping(address)
>>> client.info(digest, address)
>>> client.report(digest, address)
>>> client.whitelist(digest, address)
>>> client.check(digest, address)
To query the default server (public.pyzor.org):
>>> client.ping()
>>> client.info(digest)
>>> client.report(digest)
>>> client.whitelist(digest)
>>> client.check(digest)
Response will contain, depending on the type of request, some
of the following keys (e.g. client.ping()['Code']):
All responses will have:
- 'Diag' 'OK' or error message
- 'Code' '200' if OK
- 'PV' Protocol Version
- 'Thread'
`info` and `check` responses will also contain:
- '[WL-]Count' Whitelist/Blacklist count
`info` responses will also have:
- '[WL-]Entered' timestamp when message was first whitelisted/blacklisted
- '[WL-]Updated' timestamp when message was last whitelisted/blacklisted
"""
import sys
import time
import email
import socket
import logging
import pyzor.digest
import pyzor.account
import pyzor.message
import pyzor.hacks.py26
pyzor.hacks.py26.hack_email()
class Client(object):
timeout = 5
max_packet_size = 8192
def __init__(self, accounts=None, timeout=None):
if accounts:
self.accounts = accounts
else:
self.accounts = {}
if timeout is not None:
self.timeout = timeout
self.log = logging.getLogger("pyzor")
def ping(self, address=("public.pyzor.org", 24441)):
msg = pyzor.message.PingRequest()
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def pong(self, digest, address=("public.pyzor.org", 24441)):
msg = pyzor.message.PongRequest(digest)
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def info(self, digest, address=("public.pyzor.org", 24441)):
msg = pyzor.message.InfoRequest(digest)
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def report(self, digest, address=("public.pyzor.org", 24441),
spec=pyzor.digest.digest_spec):
msg = pyzor.message.ReportRequest(digest, spec)
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def whitelist(self, digest, address=("public.pyzor.org", 24441),
spec=pyzor.digest.digest_spec):
msg = pyzor.message.WhitelistRequest(digest, spec)
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def check(self, digest, address=("public.pyzor.org", 24441)):
msg = pyzor.message.CheckRequest(digest)
sock = self.send(msg, address)
return self.read_response(sock, msg.get_thread())
def send(self, msg, address=("public.pyzor.org", 24441)):
msg.init_for_sending()
try:
account = self.accounts[address]
except KeyError:
account = pyzor.account.AnonymousAccount
timestamp = int(time.time())
msg["User"] = account.username
msg["Time"] = str(timestamp)
msg["Sig"] = pyzor.account.sign_msg(pyzor.account.hash_key(
account.key, account.username), timestamp, msg)
self.log.debug("sending: %r", msg.as_string())
return self._send(msg, address)
def _send(self, msg, addr):
sock = None
for res in socket.getaddrinfo(addr[0], addr[1], 0, socket.SOCK_DGRAM,
socket.IPPROTO_UDP):
af, socktype, proto, _, sa = res
try:
sock = socket.socket(af, socktype, proto)
except socket.error:
sock = None
continue
try:
sock.sendto(msg.as_string().encode("utf8"), 0, sa)
except socket.timeout:
sock.close()
raise pyzor.TimeoutError("Sending to %s time-outed" % sa)
except socket.error:
sock.close()
sock = None
continue
break
if sock is None:
raise pyzor.CommError("Unable to send to %s" % addr)
return sock
def read_response(self, sock, expected_id):
sock.settimeout(self.timeout)
try:
packet, address = sock.recvfrom(self.max_packet_size)
except socket.timeout as e:
sock.close()
raise pyzor.TimeoutError("Reading response timed-out.")
except socket.error as e:
sock.close()
raise pyzor.CommError("Socket error while reading response: %s"
% e)
self.log.debug("received: %r/%r", packet, address)
msg = email.message_from_bytes(packet, _class=pyzor.message.Response)
msg.ensure_complete()
try:
thread_id = msg.get_thread()
if thread_id != expected_id:
if thread_id.in_ok_range():
raise pyzor.ProtocolError(
"received unexpected thread id %d (expected %d)" %
(thread_id, expected_id))
self.log.warn("received error thread id %d (expected %d)",
thread_id, expected_id)
except KeyError:
self.log.warn("no thread id received")
return msg
class ClientRunner(object):
__slots__ = ['routine', 'all_ok', 'log']
def __init__(self, routine):
self.log = logging.getLogger("pyzor")
self.routine = routine
self.all_ok = True
def run(self, server, args, kwargs=None):
if kwargs is None:
kwargs = {}
message = "%s:%s\t" % server
response = None
try:
response = self.routine(*args, **kwargs)
self.handle_response(response, message)
except (pyzor.CommError, KeyError, ValueError), e:
self.log.error("%s\t%s: %s", server, e.__class__.__name__, e)
self.all_ok = False
def handle_response(self, response, message):
"""mesaage is a string we've built up so far"""
if not response.is_ok():
self.all_ok = False
sys.stdout.write("%s%s\n" % (message, response.head_tuple()))
class CheckClientRunner(ClientRunner):
def __init__(self, routine, r_count=0, wl_count=0):
ClientRunner.__init__(self, routine)
self.found_hit = False
self.whitelisted = False
self.hit_count = 0
self.whitelist_count = 0
self.r_count_found = r_count
self.wl_count_clears = wl_count
def handle_response(self, response, message):
message += "%s\t" % str(response.head_tuple())
if response.is_ok():
self.hit_count = int(response['Count'])
self.whitelist_count = int(response['WL-Count'])
if self.whitelist_count > self.wl_count_clears:
self.whitelisted = True
elif self.hit_count > self.r_count_found:
self.found_hit = True
message += "%d\t%d" % (self.hit_count, self.whitelist_count)
sys.stdout.write(message + '\n')
else:
self.all_ok = False
sys.stdout.write(message + '\n')
class InfoClientRunner(ClientRunner):
def handle_response(self, response, message):
message += "%s\n" % str(response.head_tuple())
if response.is_ok():
for f in ('Count', 'Entered', 'Updated',
'WL-Count', 'WL-Entered', 'WL-Updated'):
if response.has_key(f):
val = int(response[f])
if 'Count' in f:
stringed = str(val)
elif val == -1:
stringed = 'Never'
else:
stringed = time.ctime(val)
message += ("\t%s: %s\n" % (f, stringed))
sys.stdout.write(message + "\n")
else:
self.all_ok = False
sys.stdout.write(message + "\n")

View File

@@ -0,0 +1,220 @@
"""Functions that handle parsing pyzor configuration files."""
import os
import re
import logging
import collections
import pyzor.account
# Configuration files for the Pyzor Server
def load_access_file(access_fn, accounts):
"""Load the ACL from the specified file, if it exists, and return an
ACL dictionary, where each key is a username and each value is a set
of allowed permissions (if the permission is not in the set, then it
is not allowed).
'accounts' is a dictionary of accounts that exist on the server - only
the keys are used, which must be the usernames (these are the users
that are granted permission when the 'all' keyword is used, as
described below).
Each line of the file should be in the following format:
operation : user : allow|deny
where 'operation' is a space-separated list of pyzor commands or the
keyword 'all' (meaning all commands), 'username' is a space-separated
list of usernames or the keyword 'all' (meaning all users) - the
anonymous user is called "anonymous", and "allow|deny" indicates whether
or not the specified user(s) may execute the specified operations.
The file is processed from top to bottom, with the final match for
user/operation being the value taken. Every file has the following
implicit final rule:
all : all : deny
If the file does not exist, then the following default is used:
check report ping info : anonymous : allow
"""
log = logging.getLogger("pyzord")
# A defaultdict is safe, because if we get a non-existant user, we get
# the empty set, which is the same as a deny, which is the final
# implicit rule.
acl = collections.defaultdict(set)
if not os.path.exists(access_fn):
log.info("Using default ACL: the anonymous user may use the check, "
"report, ping and info commands.")
acl[pyzor.anonymous_user] = set(("check", "report", "ping", "pong",
"info"))
return acl
for line in open(access_fn):
if not line.strip() or line[0] == "#":
continue
try:
operations, users, allowed = [part.lower().strip()
for part in line.split(":")]
except ValueError:
log.warn("Invalid ACL line: %r", line)
continue
try:
allowed = {"allow": True, "deny" : False}[allowed]
except KeyError:
log.warn("Invalid ACL line: %r", line)
continue
if operations == "all":
operations = ("check", "report", "ping", "pong", "info",
"whitelist")
else:
operations = [operation.strip()
for operation in operations.split()]
if users == "all":
users = accounts
else:
users = [user.strip() for user in users.split()]
for user in users:
if allowed:
log.debug("Granting %s to %s.", ",".join(operations), user)
# If these operations are already allowed, this will have
# no effect.
acl[user].update(operations)
else:
log.debug("Revoking %s from %s.", ",".join(operations), user)
# If these operations are not allowed yet, this will have
# no effect.
acl[user].difference_update(operations)
log.info("ACL: %r", acl)
return acl
def load_passwd_file(passwd_fn):
"""Load the accounts from the specified file.
Each line of the file should be in the format:
username : key
If the file does not exist, then an empty dictionary is returned;
otherwise, a dictionary of (username, key) items is returned.
"""
log = logging.getLogger("pyzord")
accounts = {}
if not os.path.exists(passwd_fn):
log.info("Accounts file does not exist - only the anonymous user "
"will be available.")
return accounts
for line in open(passwd_fn):
if not line.strip() or line[0] == "#":
continue
try:
user, key = line.split(":")
except ValueError:
log.warn("Invalid accounts line: %r", line)
continue
user = user.strip()
key = key.strip()
log.debug("Creating an account for %s with key %s.", user, key)
accounts[user] = key
# Don't log the keys at 'info' level, just ther usernames.
log.info("Accounts: %s", ",".join(accounts))
return accounts
# Configuration files for the Pyzor Client
def load_accounts(filepath):
"""Layout of file is: host : port : username : salt,key"""
accounts = {}
log = logging.getLogger("pyzor")
if os.path.exists(filepath):
for lineno, orig_line in enumerate(open(filepath)):
line = orig_line.strip()
if not line or line.startswith('#'):
continue
try:
host, port, username, key = [x.strip()
for x in line.split(":")]
except ValueError:
log.warn("account file: invalid line %d: wrong number of "
"parts", lineno)
continue
try:
port = int(port)
except ValueError, e:
log.warn("account file: invalid line %d: %s", lineno, e)
address = (host, port)
salt, key = pyzor.account.key_from_hexstr(key)
if not salt and not key:
log.warn("account file: invalid line %d: keystuff can't be "
"all None's", lineno)
continue
try:
accounts[address] = pyzor.account.Account(username, salt, key)
except ValueError, e:
log.warn("account file: invalid line %d: %s", lineno, e)
else:
log.warn("No accounts are setup. All commands will be executed by "
"the anonymous user.")
return accounts
def load_servers(filepath):
"""Load the servers file."""
logger = logging.getLogger("pyzor")
if not os.path.exists(filepath):
servers = []
else:
servers = []
with open(filepath) as f:
for line in f:
line = line.strip()
if re.match("[^#][a-zA-Z0-9.-]+:[0-9]+", line):
address, port = line.rsplit(":", 1)
servers.append((address, int(port)))
if not servers:
logger.info("No servers specified, defaulting to public.pyzor.org.")
servers = [("public.pyzor.org", 24441)]
return servers
# Common configurations
def setup_logging(log_name, filepath, debug):
"""Setup logging according to the specified options. Return the Logger
object.
"""
fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
stream_handler = logging.StreamHandler()
file_handler = None
if debug:
stream_log_level = logging.DEBUG
file_log_level = logging.DEBUG
else:
stream_log_level = logging.CRITICAL
file_log_level = logging.INFO
logger = logging.getLogger(log_name)
logger.setLevel(file_log_level)
stream_handler.setLevel(stream_log_level)
stream_handler.setFormatter(fmt)
logger.addHandler(stream_handler)
if filepath:
file_handler = logging.FileHandler(filepath)
file_handler.setLevel(file_log_level)
file_handler.setFormatter(fmt)
logger.addHandler(file_handler)
return logger
def expand_homefiles(homefiles, category, homedir, config):
"""Set the full file path for these configuration files."""
for filename in homefiles:
filepath = config.get(category, filename)
if not filepath:
continue
filepath = os.path.expanduser(filepath)
if not os.path.isabs(filepath):
filepath = os.path.join(homedir, filepath)
config.set(category, filename, filepath)

View File

@@ -0,0 +1,149 @@
import re
import hashlib
import HTMLParser
# Hard-coded for the moment.
digest_spec = ([(20, 3), (60, 3)])
class HTMLStripper(HTMLParser.HTMLParser):
"""Strip all tags from the HTML."""
def __init__(self, collector):
HTMLParser.HTMLParser.__init__(self)
self.reset()
self.collector = collector
def handle_data(self, data):
"""Keep track of the data."""
data = data.strip()
if data:
self.collector.append(data)
class DataDigester(object):
"""The major workhouse class."""
__slots__ = ['value', 'digest']
# Minimum line length for it to be included as part of the digest.
min_line_length = 8
# If a message is this many lines or less, then we digest the whole
# message.
atomic_num_lines = 4
# We're not going to try to match email addresses as per the spec
# because it's too difficult. Plus, regular expressions don't work well
# for them. (BNF is better at balanced parens and such).
email_ptrn = re.compile(r'\S+@\S+')
# Same goes for URLs.
url_ptrn = re.compile(r'[a-z]+:\S+', re.IGNORECASE)
# We also want to remove anything that is so long it looks like possibly
# a unique identifier.
longstr_ptrn = re.compile(r'\S{10,}')
ws_ptrn = re.compile(r'\s')
# String that the above patterns will be replaced with.
# Note that an empty string will always be used to remove whitespace.
unwanted_txt_repl = ''
def __init__(self, msg, spec=digest_spec):
self.value = None
self.digest = hashlib.sha1()
# Need to know the total number of lines in the content.
lines = []
for payload in self.digest_payloads(msg):
for line in payload.splitlines():
norm = self.normalize(line)
if self.should_handle_line(norm):
lines.append(norm.encode("utf8"))
if len(lines) <= self.atomic_num_lines:
self.handle_atomic(lines)
else:
self.handle_pieced(lines, spec)
self.value = self.digest.hexdigest()
assert len(self.value) == len(hashlib.sha1(b"").hexdigest())
assert self.value is not None
def handle_atomic(self, lines):
"""We digest everything."""
for line in lines:
self.handle_line(line)
def handle_pieced(self, lines, spec):
"""Digest stuff according to the spec."""
for offset, length in spec:
for i in xrange(length):
try:
line = lines[int(offset * len(lines) // 100) + i]
except IndexError:
pass
else:
self.handle_line(line)
def handle_line(self, line):
self.digest.update(line.rstrip())
@classmethod
def normalize(cls, s):
repl = cls.unwanted_txt_repl
s = cls.longstr_ptrn.sub(repl, s)
s = cls.email_ptrn.sub(repl, s)
s = cls.url_ptrn.sub(repl, s)
# Make sure we do the whitespace last because some of the previous
# patterns rely on whitespace.
return cls.ws_ptrn.sub('', s).strip()
@staticmethod
def normalize_html_part(s):
data = []
stripper = HTMLStripper(data)
try:
stripper.feed(s)
except (UnicodeDecodeError, HTMLParser.HTMLParseError):
# We can't parse the HTML, so just strip it. This is still
# better than including generic HTML/CSS text.
pass
return " ".join(data)
@classmethod
def should_handle_line(cls, s):
return len(s) and cls.min_line_length <= len(s)
@classmethod
def digest_payloads(cls, msg):
for part in msg.walk():
if part.get_content_maintype() == "text":
payload = part.get_payload(decode=True)
charset = part.get_content_charset()
if not charset:
charset = "ascii"
try:
payload = payload.decode(charset, "ignore")
except LookupError:
payload = payload.decode("ascii", "ignore")
if part.get_content_subtype() == "html":
yield cls.normalize_html_part(payload)
else:
yield payload
elif part.is_multipart():
# Skip, because walk() will give us the payload next.
pass
else:
# Non-text parts are passed through as-is.
yield part.get_payload()
class PrintingDataDigester(DataDigester):
"""Extends DataDigester: prints out what we're digesting."""
def handle_line(self, line):
print line.decode("utf8")
super(PrintingDataDigester, self).handle_line(line)
# Convenience function.
def get_digest(msg):
return DataDigester(msg).value

View File

@@ -0,0 +1,27 @@
"""Database backends for pyzord.
The database class must expose a dictionary-like interface, allowing access
via __getitem__, __setitem__, and __delitem__. The key will be a forty
character string, and the value should be an instance of the Record class.
If the database backend cannot store the Record objects natively, then it
must transparently take care of translating to/from Record objects in
__setitem__ and __getitem__.
The database class should take care of expiring old values at the
appropriate interval.
"""
from pyzor.engines import gdbm_
from pyzor.engines import mysql
from pyzor.engines import redis_
__all__ = ["database_classes"]
database_classes = {"gdbm": gdbm_.handle,
"mysql": mysql.handle,
"redis": redis_.handle,
}

View File

@@ -0,0 +1,51 @@
"""Common library shared by different engines."""
import sys
import datetime
from collections import namedtuple
__all__ = ["DBHandle", "DatabaseError", "Record"]
DBHandle = namedtuple("DBHandle", ["single_threaded", "multi_threaded",
"multi_processing"])
class DatabaseError(Exception):
pass
class Record(object):
"""Prefix conventions used in this class:
r = report (spam)
wl = whitelist
"""
def __init__(self, r_count=0, wl_count=0, r_entered=None,
r_updated=None, wl_entered=None, wl_updated=None):
self.r_count = r_count
self.wl_count = wl_count
self.r_entered = r_entered
self.r_updated = r_updated
self.wl_entered = wl_entered
self.wl_updated = wl_updated
def wl_increment(self):
# overflow prevention
if self.wl_count < sys.maxint:
self.wl_count += 1
if self.wl_entered is None:
self.wl_entered = datetime.datetime.now()
self.wl_update()
def r_increment(self):
# overflow prevention
if self.r_count < sys.maxint:
self.r_count += 1
if self.r_entered is None:
self.r_entered = datetime.datetime.now()
self.r_update()
def r_update(self):
self.r_updated = datetime.datetime.now()
def wl_update(self):
self.wl_updated = datetime.datetime.now()

View File

@@ -0,0 +1,174 @@
"""Gdbm database engine."""
try:
import gdbm
except ImportError:
gdbm = None
import sys
import time
import logging
import datetime
import threading
from pyzor.engines.common import *
class GdbmDBHandle(object):
absolute_source = True
sync_period = 60
reorganize_period = 3600 * 24 # 1 day
_dt_decode = lambda x: None if x == 'None' else datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f")
fields = (
'r_count', 'r_entered', 'r_updated',
'wl_count', 'wl_entered', 'wl_updated',
)
_fields = [('r_count', int),
('r_entered', _dt_decode),
('r_updated', _dt_decode),
('wl_count', int),
('wl_entered', _dt_decode),
('wl_updated', _dt_decode)]
this_version = '1'
log = logging.getLogger("pyzord")
def __init__(self, fn, mode, max_age=None):
self.max_age = max_age
self.db = gdbm.open(fn, mode)
self.start_reorganizing()
self.start_syncing()
def apply_method(self, method, varargs=(), kwargs=None):
if kwargs is None:
kwargs = {}
return apply(method, varargs, kwargs)
def __getitem__(self, key):
return self.apply_method(self._really_getitem, (key,))
def _really_getitem(self, key):
return GdbmDBHandle.decode_record(self.db[key])
def __setitem__(self, key, value):
self.apply_method(self._really_setitem, (key, value))
def _really_setitem(self, key, value):
self.db[key] = GdbmDBHandle.encode_record(value)
def __delitem__(self, key):
self.apply_method(self._really_delitem, (key,))
def _really_delitem(self, key):
del self.db[key]
def start_syncing(self):
if self.db:
self.apply_method(self._really_sync)
self.sync_timer = threading.Timer(self.sync_period,
self.start_syncing)
self.sync_timer.setDaemon(True)
self.sync_timer.start()
def _really_sync(self):
self.db.sync()
def start_reorganizing(self):
if not self.max_age:
return
if self.db:
self.apply_method(self._really_reorganize)
self.reorganize_timer = threading.Timer(self.reorganize_period,
self.start_reorganizing)
self.reorganize_timer.setDaemon(True)
self.reorganize_timer.start()
def _really_reorganize(self):
self.log.debug("reorganizing the database")
key = self.db.firstkey()
breakpoint = time.time() - self.max_age
while key is not None:
rec = self._really_getitem(key)
delkey = None
if int(time.mktime(rec.r_updated.timetuple())) < breakpoint:
self.log.debug("deleting key %s", key)
delkey = key
key = self.db.nextkey(key)
if delkey:
self._really_delitem(delkey)
self.db.reorganize()
@classmethod
def encode_record(cls, value):
values = [cls.this_version]
values.extend(["%s" % getattr(value, x) for x in cls.fields])
return ",".join(values)
@classmethod
def decode_record(cls, s):
try:
s = s.decode("utf8")
except UnicodeError:
raise StandardError("don't know how to handle db value %s" %
repr(s))
parts = s.split(',')
dispatch = None
version = parts[0]
if len(parts) == 3:
dispatch = cls.decode_record_0
elif version == '1':
dispatch = cls.decode_record_1
else:
raise StandardError("don't know how to handle db value %s" %
repr(s))
return dispatch(s)
@staticmethod
def decode_record_0(s):
r = Record()
parts = s.split(',')
fields = ('r_count', 'r_entered', 'r_updated')
assert len(parts) == len(fields)
for i in range(len(parts)):
setattr(r, fields[i], int(parts[i]))
return r
@classmethod
def decode_record_1(cls, s):
r = Record()
parts = s.split(',')[1:]
assert len(parts) == len(cls.fields)
for part, field in zip(parts, cls._fields):
f, decode = field
setattr(r, f, decode(part))
return r
class ThreadedGdbmDBHandle(GdbmDBHandle):
"""Like GdbmDBHandle, but handles multi-threaded access."""
def __init__(self, fn, mode, max_age=None, bound=None):
self.db_lock = threading.Lock()
GdbmDBHandle.__init__(self, fn, mode, max_age=max_age)
def apply_method(self, method, varargs=(), kwargs=None):
if kwargs is None:
kwargs = {}
with self.db_lock:
return GdbmDBHandle.apply_method(self, method, varargs=varargs,
kwargs=kwargs)
# This won't work because the gdbm object needs to be in shared memory of the
# spawned processes.
# class ProcessGdbmDBHandle(ThreadedGdbmDBHandle):
# def __init__(self, fn, mode, max_age=None, bound=None):
# ThreadedGdbmDBHandle.__init__(self, fn, mode, max_age=max_age,
# bound=bound)
# self.db_lock = multiprocessing.Lock()
if sys.version_info[0] != 3 and gdbm is None:
handle = DBHandle(single_threaded=None,
multi_threaded=None,
multi_processing=None)
else:
handle = DBHandle(single_threaded=GdbmDBHandle,
multi_threaded=ThreadedGdbmDBHandle,
multi_processing=None)

View File

@@ -0,0 +1,268 @@
"""MySQLdb database engine."""
import time
import Queue
import logging
import datetime
import threading
try:
import MySQLdb
except ImportError:
# The SQL database backend will not work.
MySQLdb = None
from pyzor.engines.common import *
class MySQLDBHandle(object):
absolute_source = False
# The table must already exist, and have this schema:
# CREATE TABLE `public` (
# `digest` char(40) default NULL,
# `r_count` int(11) default NULL,
# `wl_count` int(11) default NULL,
# `r_entered` datetime default NULL,
# `wl_entered` datetime default NULL,
# `r_updated` datetime default NULL,
# `wl_updated` datetime default NULL,
# PRIMARY KEY (`digest`)
# )
# XXX Re-organising might be faster with a r_updated index. However,
# XXX the re-organisation time isn't that important, and that would
# XXX (slightly) slow down all inserts, so we leave it for now.
reorganize_period = 3600 * 24 # 1 day
reconnect_period = 60 # seconds
log = logging.getLogger("pyzord")
def __init__(self, fn, mode, max_age=None):
self.max_age = max_age
self.db = None
# The 'fn' is host,user,password,db,table. We ignore mode.
# We store the authentication details so that we can reconnect if
# necessary.
self.host, self.user, self.passwd, self.db_name, \
self.table_name = fn.split(",")
self.last_connect_attempt = 0 # We have never connected.
self.reconnect()
self.start_reorganizing()
def _get_new_connection(self):
"""Returns a new db connection."""
db = MySQLdb.connect(host=self.host, user=self.user,
db=self.db_name, passwd=self.passwd)
db.autocommit(True)
return db
def _check_reconnect_time(self):
if time.time() - self.last_connect_attempt < self.reconnect_period:
# Too soon to reconnect.
self.log.debug("Can't reconnect until %s",
(time.ctime(self.last_connect_attempt +
self.reconnect_period)))
return False
return True
def reconnect(self):
if not self._check_reconnect_time():
return
if self.db:
try:
self.db.close()
except MySQLdb.Error:
pass
try:
self.db = self._get_new_connection()
except MySQLdb.Error, e:
self.log.error("Unable to connect to database: %s", e)
self.db = None
# Keep track of when we connected, so that we don't retry too often.
self.last_connect_attempt = time.time()
def __del__(self):
"""Close the database when the object is no longer needed."""
try:
if self.db:
self.db.close()
except MySQLdb.Error:
pass
def _safe_call(self, name, method, args):
try:
return method(*args, db=self.db)
except (MySQLdb.Error, AttributeError), e:
self.log.error("%s failed: %s", name, e)
self.reconnect()
# Retrying just complicates the logic - we don't really care if
# a single query fails (and it's possible that it would fail)
# on the second attempt anyway. Any exceptions are caught by
# the server, and a 'nice' message provided to the caller.
raise DatabaseError("Database temporarily unavailable.")
def __getitem__(self, key):
return self._safe_call("getitem", self._really__getitem__, (key,))
def __setitem__(self, key, value):
return self._safe_call("setitem", self._really__setitem__,
(key, value))
def __delitem__(self, key):
return self._safe_call("delitem", self._really__delitem__, (key,))
def _really__getitem__(self, key, db=None):
"""__getitem__ without the exception handling."""
c = db.cursor()
# The order here must match the order of the arguments to the
# Record constructor.
c.execute("SELECT r_count, wl_count, r_entered, r_updated, "
"wl_entered, wl_updated FROM %s WHERE digest=%%s" %
self.table_name, (key,))
try:
try:
return Record(*c.fetchone())
except TypeError:
# fetchone() returned None, i.e. there is no such record
raise KeyError()
finally:
c.close()
def _really__setitem__(self, key, value, db=None):
"""__setitem__ without the exception handling."""
c = db.cursor()
try:
c.execute("INSERT INTO %s (digest, r_count, wl_count, "
"r_entered, r_updated, wl_entered, wl_updated) "
"VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s) ON "
"DUPLICATE KEY UPDATE r_count=%%s, wl_count=%%s, "
"r_entered=%%s, r_updated=%%s, wl_entered=%%s, "
"wl_updated=%%s" % self.table_name,
(key, value.r_count, value.wl_count, value.r_entered,
value.r_updated, value.wl_entered, value.wl_updated,
value.r_count, value.wl_count, value.r_entered,
value.r_updated, value.wl_entered, value.wl_updated))
finally:
c.close()
def _really__delitem__(self, key, db=None):
"""__delitem__ without the exception handling."""
c = db.cursor()
try:
c.execute("DELETE FROM %s WHERE digest=%%s" % self.table_name,
(key,))
finally:
c.close()
def start_reorganizing(self):
if not self.max_age:
return
self.log.debug("reorganizing the database")
breakpoint = (datetime.datetime.now() -
datetime.timedelta(seconds=self.max_age))
db = self._get_new_connection()
c = db.cursor()
try:
c.execute("DELETE FROM %s WHERE r_updated<%%s" %
self.table_name, (breakpoint,))
except (MySQLdb.Error, AttributeError), e:
self.log.warn("Unable to reorganise: %s", e)
finally:
c.close()
db.close()
self.reorganize_timer = threading.Timer(self.reorganize_period,
self.start_reorganizing)
self.reorganize_timer.setDaemon(True)
self.reorganize_timer.start()
class ThreadedMySQLDBHandle(MySQLDBHandle):
def __init__(self, fn, mode, max_age=None, bound=None):
self.bound = bound
if self.bound:
self.db_queue = Queue.Queue()
MySQLDBHandle.__init__(self, fn, mode, max_age=max_age)
def _get_connection(self):
if self.bound:
return self.db_queue.get()
else:
return self._get_new_connection()
def _release_connection(self, db):
if self.bound:
self.db_queue.put(db)
else:
db.close()
def _safe_call(self, name, method, args):
db = self._get_connection()
try:
return method(*args, db=db)
except (MySQLdb.Error, AttributeError) as e:
self.log.error("%s failed: %s", name, e)
if not self.bound:
raise DatabaseError("Database temporarily unavailable.")
try:
# Connection might be timeout, ping and retry
db.ping(True)
return method(*args, db=db)
except (MySQLdb.Error, AttributeError) as e:
# attempt a new connection, if we can retry
db = self._reconnect(db)
raise DatabaseError("Database temporarily unavailable.")
finally:
self._release_connection(db)
def reconnect(self):
if not self.bound:
return
for _ in xrange(self.bound):
self.db_queue.put(self._get_new_connection())
def _reconnect(self, db):
if not self._check_reconnect_time():
return db
else:
self.last_connect_attempt = time.time()
return self._get_new_connection()
def __del__(self):
if not self.bound:
return
for db in iter(self.db_queue.get_nowait):
try:
db.close()
except MySQLdb.Error:
continue
except Queue.Empty:
break
class ProcessMySQLDBHandle(MySQLDBHandle):
def __init__(self, fn, mode, max_age=None):
MySQLDBHandle.__init__(self, fn, mode, max_age=max_age)
def reconnect(self):
pass
def __del__(self):
pass
def _safe_call(self, name, method, args):
db = None
try:
db = self._get_new_connection()
return method(*args, db=db)
except (MySQLdb.Error, AttributeError) as e:
self.log.error("%s failed: %s", name, e)
raise DatabaseError("Database temporarily unavailable.")
finally:
if db is not None:
db.close()
if MySQLdb is None:
handle = DBHandle(single_threaded=None,
multi_threaded=None,
multi_processing=None)
else:
handle = DBHandle(single_threaded=MySQLDBHandle,
multi_threaded=ThreadedMySQLDBHandle,
multi_processing=ProcessMySQLDBHandle)

View File

@@ -0,0 +1,110 @@
"""Redis database engine."""
import logging
import datetime
try:
import redis
except ImportError:
redis = None
from pyzor.engines.common import *
NAMESPACE = "pyzord.digest"
encode_date = lambda d: "" if d is None else d.strftime("%Y-%m-%d %H:%M:%S")
decode_date = lambda x: None if x == "" else datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
def safe_call(f):
"""Decorator that wraps a method for handling database operations."""
def wrapped_f(self, *args, **kwargs):
# This only logs the error and raise the usual Error for consistency,
# the redis library takes care of reconnecting and everything else.
try:
return f(self, *args, **kwargs)
except redis.exceptions.RedisError as e:
self.log.error("Redis error while calling %s: %s",
f.__name__, e)
raise DatabaseError("Database temporarily unavailable.")
return wrapped_f
class RedisDBHandle(object):
absolute_source = False
log = logging.getLogger("pyzord")
def __init__(self, fn, mode, max_age=None):
self.max_age = max_age
# The 'fn' is host,port,password,db. We ignore mode.
# We store the authentication details so that we can reconnect if
# necessary.
fn = fn.split(",")
self.host = fn[0] or "localhost"
self.port = fn[1] or "6379"
self.passwd = fn[2] or None
self.db_name = fn[3] or "0"
self.db = self._get_new_connection()
@staticmethod
def _encode_record(r):
return ("%s,%s,%s,%s,%s,%s" %
(r.r_count,
encode_date(r.r_entered),
encode_date(r.r_updated),
r.wl_count,
encode_date(r.wl_entered),
encode_date(r.wl_updated))).encode()
@staticmethod
def _decode_record(r):
if r is None:
return Record()
fields = r.decode().split(",")
return Record(r_count=int(fields[0]),
r_entered=decode_date(fields[1]),
r_updated=decode_date(fields[2]),
wl_count=int(fields[3]),
wl_entered=decode_date(fields[4]),
wl_updated=decode_date(fields[5]))
@staticmethod
def _real_key(key):
return "%s.%s" % (NAMESPACE, key)
@safe_call
def _get_new_connection(self):
return redis.StrictRedis(host=self.host, port=int(self.port),
db=int(self.db_name), password=self.passwd)
@safe_call
def __getitem__(self, key):
return self._decode_record(self.db.get(self._real_key(key)))
@safe_call
def __setitem__(self, key, value):
if self.max_age is None:
self.db.set(self._real_key(key), self._encode_record(value))
else:
self.db.setex(self._real_key(key), self.max_age,
self._encode_record(value))
@safe_call
def __delitem__(self, key):
self.db.delete(self._real_key(key))
class ThreadedRedisDBHandle(RedisDBHandle):
def __init__(self, fn, mode, max_age=None, bound=None):
RedisDBHandle.__init__(self, fn, mode, max_age=max_age)
if redis is None:
handle = DBHandle(single_threaded=None,
multi_threaded=None,
multi_processing=None)
else:
handle = DBHandle(single_threaded=RedisDBHandle,
multi_threaded=ThreadedRedisDBHandle,
multi_processing=None)

View File

@@ -0,0 +1 @@
"""Various hack to make pyzor compatible with different Python versions."""

View File

@@ -0,0 +1,42 @@
"""Hacks for Python 2.6"""
__all__ = ["hack_all", "hack_email", "hack_select"]
def hack_all(email=True, select=True):
if email:
hack_email()
if select:
hack_select()
def hack_email():
"""The python2.6 version of email.message_from_string, doesn't work with
unicode strings. And in python3 it will only work with a decoded.
So switch to using only message_from_bytes.
"""
import email
if not hasattr(email, "message_from_bytes"):
email.message_from_bytes = email.message_from_string
def hack_select():
"""The python2.6 version of SocketServer does not handle interrupt calls
from signals. Patch the select call if necessary.
"""
import sys
if sys.version_info[0] == 2 and sys.version_info[1] == 6:
import select
import errno
real_select = select.select
def _eintr_retry(*args):
"""restart a system call interrupted by EINTR"""
while True:
try:
return real_select(*args)
except (OSError, select.error) as e:
if e.args[0] != errno.EINTR:
raise
select.select = _eintr_retry

View File

@@ -0,0 +1,152 @@
"""This modules contains the various messages used in the pyzor client server
communication.
"""
import random
import email.message
import pyzor
class Message(email.message.Message):
def __init__(self):
email.message.Message.__init__(self)
self.setup()
def setup(self):
pass
def init_for_sending(self):
self.ensure_complete()
def __str__(self):
# The parent class adds the unix From header.
return self.as_string()
def ensure_complete(self):
pass
class ThreadedMessage(Message):
def init_for_sending(self):
if not self.has_key('Thread'):
self.set_thread(ThreadId.generate())
assert self.has_key('Thread')
self["PV"] = str(pyzor.proto_version)
Message.init_for_sending(self)
def ensure_complete(self):
if not (self.has_key('PV') and self.has_key('Thread')):
raise pyzor.IncompleteMessageError("Doesn't have fields for a "
"ThreadedMessage.")
Message.ensure_complete(self)
def get_protocol_version(self):
return float(self['PV'])
def get_thread(self):
return ThreadId(self['Thread'])
def set_thread(self, i):
self['Thread'] = str(i)
class Response(ThreadedMessage):
ok_code = 200
def ensure_complete(self):
if not (self.has_key('Code') and self.has_key('Diag')):
raise pyzor.IncompleteMessageError("doesn't have fields for a "
"Response")
ThreadedMessage.ensure_complete(self)
def is_ok(self):
return self.get_code() == self.ok_code
def get_code(self):
return int(self['Code'])
def get_diag(self):
return self['Diag']
def head_tuple(self):
return self.get_code(), self.get_diag()
class Request(ThreadedMessage):
"""This is the class that should be used to read in Requests of any type.
Subclasses are responsible for setting 'Op' if they are generating a
message,"""
def get_op(self):
return self['Op']
def ensure_complete(self):
if not self.has_key('Op'):
raise pyzor.IncompleteMessageError("doesn't have fields for a "
"Request")
ThreadedMessage.ensure_complete(self)
class ClientSideRequest(Request):
op = None
def setup(self):
Request.setup(self)
self["Op"] = self.op
class SimpleDigestBasedRequest(ClientSideRequest):
def __init__(self, digest):
ClientSideRequest.__init__(self)
self["Op-Digest"] = digest
class SimpleDigestSpecBasedRequest(SimpleDigestBasedRequest):
def __init__(self, digest, spec):
SimpleDigestBasedRequest.__init__(self, digest)
flat_spec = [item for sublist in spec for item in sublist]
self["Op-Spec"] = ",".join(str(part) for part in flat_spec)
class PingRequest(ClientSideRequest):
op = "ping"
class PongRequest(SimpleDigestBasedRequest):
op = "pong"
class CheckRequest(SimpleDigestBasedRequest):
op = "check"
class InfoRequest(SimpleDigestBasedRequest):
op = "info"
class ReportRequest(SimpleDigestSpecBasedRequest):
op = "report"
class WhitelistRequest(SimpleDigestSpecBasedRequest):
op = "whitelist"
class ThreadId(int):
# (0, 1024) is reserved
full_range = (0, 2 ** 16)
ok_range = (1024, full_range[1])
error_value = 0
def __new__(cls, i):
self = int.__new__(cls, i)
if not (cls.full_range[0] <= self < cls.full_range[1]):
raise ValueError("value outside of range")
return self
@classmethod
def generate(cls):
return cls(random.randrange(*cls.ok_range))
def in_ok_range(self):
return self.ok_range[0] <= self < self.ok_range[1]

View File

@@ -0,0 +1,299 @@
"""Networked spam-signature detection server.
The server receives the request in the form of a RFC5321 message, and
responds with another RFC5321 message. Neither of these messages has a
body - all of the data is encapsulated in the headers.
The response headers will always include a "Code" header, which is a
HTTP-style response code, and a "Diag" header, which is a human-readable
message explaining the response code (typically this will be "OK").
Both the request and response headers always include a "PV" header, which
indicates the protocol version that is being used (in a major.minor format).
Both the requestion and response headers also always include a "Thread",
which uniquely identifies the request (this is a requirement of using UDP).
Responses to requests may arrive in any order, but the "Thread" header of
a response will always match the "Thread" header of the appropriate request.
Authenticated requests must also have "User", "Time" (timestamp), and "Sig"
(signature) headers.
"""
import sys
import time
import socket
import signal
import logging
import StringIO
import threading
import traceback
import SocketServer
import email.message
import pyzor.config
import pyzor.account
import pyzor.engines.common
import pyzor.hacks.py26
pyzor.hacks.py26.hack_all()
class Server(SocketServer.UDPServer):
"""The pyzord server. Handles incoming UDP connections in a single
thread and single process."""
max_packet_size = 8192
time_diff_allowance = 180
def __init__(self, address, database, passwd_fn, access_fn):
if ":" in address[0]:
Server.address_family = socket.AF_INET6
else:
Server.address_family = socket.AF_INET
self.log = logging.getLogger("pyzord")
self.usage_log = logging.getLogger("pyzord-usage")
self.database = database
# Handle configuration files
self.passwd_fn = passwd_fn
self.access_fn = access_fn
self.load_config()
self.log.debug("Listening on %s", address)
SocketServer.UDPServer.__init__(self, address, RequestHandler,
bind_and_activate=False)
try:
self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
except (AttributeError, socket.error) as e:
self.log.debug("Unable to set IPV6_V6ONLY to false %s", e)
self.server_bind()
self.server_activate()
# Finally, set signals
signal.signal(signal.SIGUSR1, self.reload_handler)
signal.signal(signal.SIGTERM, self.shutdown_handler)
def load_config(self):
"""Reads the configuration files and loads the accounts and ACLs."""
self.accounts = pyzor.config.load_passwd_file(self.passwd_fn)
self.acl = pyzor.config.load_access_file(self.access_fn, self.accounts)
def shutdown_handler(self, *args, **kwargs):
"""Handler for the SIGTERM signal. This should be used to kill the
daemon and ensure proper clean-up.
"""
self.log.info("SIGTERM received. Shutting down.")
t = threading.Thread(target=self.shutdown)
t.start()
def reload_handler(self, *args, **kwargs):
"""Handler for the SIGUSR1 signal. This should be used to reload
the configuration files.
"""
self.log.info("SIGUSR1 received. Reloading configuration.")
t = threading.Thread(target=self.load_config)
t.start()
class ThreadingServer(SocketServer.ThreadingMixIn, Server):
"""A threaded version of the pyzord server. Each connection is served
in a new thread. This may not be suitable for all database types."""
pass
class BoundedThreadingServer(ThreadingServer):
"""Same as ThreadingServer but this also accepts a limited number of
concurrent threads.
"""
def __init__(self, address, database, passwd_fn, access_fn, max_threads):
ThreadingServer.__init__(self, address, database, passwd_fn, access_fn)
self.semaphore = threading.Semaphore(max_threads)
def process_request(self, request, client_address):
self.semaphore.acquire()
ThreadingServer.process_request(self, request, client_address)
def process_request_thread(self, request, client_address):
ThreadingServer.process_request_thread(self, request, client_address)
self.semaphore.release()
class ProcessServer(SocketServer.ForkingMixIn, Server):
"""A multi-processing version of the pyzord server. Each connection is
served in a new process. This may not be suitable for all database types.
"""
def __init__(self, address, database, passwd_fn, access_fn,
max_children=40):
ProcessServer.max_children = max_children
Server.__init__(self, address, database, passwd_fn, access_fn)
class RequestHandler(SocketServer.DatagramRequestHandler):
"""Handle a single pyzord request."""
def __init__(self, *args, **kwargs):
self.response = email.message.Message()
SocketServer.DatagramRequestHandler.__init__(self, *args, **kwargs)
def handle(self):
"""Handle a pyzord operation, cleanly handling any errors."""
self.response["Code"] = "200"
self.response["Diag"] = "OK"
self.response["PV"] = "%s" % pyzor.proto_version
try:
self._really_handle()
except NotImplementedError, e:
self.handle_error(501, "Not implemented: %s" % e)
except pyzor.UnsupportedVersionError, e:
self.handle_error(505, "Version Not Supported: %s" % e)
except pyzor.ProtocolError, e:
self.handle_error(400, "Bad request: %s" % e)
except pyzor.SignatureError, e:
self.handle_error(401, "Unauthorized: Signature Error: %s" % e)
except pyzor.AuthorizationError, e:
self.handle_error(403, "Forbidden: %s" % e)
except Exception, e:
self.handle_error(500, "Internal Server Error: %s" % e)
trace = StringIO.StringIO()
traceback.print_exc(file=trace)
trace.seek(0)
self.server.log.error(trace.read())
self.server.log.debug("Sending: %r", self.response.as_string())
self.wfile.write(self.response.as_string().encode("utf8"))
def _really_handle(self):
"""handle() without the exception handling."""
self.server.log.debug("Received: %r", self.packet)
# Read the request.
# Old versions of the client sent a double \n after the signature,
# which screws up the RFC5321 format. Specifically handle that
# here - this could be removed in time.
request = email.message_from_bytes(
self.rfile.read().replace(b"\n\n", b"\n") + b"\n")
# Ensure that the response can be paired with the request.
self.response["Thread"] = request["Thread"]
# If this is an authenticated request, then check the authentication
# details.
user = request["User"] or pyzor.anonymous_user
if user != pyzor.anonymous_user:
try:
pyzor.account.verify_signature(request,
self.server.accounts[user])
except KeyError:
raise pyzor.SignatureError("Unknown user.")
if "PV" not in request:
raise pyzor.ProtocolError("Protocol Version not specified in request")
# The protocol version is compatible if the major number is
# identical (changes in the minor number are unimportant).
if int(float(request["PV"])) != int(pyzor.proto_version):
raise pyzor.UnsupportedVersionError()
# Check that the user has permission to execute the requested
# operation.
opcode = request["Op"]
if opcode not in self.server.acl[user]:
raise pyzor.AuthorizationError(
"User is not authorized to request the operation.")
self.server.log.debug("Got a %s command from %s", opcode,
self.client_address[0])
# Get a handle to the appropriate method to execute this operation.
try:
dispatch = self.dispatches[opcode]
except KeyError:
raise NotImplementedError("Requested operation is not "
"implemented.")
# Get the existing record from the database (or a blank one if
# there is no matching record).
digest = request["Op-Digest"]
# Do the requested operation, log what we have done, and return.
if dispatch:
try:
record = self.server.database[digest]
except KeyError:
record = pyzor.engines.common.Record()
dispatch(self, digest, record)
self.server.usage_log.info("%s,%s,%s,%r,%s", user,
self.client_address[0], opcode, digest,
self.response["Code"])
def handle_error(self, code, message):
"""Create an appropriate response for an error."""
self.server.usage_log.error("%s: %s", code, message)
self.response.replace_header("Code", "%d" % code)
self.response.replace_header("Diag", message)
def handle_pong(self, digest, _):
"""Handle the 'pong' command.
This command returns maxint for report counts and 0 whitelist.
"""
self.server.log.debug("Request pong for %s", digest)
self.response["Count"] = "%d" % sys.maxint
self.response["WL-Count"] = "%d" % 0
def handle_check(self, digest, record):
"""Handle the 'check' command.
This command returns the spam/ham counts for the specified digest.
"""
self.server.log.debug("Request to check digest %s", digest)
self.response["Count"] = "%d" % record.r_count
self.response["WL-Count"] = "%d" % record.wl_count
def handle_report(self, digest, record):
"""Handle the 'report' command.
This command increases the spam count for the specified digest."""
self.server.log.debug("Request to report digest %s", digest)
# Increase the count, and store the altered record back in the
# database.
record.r_increment()
self.server.database[digest] = record
def handle_whitelist(self, digest, record):
"""Handle the 'whitelist' command.
This command increases the ham count for the specified digest."""
self.server.log.debug("Request to whitelist digest %s", digest)
# Increase the count, and store the altered record back in the
# database.
record.wl_increment()
self.server.database[digest] = record
def handle_info(self, digest, record):
"""Handle the 'info' command.
This command returns diagnostic data about a digest (timestamps for
when the digest was first/last seen as spam/ham, and spam/ham
counts).
"""
self.server.log.debug("Request for information about digest %s",
digest)
def time_output(time_obj):
"""Convert a datetime object to a POSIX timestamp.
If the object is None, then return 0.
"""
if not time_obj:
return 0
return time.mktime(time_obj.timetuple())
self.response["Entered"] = "%d" % time_output(record.r_entered)
self.response["Updated"] = "%d" % time_output(record.r_updated)
self.response["WL-Entered"] = "%d" % time_output(record.wl_entered)
self.response["WL-Updated"] = "%d" % time_output(record.wl_updated)
self.response["Count"] = "%d" % record.r_count
self.response["WL-Count"] = "%d" % record.wl_count
dispatches = {
'ping' : None,
'pong' : handle_pong,
'info' : handle_info,
'check' : handle_check,
'report' : handle_report,
'whitelist' : handle_whitelist,
}

View File

@@ -0,0 +1,10 @@
# Depending on what engine type you want to use
# you will need one of the following
MySQL-python==1.2.4
redis==2.9.1
# python-gdbm # not available via pip
# If you want to use gevent you will also require
# this.
gevent==1.0.1

View File

@@ -0,0 +1,3 @@
pyzord.db
pyzord.log
pyzord.pid

View File

@@ -0,0 +1,309 @@
#! /usr/bin/env python
"""Pyzor client."""
import os
import sys
import email
import random
import mailbox
import hashlib
import getpass
import logging
import optparse
import tempfile
import ConfigParser
import pyzor.digest
import pyzor.client
import pyzor.config
def load_configuration():
"""Load the configuration for the server.
The configuration comes from three sources: the default values, the
configuration file, and command-line options."""
# Work out the default directory for configuration files.
# If $HOME is defined, then use $HOME/.pyzor, otherwise use /etc/pyzor.
userhome = os.getenv("HOME")
if userhome:
homedir = os.path.join(userhome, '.pyzor')
else:
homedir = os.path.join("/etc", "pyzor")
# Configuration defaults. The configuration file overrides these, and
# then the command-line options override those.
defaults = {
"ServersFile" : "servers",
"AccountsFile" : "accounts",
"LogFile" : "",
"Timeout" : "5", # seconds
"Style" : "msg",
"ReportThreshold" : "0",
"WhitelistThreshold" : "0"
}
# Process any command line options.
description = ("Read data from stdin and execute the requested command "
"(one of 'check', 'report', 'ping', 'pong', 'digest', "
"'predigest', 'genkey').")
opt = optparse.OptionParser(description=description)
opt.add_option("-n", "--nice", dest="nice", type="int",
help="'nice' level", default=0)
opt.add_option("-d", "--debug", action="store_true", default=False,
dest="debug", help="enable debugging output")
opt.add_option("--homedir", action="store", default=homedir,
dest="homedir", help="configuration directory")
opt.add_option("-s", "--style", action="store",
dest="Style", default=None,
help="input style: 'msg' (individual RFC5321 message), "
"'mbox' (mbox file of messages), 'digests' (Pyzor "
"digests, one per line).")
opt.add_option("--log-file", action="store", default=None,
dest="LogFile", help="name of log file")
opt.add_option("--servers-file", action="store", default=None,
dest="ServersFile", help="name of servers file")
opt.add_option("--accounts-file", action="store", default=None,
dest="AccountsFile", help="name of accounts file")
opt.add_option("-t", "--timeout", dest="Timeout", type="int",
help="timeout (in seconds)", default=None)
opt.add_option("-r", "--report-threshold", dest="ReportThreshold",
type="int", default=None,
help="threshold for number of reports")
opt.add_option("-w", "--whitelist-threshold", dest="WhitelistThreshold",
type="int", default=None,
help="threshold for number of whitelist")
opt.add_option("-V", "--version", action="store_true", default=False,
dest="version", help="print version and exit")
options, args = opt.parse_args()
if options.version:
print "%s %s" % (sys.argv[0], pyzor.__version__)
sys.exit(0)
if not len(args):
opt.print_help()
sys.exit()
os.nice(options.nice)
# Create the configuration directory if it doesn't already exist.
if not os.path.exists(options.homedir):
os.mkdir(options.homedir)
# Load the configuration.
config = ConfigParser.ConfigParser()
# Set the defaults.
config.add_section("client")
for key, value in defaults.iteritems():
config.set("client", key, value)
# Override with the configuration.
config.read(os.path.join(options.homedir, "config"))
# Override with the command-line options.
for key in defaults:
value = getattr(options, key)
if value is not None:
config.set("client", key, str(value))
return config, options, args
def main():
"""Execute any requested actions."""
# Set umask - this restricts this process from granting any world access
# to files/directories created by this process.
os.umask(0077)
config, options, args = load_configuration()
homefiles = ["LogFile", "ServersFile", "AccountsFile"]
pyzor.config.expand_homefiles(homefiles, "client", options.homedir, config)
logger = pyzor.config.setup_logging("pyzor",
config.get("client", "LogFile"),
options.debug)
servers = pyzor.config.load_servers(config.get("client", "ServersFile"))
accounts = pyzor.config.load_accounts(config.get("client", "AccountsFile"))
# Run the specified commands.
client = pyzor.client.Client(accounts,
int(config.get("client", "Timeout")))
for command in args:
try:
dispatch = DISPATCHES[command]
except KeyError:
logger.error("Unknown command: %s", command)
else:
try:
if not dispatch(client, servers, config):
sys.exit(1)
except pyzor.TimeoutError:
# Note that most of the methods will trap their own timeout
# error.
logger.error("Timeout from server in %s", command)
def get_input_handler(style="msg", digester=pyzor.digest.DataDigester):
"""Return an object that can be iterated over to get all the digests."""
if style not in ("msg", "mbox", "digests"):
raise ValueError("Unknown input style.")
if style == "digests":
for line in sys.stdin:
yield line.strip()
return
if style == "msg":
tfile = None
msg = email.message_from_file(sys.stdin)
mbox = [msg]
elif style == 'mbox':
# We have to write the mbox to disk in order to use mailbox to work
# with it.
tfile = tempfile.NamedTemporaryFile()
tfile.write(sys.stdin.read().encode("utf8"))
tfile.seek(0)
mbox = mailbox.mbox(tfile.name)
for msg in mbox:
digested = digester(msg).value
if digested:
yield digested
if tfile:
tfile.close()
def ping(client, servers, config):
"""Check that the server is reachable."""
# pylint: disable-msg=W0613
runner = pyzor.client.ClientRunner(client.ping)
for server in servers:
runner.run(server, (server,))
return runner.all_ok
def pong(client, servers, config):
"""Used to test pyzor."""
rt = int(config.get("client", "ReportThreshold"))
wt = int(config.get("client", "WhitelistThreshold"))
style = config.get("client", "Style")
runner = pyzor.client.CheckClientRunner(client.pong, rt, wt)
for digested in get_input_handler(style):
if digested:
for server in servers:
runner.run(server, (digested, server))
return runner.all_ok and runner.found_hit and not runner.whitelisted
def info(client, servers, config):
"""Get information about each message."""
style = config.get("client", "Style")
runner = pyzor.client.InfoClientRunner(client.info)
for digested in get_input_handler(style):
if digested:
for server in servers:
runner.run(server, (digested, server))
return runner.all_ok
def check(client, servers, config):
"""Check each message against each server.
The return value is 'failure' if there is a positive spam count and
*zero* whitelisted count; otherwise 'success'.
"""
rt = int(config.get("client", "ReportThreshold"))
wt = int(config.get("client", "WhitelistThreshold"))
style = config.get("client", "Style")
runner = pyzor.client.CheckClientRunner(client.check, rt, wt)
for digested in get_input_handler(style):
if digested:
for server in servers:
runner.run(server, (digested, server))
return runner.all_ok and runner.found_hit and not runner.whitelisted
def send_digest(digested, spec, client_method, servers):
"""Send these digests to each server."""
# Digest can be None; if so, nothing is sent.
if not digested:
return
runner = pyzor.client.ClientRunner(client_method)
for server in servers:
runner.run(server, (digested, server, spec))
return runner.all_ok
def report(client, servers, config):
"""Report each message as spam."""
style = config.get("client", "Style")
all_ok = True
for digested in get_input_handler(style):
if digested and not send_digest(digested, pyzor.digest.digest_spec,
client.report, servers):
all_ok = False
return all_ok
def whitelist(client, servers, config):
"""Report each message as ham."""
style = config.get("client", "Style")
all_ok = True
for digested in get_input_handler(style):
if digested and not send_digest(digested, pyzor.digest.digest_spec,
client.whitelist, servers):
all_ok = False
return all_ok
def digest(client, servers, config):
"""Generate a digest for each message.
This method can be used to look up digests in the database when
diagnosing, or to report digests in a two-stage operation (digest,
then report with --digests)."""
style = config.get("client", "Style")
for digested in get_input_handler(style):
if digested:
print digested
return True
def predigest(client, servers, config):
"""Output the normalised version of each message, which is used to
create the digest.
This method can be used to diagnose which parts of the message are
used to determine uniqueness."""
for unused in get_input_handler(
"msg", digester=pyzor.digest.PrintingDataDigester):
pass
return True
def genkey(client, servers, config, hash_func=hashlib.sha1):
"""Generate a key to use to authenticate pyzor requests. This method
will prompt for a password (and confirmation).
A random salt is generated (which makes it extremely difficult to
reverse the generated key to get the original password) and combined
with the entered password to provide a key. This key (but not the salt)
should be provided to the pyzord administrator, along with a username.
"""
# pylint: disable-msg=W0613
password = getpass.getpass(prompt="Enter passphrase: ")
if getpass.getpass(prompt="Enter passphrase again: ") != password:
log = logging.getLogger("pyzor")
log.error("Passwords do not match.")
return False
# pylint: disable-msg=W0612
salt = "".join([chr(random.randint(0, 255))
for unused in xrange(hash_func(b"").digest_size)])
if sys.version_info >= (3, 0):
salt = salt.encode("utf8")
salt_digest = hash_func(salt)
pass_digest = hash_func(salt_digest.digest())
pass_digest.update(password.encode("utf8"))
print "salt,key:"
print "%s,%s" % (salt_digest.hexdigest(), pass_digest.hexdigest())
return True
DISPATCHES = {
"ping" : ping,
"pong" : pong,
"info" : info,
"check" : check,
"report" : report,
"whitelist" : whitelist,
"digest" : digest,
"predigest" : predigest,
"genkey" : genkey,
}
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,312 @@
#! /usr/bin/env python
"""A front-end interface to the pyzor daemon."""
import os
import sys
import optparse
import traceback
import ConfigParser
import pyzor.config
import pyzor.server
import pyzor.engines
def detach(stdout="/dev/null", stderr=None, stdin="/dev/null", pidfile=None):
"""This forks the current process into a daemon.
The stdin, stdout, and stderr arguments are file names that
will be opened and be used to replace the standard file descriptors
in sys.stdin, sys.stdout, and sys.stderr.
These arguments are optional and default to /dev/null.
Note that stderr is opened unbuffered, so if it shares a file with
stdout then interleaved output may not appear in the order that you
expect."""
# Do first fork.
try:
pid = os.fork()
if pid > 0:
# Exit first parent.
sys.exit(0)
except OSError as err:
print >> sys.stderr, "Fork #1 failed: (%d) %s" % \
(err.errno, err.strerror)
sys.exit(1)
# Decouple from parent environment.
os.chdir("/")
os.umask(0)
os.setsid()
# Do second fork.
try:
pid = os.fork()
if pid > 0:
# Exit second parent.
sys.exit(0)
except OSError as err:
print >> sys.stderr, "Fork #2 failed: (%d) %s" % \
(err.errno, err.strerror)
sys.exit(1)
# Open file descriptors and print start message.
if not stderr:
stderr = stdout
stdi = open(stdin, "r")
stdo = open(stdout, "a+")
stde = open(stderr, "a+", 0)
pid = str(os.getpid())
if pidfile:
open(pidfile, "w+").write("%s\n" % pid)
# Redirect standard file descriptors.
os.dup2(stdi.fileno(), sys.stdin.fileno())
os.dup2(stdo.fileno(), sys.stdout.fileno())
os.dup2(stde.fileno(), sys.stderr.fileno())
def load_configuration():
"""Load the configuration for the server.
The configuration comes from three sources: the default values, the
configuration file, and command-line options."""
# Work out the default directory for configuration files.
# If $HOME is defined, then use $HOME/.pyzor, otherwise use /etc/pyzor.
userhome = os.getenv("HOME")
if userhome:
homedir = os.path.join(userhome, '.pyzor')
else:
homedir = os.path.join("/etc", "pyzor")
# Configuration defaults. The configuration file overrides these, and
# then the command-line options override those.
defaults = {
"Port" : "24441",
"ListenAddress" : "0.0.0.0",
"Engine" : "gdbm",
"DigestDB" : "pyzord.db",
"CleanupAge" : str(60 * 60 * 24 * 30 * 4), # approximately 4 months
"Threads": "False",
"MaxThreads": "0",
"Processes": "False",
"MaxProcesses": "40",
"DBConnections": "0",
"Gevent": "False",
"PasswdFile" : "pyzord.passwd",
"AccessFile" : "pyzord.access",
"LogFile" : "",
"UsageLogFile": "",
"PidFile": "pyzord.pid"
}
# Process any command line options.
description = "Listen for and process incoming Pyzor connections."
opt = optparse.OptionParser(description=description)
opt.add_option("-n", "--nice", dest="nice", type="int",
help="'nice' level", default=0)
opt.add_option("-d", "--debug", action="store_true", default=False,
dest="debug", help="enable debugging output")
opt.add_option("--homedir", action="store", default=homedir,
dest="homedir", help="configuration directory")
opt.add_option("-a", "--address", action="store", default=None,
dest="ListenAddress", help="listen on this IP")
opt.add_option("-p", "--port", action="store", type="int", default=None,
dest="Port", help="listen on this port")
opt.add_option("-e", "--database-engine", action="store", default=None,
dest="Engine", help="select database backend")
opt.add_option("--dsn", action="store", default=None, dest="DigestDB",
help="data source name (filename for gdbm, host,user,"
"password,database,table for MySQL)")
opt.add_option("--gevent", action="store", default=None, dest="Gevent",
help="set to true to use the gevent library")
opt.add_option("--threads", action="store", default=None, dest="Threads",
help="set to true if multi-threading should be used"
" (this may not apply to all engines)")
opt.add_option("--max-threads", action="store", default=None, type="int",
dest="MaxThreads", help="the maximum number of concurrent "
"threads (defaults to 0 which is unlimited)")
opt.add_option("--processes", action="store", default=None,
dest="Processes", help="set to true if multi-processing "
"should be used (this may not apply to all engines)")
opt.add_option("--max-processes", action="store", default=None, type="int",
dest="MaxProcesses", help="the maximum number of concurrent "
"processes (defaults to 40)")
opt.add_option("--db-connections", action="store", default=None, type="int",
dest="DBConnections", help="the number of db connections "
"that will be kept by the server. This only applies if "
"threads are used. Defaults to 0 which means a new "
"connection is used for every thread. (this may not apply "
"all engines)")
opt.add_option("--password-file", action="store", default=None,
dest="PasswdFile", help="name of password file")
opt.add_option("--access-file", action="store", default=None,
dest="AccessFile", help="name of ACL file")
opt.add_option("--cleanup-age", action="store", default=None,
dest="CleanupAge",
help="time before digests expire (in seconds)")
opt.add_option("--log-file", action="store", default=None,
dest="LogFile", help="name of the log file")
opt.add_option("--usage-log-file", action="store", default=None,
dest="UsageLogFile", help="name of the usage log file")
opt.add_option("--pid-file", action="store", default=None,
dest="PidFile", help="save the pid in this file after the "
"server is daemonized")
opt.add_option("--detach", action="store", default=None,
dest="detach", help="daemonizes the server and redirects "
"any output to the specified file")
opt.add_option("-V", "--version", action="store_true", default=False,
dest="version", help="print version and exit")
options, args = opt.parse_args()
if options.version:
print "%s %s" % (sys.argv[0], pyzor.__version__)
sys.exit(0)
if len(args):
opt.print_help()
sys.exit()
os.nice(options.nice)
# Create the configuration directory if it doesn't already exist.
if not os.path.exists(options.homedir):
os.mkdir(options.homedir)
# Load the configuration.
config = ConfigParser.ConfigParser()
# Set the defaults.
config.add_section("server")
for key, value in defaults.iteritems():
config.set("server", key, value)
# Override with the configuration.
config.read(os.path.join(options.homedir, "config"))
# Override with the command-line options.
for key in defaults:
value = getattr(options, key)
if value is not None:
config.set("server", key, str(value))
return config, options
def main():
"""Run the pyzor daemon."""
# Set umask - this restricts this process from granting any world access
# to files/directories created by this process.
os.umask(0077)
config, options = load_configuration()
homefiles = ["LogFile", "UsageLogFile", "PasswdFile", "AccessFile",
"PidFile"]
engine = config.get("server", "Engine")
database_classes = pyzor.engines.database_classes[engine]
use_gevent = config.get("server", "Gevent").lower() == "true"
use_threads = config.get("server", "Threads").lower() == "true"
use_processes = config.get("server", "Processes").lower() == "true"
if use_threads and use_processes:
print "You cannot use both processes and threads at the same time"
sys.exit(1)
# We prefer to use the threaded server, but some database engines
# cannot handle it.
if use_threads and database_classes.multi_threaded:
use_processes = False
database_class = database_classes.multi_threaded
elif use_processes and database_classes.multi_processing:
use_threads = False
database_class = database_classes.multi_processing
else:
use_threads = False
use_processes = False
database_class = database_classes.single_threaded
# If the DSN is a filename, then we make it absolute.
if database_class.absolute_source:
homefiles.append("DigestDB")
pyzor.config.expand_homefiles(homefiles, "server", options.homedir, config)
logger = pyzor.config.setup_logging("pyzord",
config.get("server", "LogFile"),
options.debug)
pyzor.config.setup_logging("pyzord-usage",
config.get("server", "UsageLogFile"),
options.debug)
db_file = config.get("server", "DigestDB")
passwd_fn = config.get("server", "PasswdFile")
access_fn = config.get("server", "AccessFile")
pidfile_fn = config.get("server", "PidFile")
address = (config.get("server", "ListenAddress"),
int(config.get("server", "port")))
cleanup_age = int(config.get("server", "CleanupAge"))
if use_gevent:
# Monkey patch the std libraries with gevent ones
try:
import signal
import gevent
import gevent.monkey
except ImportError as e:
logger.critical("Gevent library not found: %s", e)
sys.exit(1)
gevent.monkey.patch_all()
# The signal method does not get patched in patch_all
signal.signal = gevent.signal
# XXX The gevent libary might already be doing this.
# Enssure that all modules are reloaded so they benefit from
# the gevent library.
for module in (os, sys, pyzor, pyzor.server, pyzor.engines):
reload(module)
if options.detach:
detach(stdout=options.detach, pidfile=pidfile_fn)
if use_threads:
max_threads = int(config.get("server", "MaxThreads"))
bound = int(config.get("server", "DBConnections"))
database = database_class(db_file, "c", cleanup_age, bound)
if max_threads == 0:
logger.info("Starting multi-threaded pyzord server.")
server = pyzor.server.ThreadingServer(address, database, passwd_fn,
access_fn)
else:
logger.info("Starting bounded (%s) multi-threaded pyzord server.",
max_threads)
server = pyzor.server.BoundedThreadingServer(address, database,
passwd_fn, access_fn,
max_threads)
elif use_processes:
max_children = int(config.get("server", "MaxProcesses"))
database = database_class(db_file, "c", cleanup_age)
logger.info("Starting bounded (%s) multi-processing pyzord server.",
max_children)
server = pyzor.server.ProcessServer(address, database, passwd_fn, access_fn,
max_children)
else:
database = database_class(db_file, "c", cleanup_age)
logger.info("Starting pyzord server.")
server = pyzor.server.Server(address, database, passwd_fn, access_fn)
try:
server.serve_forever()
except:
logger.critical("Failure: %s", traceback.format_exc())
finally:
logger.info("Server shutdown.")
server.server_close()
if options.detach and os.path.exists(pidfile_fn):
try:
os.remove(pidfile_fn)
except Exception as e:
logger.warning("Unable to remove pidfile %r: %s",
pidfile_fn, e)
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,59 @@
import sys
import setuptools
import distutils.core
import pyzor
try:
# These automatically run 2to3 and modules and scripts while installing,
# when ran under Python 3
from distutils.command.build_py import build_py_2to3 as build_py
from distutils.command.build_scripts import build_scripts_2to3 as build_scripts
except ImportError:
from distutils.command.build_py import build_py
from distutils.command.build_scripts import build_scripts
long_description = """
Pyzor is spam-blocking networked system that uses spam signatures
to identify them.
"""
classifiers = ["Operating System :: POSIX",
"Environment :: Console",
"Environment :: No Input/Output (Daemon)",
"Programming Language :: Python"
"Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 3",
"Intended Audience :: System Administrators",
"Topic :: Communications :: Email",
"Topic :: Communications :: Email :: Filters",
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
]
distutils.core.setup(name='pyzor',
version=pyzor.__version__,
description='networked spam-signature detection',
long_description=long_description,
author='Frank J. Tobin',
author_email='ftobin@neverending.org',
license='GPL',
platforms='POSIX',
keywords='spam',
url='http://pyzor.sourceforge.net/',
scripts=['scripts/pyzor', 'scripts/pyzord'],
packages=['pyzor',
'pyzor.engines',
'pyzor.hacks'],
classifiers=classifiers,
test_suite="tests.suite",
cmdclass={'build_py': build_py,
'build_scripts': build_scripts,
},
)

View File

@@ -0,0 +1,18 @@
"""Package reserved for tests and test utilities."""
import unittest
import unit
import functional
def suite():
"""Gather all the tests from this package in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unit.suite())
test_suite.addTest(functional.suite())
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest='suite')

View File

@@ -0,0 +1,33 @@
"""A suite of functional tests that verifies the correct behaviour of the
pyzor client and server as a whole.
Functional test should not touch real data and are usually safe, but it's not
recommended to run theses on production servers.
Note these tests the installed version of pyzor, not the version from the
source.
"""
import unittest
import test_gdbm
import test_pyzor
import test_mysql
import test_redis
import test_digest
import test_account
def suite():
"""Gather all the tests from this package in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(test_gdbm.suite())
test_suite.addTest(test_mysql.suite())
test_suite.addTest(test_redis.suite())
test_suite.addTest(test_pyzor.suite())
test_suite.addTest(test_digest.suite())
test_suite.addTest(test_account.suite())
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest='suite')

View File

@@ -0,0 +1,138 @@
import unittest
from tests.util import *
class AccountPyzorTest(PyzorTestBase):
# test bob which has access to everything
def test_ping(self):
self.check_pyzor("ping", "bob", code=200, exit_code=0)
def test_pong(self):
self.check_pyzor("pong", "bob", input=msg, code=200, exit_code=0)
def test_check(self):
self.check_pyzor("check", "bob", input=msg, code=200)
def test_report(self):
self.check_pyzor("report", "bob", input=msg, code=200, exit_code=0)
def test_whitelist(self):
self.check_pyzor("whitelist", "bob", input=msg, code=200, exit_code=0)
def test_info(self):
self.check_pyzor("info", "bob", input=msg, code=200, exit_code=0)
# test alice which does not has access to anything
# Error should be 403 Forbidden
def test_ping_forbidden(self):
self.check_pyzor("ping", "alice", code=403, exit_code=1)
def test_pong_forbidden(self):
self.check_pyzor("pong", "alice", input=msg, code=403, exit_code=1)
def test_check_forbidden(self):
self.check_pyzor("check", "alice", input=msg, code=403, exit_code=1)
def test_report_forbidden(self):
self.check_pyzor("report", "alice", input=msg, code=403, exit_code=1)
def test_whitelist_forbidden(self):
self.check_pyzor("whitelist", "alice", input=msg, code=403, exit_code=1)
def test_info_forbidden(self):
self.check_pyzor("info", "alice", input=msg, code=403, exit_code=1)
# test chuck which does tries to steal bob's account but has the wrong key
# Error should be 401 Unauthorized
def test_ping_unauthorized(self):
self.check_pyzor("ping", "chuck", code=401, exit_code=1)
def test_pong_unauthorized(self):
self.check_pyzor("pong", "chuck", input=msg, code=401, exit_code=1)
def test_check_unauthorized(self):
self.check_pyzor("check", "chuck", input=msg, code=401, exit_code=1)
def test_report_unauthorized(self):
self.check_pyzor("report", "chuck", input=msg, code=401, exit_code=1)
def test_whitelist_unauthorized(self):
self.check_pyzor("whitelist", "chuck", input=msg, code=401, exit_code=1)
def test_info_unauthorized(self):
self.check_pyzor("info", "chuck", input=msg, code=401, exit_code=1)
# test dan account, which has some access
def test_ping_combo(self):
self.check_pyzor("ping", "dan", code=200, exit_code=0)
def test_pong_combo(self):
self.check_pyzor("pong", "dan", input=msg, code=403, exit_code=1)
def test_check_combo(self):
self.check_pyzor("check", "dan", input=msg, code=200)
def test_report_combo(self):
self.check_pyzor("report", "dan", input=msg, code=200, exit_code=0)
def test_whitelist_combo(self):
self.check_pyzor("whitelist", "dan", input=msg, code=403, exit_code=1)
def test_info_combo(self):
self.check_pyzor("info", "dan", input=msg, code=403, exit_code=1)
# test anonymous account, which should is not currently set up in the server
def test_ping_anonymous(self):
self.check_pyzor("ping", None, code=403, exit_code=1)
def test_pong_anonymous(self):
self.check_pyzor("pong", None, input=msg, code=403, exit_code=1)
def test_check_anonymous(self):
self.check_pyzor("check", None, input=msg, code=403, exit_code=1)
def test_report_anonymous(self):
self.check_pyzor("report", None, input=msg, code=403, exit_code=1)
def test_whitelist_anonymous(self):
self.check_pyzor("whitelist", None, input=msg, code=403, exit_code=1)
def test_info_anonymous(self):
self.check_pyzor("info", None, input=msg, code=403, exit_code=1)
class AnonymousPyzorTest(PyzorTestBase):
"""Test accounts with no access or password file set-up. And test
anonymous default access.
"""
access_file = None
password_file = None
def test_ping(self):
self.check_pyzor("ping", None, code=200, exit_code=0)
def test_pong(self):
self.check_pyzor("pong", None, input=msg, code=200, exit_code=0)
def test_check(self):
self.check_pyzor("check", None, input=msg, code=200)
def test_report(self):
self.check_pyzor("report", None, input=msg, code=200, exit_code=0)
def test_whitelist(self):
# anonymous account are not allowed to whitelist by default
self.check_pyzor("whitelist", None, input=msg, code=403, exit_code=1)
def test_info(self):
self.check_pyzor("info", None, input=msg, code=200, exit_code=0)
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(AccountPyzorTest))
test_suite.addTest(unittest.makeSuite(AnonymousPyzorTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,843 @@
# -*- coding: utf-8 -*-
import sys
import hashlib
import unittest
from tests.util import *
TEXT = """MIME-Version: 1.0
Sender: chirila@spamexperts.com
Received: by 10.216.90.129 with HTTP; Fri, 23 Aug 2013 01:59:03 -0700 (PDT)
Date: Fri, 23 Aug 2013 11:59:03 +0300
Delivered-To: chirila@spamexperts.com
X-Google-Sender-Auth: p6ay4c-tEtdFpavndA9KBmP0CVs
Message-ID: <CAK-mJS9aV6Kb7Z5XCRJ_z_UOKEaQjRY8gMzsuxUQcN5iqxNWUg@mail.gmail.com>
Subject: Test
From: Alexandru Chirila <chirila@spamexperts.com>
To: Alexandru Chirila <chirila@spamexperts.com>
Content-Type: multipart/alternative; boundary=001a11c2893246a9e604e4999ea3
--001a11c2893246a9e604e4999ea3
Content-Type: text/plain; charset=ISO-8859-1
%s
--001a11c2893246a9e604e4999ea3
"""
HTML_TEXT = """MIME-Version: 1.0
Sender: chirila@gapps.spamexperts.com
Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST)
Date: Thu, 16 Jan 2014 10:43:31 +0200
Delivered-To: chirila@gapps.spamexperts.com
X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q
Message-ID: <CAK-mJS8sE-V6qtspzzZ+bZ1eSUE_FNMt3K-5kBOG-z3NMgU_Rg@mail.gmail.com>
Subject: Test
From: Alexandru Chirila <chirila@spamexperts.com>
To: Alexandru Chirila <chirila@gapps.spamexperts.com>
Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd
--001a11c25ff293069304f0126bfd
Content-Type: text/plain; charset=ISO-8859-1
Email spam.
Email spam, also known as junk email or unsolicited bulk email, is a subset
of electronic spam involving nearly identical messages sent to numerous
recipients by email. Clicking on links in spam email may send users to
phishing web sites or sites that are hosting malware.
--001a11c25ff293069304f0126bfd
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable
<div dir=3D"ltr"><div>Email spam.</div><div><br></div><div>Email spam, also=
known as junk email or unsolicited bulk email, is a subset of electronic s=
pam involving nearly identical messages sent to numerous recipients by emai=
l. Clicking on links in spam email may send users to phishing web sites or =
sites that are hosting malware.</div>
</div>
--001a11c25ff293069304f0126bfd--
"""
TEXT_ATTACHMENT = """MIME-Version: 1.0
Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
Date: Fri, 17 Jan 2014 12:21:43 +0200
Delivered-To: chirila.s.alexandru@gmail.com
Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
Subject: Test
From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
--f46d040a62c49bb1c804f027e8cc
Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
--f46d040a62c49bb1c404f027e8ca
Content-Type: text/plain; charset=ISO-8859-1
This is a test mailing
--f46d040a62c49bb1c404f027e8ca--
--f46d040a62c49bb1c804f027e8cc
Content-Type: image/png; name="tar.png"
Content-Disposition: attachment; filename="tar.png"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_hqjas5ad0
iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD
GGlDQ1BQaG90b3Nob3AgSUNDIHByb2ZpbGUAAHjaY2BgnuDo4uTKJMDAUFBUUuQe5BgZERmlwH6e
gY2BmYGBgYGBITG5uMAxIMCHgYGBIS8/L5UBFTAyMHy7xsDIwMDAcFnX0cXJlYE0wJpcUFTCwMBw
gIGBwSgltTiZgYHhCwMDQ3p5SUEJAwNjDAMDg0hSdkEJAwNjAQMDg0h2SJAzAwNjCwMDE09JakUJ
AwMDg3N+QWVRZnpGiYKhpaWlgmNKflKqQnBlcUlqbrGCZ15yflFBflFiSWoKAwMD1A4GBgYGXpf8
EgX3xMw8BSMDVQYqg4jIKAUICxE+CDEESC4tKoMHJQODAIMCgwGDA0MAQyJDPcMChqMMbxjFGV0Y
SxlXMN5jEmMKYprAdIFZmDmSeSHzGxZLlg6WW6x6rK2s99gs2aaxfWMPZ9/NocTRxfGFM5HzApcj
1xZuTe4FPFI8U3mFeCfxCfNN45fhXyygI7BD0FXwilCq0A/hXhEVkb2i4aJfxCaJG4lfkaiQlJM8
JpUvLS19QqZMVl32llyfvIv8H4WtioVKekpvldeqFKiaqP5UO6jepRGqqaT5QeuA9iSdVF0rPUG9
V/pHDBYY1hrFGNuayJsym740u2C+02KJ5QSrOutcmzjbQDtXe2sHY0cdJzVnJRcFV3k3BXdlD3VP
XS8Tbxsfd99gvwT//ID6wIlBS4N3hVwMfRnOFCEXaRUVEV0RMzN2T9yDBLZE3aSw5IaUNak30zky
LDIzs+ZmX8xlz7PPryjYVPiuWLskq3RV2ZsK/cqSql01jLVedVPrHzbqNdU0n22VaytsP9op3VXU
fbpXta+x/+5Em0mzJ/+dGj/t8AyNmf2zvs9JmHt6vvmCpYtEFrcu+bYsc/m9lSGrTq9xWbtvveWG
bZtMNm/ZarJt+w6rnft3u+45uy9s/4ODOYd+Hmk/Jn58xUnrU+fOJJ/9dX7SRe1LR68kXv13fc5N
m1t379TfU75/4mHeY7En+59lvhB5efB1/lv5dxc+NH0y/fzq64Lv4T8Ffp360/rP8f9/AA0ADzT6
lvFdAAAAIGNIUk0AAHolAACAgwAA+f8AAIDpAAB1MAAA6mAAADqYAAAXb5JfxUYAAGrVSURBVHja
7J13YE3nG8c/Nzc7BLFX7F01ajcENVrU3qpVtalds2oWNWtTe1Zqj1IzLaVG0VJq7y1myM79/v44
92ZI/FDNoHn/4Nxz3vOcc28+5z3P+7zPoBivT+sko9V6je65nvWeP32N7rm89Z77m16fe34b0tV8
XZqLl/UX9nyNqMhuveeiLq/N75w6hfWea9q/NvecCaip16V5vtYke742v3PFCJJTvDb33DyJ5CSS
k0hORCSn/PDpPXaZ8nokkZxE8mtCcrpKn45Ze+yeJJWNSk+1dmseSjr1uW3G4tX5k7RRO6RqPaRv
FfuIj3mbZk8iOYnkBCP5o5uSpCcXjh7dN8otCjzOku78uHRnkL6iaxko+4eknYC5Qds8QJWldyXp
WM6Ut1eZgU9CdTzy5DKdxyzZuGzmrA+hWiHK9Ju6YtPBP5aXsB6uNXjh4iVfN47xVijewBGwy+Dp
mkRyEskvSXKFKZP7t6mrqUCZbXcO1HCNJPlgLoBs14PzaDSZ74Uv/qRrObBbKoV8BD5aX7tw7e+1
30saB++HXT8vZ+PUCtuDbFdbiJvW5QiWJP8wzbLK3mscDKmKqUbnombodaMkeG6RRpuqrrkjhf/e
PInkJJJfXk9OoflQIzjotPRwZgpjn4N2GBtD1EU+DFJf42MH/dnhYWBu1qoQwDb1ljQ+z92Qsnts
JPd+fPwHrcqeMnv2zA6gvdk1J4+HG3jZxvwzux1T5O4ZoKHmxZK2ubBXp1yd/9Yhbamvez9Omb41
XF2SSE4i+aVJzq7vyXrvbvF0jebe1NFkhvqstcbBpvoqfC9LVZDcH6SF46F5aa8ZjFAFsnecd1Ff
aM453dTnnAqxybMnh+bTcrsT4KjddZ5e0Qj5gYprw7TLY7Sml9+mBeyWuvbUUg/tLqsVAO21O4nk
JJJfmuTimsV3agGQbJmaAVBU842D3dT10V766MTPUnD1KvIBN/875r7qfE5S4IQK+jrfBY3AMeRc
pERvTWRTIEBabe+uKcOnZo08mEmTW0saayoRvsKE41+Wwr+HBf55KTiTo/bk0lJMVdeHW1olkZxE
8j8geYZ70EXDDPG5PjdMFxphHFwsL//fcFwi/TZZGyboxJX79wNVubsu6maPova8r8G45IA82hQp
sYEGcOVvnEq2maoVAyQpyoJ+RQ0o/KekcdtDMwKfatTNs4ukJWTStg80qfIx6a/3k/TkJJJfnuT8
+qGGJhrbPqps1YfbAZDsvp+LfIH875jwu3BIunns+GlN76bRAdpZDOqqFwA1NDlSYm996hZ+b88T
SZrVTz82zB+Fy5ZqC2k/PiWtByigjUH7Cwf6l6CwlrdWzyuyjHdOsl0kPMmWtc1yOznl+/i8JIVK
kq58UdQzp3evvVK49+hESHI2rRmu6gDkCr7kCMBY1QBguL7NpO+tHe9cDTntDrgGnO6l+nk2KrQZ
LdXZyme3SImTVC6vFHp4Rqe+Gt1PjaNxOVQfWl1sPgbIpZ+1nBxpoaYm9lfjTrekOz1MSSQnNMnT
wSlfHleS7dLdRuaShyW9Rfp8acHUTTdonAhJLqylK5QLIM1xtTb2rVYegKrBd9N76dt3CwIU1BWt
AmC/uqk11HryOHsrGRazFuofKXGO8qXXFjegokZ1iMo4MEclALpLtY3Rf56mA/CZ+g1XTey8vw1Q
+ySSE5rkAQwJk0LHUuVJUbyh2oUAB5ZL11Y5my2XqJsISS6qKUuVBUwNzmmhdTA8EeoI5h4BIVX4
RN1O/QQk36ZFGmfVQdrpS6CHhtpIbmhTrAFmqaynlhr69oQGGgEOmTLbVIb1ygFk8rNovhvJugc8
qaIZVh29zZdqCFBZ3yWRnNAkL2W6pIvV6NiftuETK5LhxnoG3v97bx2q6TAdEyHJxfRtb61rN+ys
QobY3uqPzuFS6y/5VYEv1Wh6eAHHcoe1tYzGALDSUk2LAI+wAy0MowfemhIpsYfaptM6Q3fYVkG/
LD8VKoXuywfArypcpdv82xpwVgHXpXsV8+trANqqVxu1BZL/qk+SSE5okr9nYphvV0eq3PFIfl9S
FwYe57OKQLEb+pk+iZDkIprvsk9S6KrIiZlfwIlQaWMGYIq8K0kh0mKnytbpXaYmTnemAYwbVtuq
YWe2XIh0NyqhOXYPtgJwOLi4pJBj69f8eiq7oZVLkgK/sss46cSVvSMzkvJSPQBKak9dDSmX5dPT
WulxsHUSyQlL8kaaZYMMMyx7DZ34Au8cpeOYLJB+nzYyPBGS7DykHHbvd6qTKsow2N8/5PRcbwC+
uZmPtod/X9rQhH2dbLYODjYXurbWpbtxDwtGnG1qk5eMRo/CTWjRtnikpxHmGUfXT2z3rktMRyHT
z2rb8RdJltkuH0Y1hSSRnDAkQ4v1wdK3TJSkMLPnUdpKFweaCmgbIxIhyc9p9uYX6mZK8+o+bxvz
UWD5yq8LkPyUvJNITmiSU5L+qwfSeFZJ0jGq7qCzJBXjzs0SPq8fyQnQPPZFuBslkZxQJG9geDMX
0qzVHIZLstRl8hoGSFIZbib83b4eJDe2zHFMIjmBSd7CQN2f4Gb+6aI540n9XYdSoQuYoNA/2lFU
D8aGvN4kN+oeLyTbpUha40twkn+jnaTf7DM+7os5GVS4q+9IW9AFMpxSZUYlQpI/XLVkZLWo5Dja
Q54Je87/Nb18dMIyPvS3i5i55Snf+JNSUQ5mqPnl9Jkdnklnnur2MXc2O7C5EoBjUsxIoiP5bxpI
UlcWWxYVyVttdpjkg8k1e52ZD6Rh5pmJj+TmFkkag0dZsAP40O9q6nmhYVf+uClNp9YYaF0AAJe9
GmCc8+7swwGS9MgRINXhud/9fF2SdAMgU3bnZvM2bVg0xCvK8stj7c1k3X7rLQB+nPKxRQprCyVD
PgTsK6YE76+m9i7rkERywpP8xFRTks64T4zYFTThqG0zMPHpyWnv+hXOVu7Hrxn8yH7PrlTwUZhU
R3fTASUPq95ykTrwURnAvEY/WO0YV+S/b/GYnt0HlQf4QJIlxH9401L5kmf5cMQey52Tkn+opDkR
lo1j2q4LVvPGXwcBclr+ehBStOKN0JL0UQPAW11cf5Qk3WqTRHKCk6x5Pxv4Js67jUlyIw0xNr6Q
5xMdz1It+KHlrsN62RsZi/rvt0CNsNvZYLx22LSAhSoCwC41B/qoSybXGZas9AiQJP3lo1+yQsYG
Fy0201xl7WGCFhkfVoS5A/10TLOgorawUsWAypqUSz8XSllqSoB6JJGc4CQn7haT5A5qa2x0VGP5
Wa49DGir5cxXFrBr/igk39VbQFcdcqtlORexitdHnwGwSvcywTDVhSGq5fDkypGtExtmwSPcFyDT
7Se2iKeZqofbtaD0AEzWu2A+F7ZXFcB0OtTtXIA9UE1TyhuedXmfHE4iOZGRfDXRk1xFJycdvXJ2
hGNrfar5TUJD6rVUd2ZoVOsx5/WwtosOAqYF2uwXWjqKK/3MblM72jFWWgQzVQm6qmUaTc74Vi4T
cOcUUO6s+tlOOPvEGXpYw/P6qTW8r1VXgxyB5SoT/hdAdY2qYT3jmF8SyYmL5JVudxM7yXwnhV3x
15yO6qUl5MpHN31mrE+GrvbkLS0BcPpNGhjJVi1JUh76yWIpwXQVgubqkEeSNBk4GTx/6XlZ/Y2A
FJZrk5f9ckErAaivYTBDLRTw593wx0H6TqsBmqt3LQ0GSBF6IInkxEXyZZYlepKpXSs56R486KyP
QvcDfKyBTNaYHnXTAy30JQDp74VFcZX4RL99Uak8dNHs8KPO01UUPtYnBXVu4/LN3YCTkkK31Yzo
X0GSFBB2ycjyoo3YXfF/Vwq8cPTYfR02+G2pjtU0FaCdRiaRnMi0i6z9Ez/JABxVbzW/cBmgrBbz
jaoY+4epvrGxTMUje3ezKtef6fMJmj1dReEzNc9udZmHw7qlsOWR12ithRWyubFLyQFMN54kq6Bl
dY25Zj9dNaJKmqtzSS0DMvo/Tp9EciIjuVb1xE5yjqwAyZ6c76jmu0IB3J7csusha3TzChvAtgAp
AAYbHvF8ou5Ox3ROxaGPGuQyvOuBo8FOH5/Sg4a2/q3UEeBblQNgguqPVwvr49BPfmoK0ExfZtMW
sN9mfQ8kkZyISO6XNbGTfOOgEzBQYzqq6XoBsFRVG9hsc9uV3v2jaSva0cUIuzPaN1ase6sJJZ9I
FaG7ambRGmwhJ+DY/XFIhYgxuYPhgd/K6om88UJgqsGqCTBKUgWAUlqeTL+Td5t22CeRnNhInmd6
nMhJHqdfmndZrlsebVVvdjAAdTQml29V4/AGrfaXdM++VtTA0rkqZ1/owxYdh2/PDU3D9QF8oQZO
utj/61GTF256Z588gFo6ZUWyur4wArBHGZ/3SasZojoAC3RRt+763bjd7GJIZv8Hq4O1LnnSGl+i
I/ligfuJnGS3jZL0ZyEyj8/oYey1/65OBEUfSbdHls2Yn3RfZYxka6aOGosgXgBf3csHDYLrcc2Q
HfbJCpU34lo7Gv2T+w0y5pITjM+VpDZ436gIUP7rWmcuXLhw6Ub5kl91lfR73SS/i0RIcmJrsc34
6g3oXObZ3vQVKzjE3NlF4We+H9qtYx/Dochs/cf1g5pVvMoWTkPuTi4A7kNtuYfM1lQCtpimnnNd
YrvYOzO6FkzyIEoi+VVsFy/ZUjj8o9MyJOVPfqrd27fx5yexHQg/fCA4MZBsORRNP1558uWEXvvx
mYf2DAxPBCQn5bT/N0gOnVAJIPXiyF1XN0/+bvkJ6UBOSDfbknAk+/1p/L+QL6L0uW1OfT066Gcl
6fQPP9yTFHjr9N41s8ZaHebOBkpqwQ5b16gPZuhhqR33k0h+Q0h+hFPVLv0+c2aeLnUsuVz6u7kd
gKn2dhe7mp+4MzDhSK5u5ytJ9zKZ9xxftewPaXx3i6Q+TLC9NGY/kDTBdER3qwH1FNrZyPNqhEgp
0HWYpA10ubo2UJKqpQ/b/0D+mQdK0hguqrRzcBLJb8qY7JZTklbgtT8FkO9wDz5Z5us7OzdvmxZJ
5z3sgxKM5HXUl6T2VMkO0OW+0/uStI1W1g4/4iNpCaPUEa9JK0/rJrnqftJ9vM/GW5KkX5gkKcCp
RH+2SLKkcb7E17pDDUnrTG9JTgX0xpDcKd9/XE/Ol14KX5OOKekdv+1fmBLfsl7SAvvMm76VdNrd
PeFIViGn29KvZvfkDm3fKpODdoYvzjW8rcf786ek3bRSMe5I0gn6RRU3jYWSVMalAVMkXaD6X3SW
cruc0zkPp726yvv/OsnODXI+k6LSk0d7AThGTvGqzqj0f7BLnzv6ZLBov0IR22UnZyZa1s5ZT1cW
HTH969rm/w7JhdPPrO2JecpgxkiqRm8m63x7UvwsKXxVWsYl4Izva7rqbh5TWQboyaNVOLBOkkRh
6/Gy5jBJl6mm+uy6tGDZkz18GVVcf36SpCbkJH+ItIrel6grLef9gKKmBdIR2v3rJJfxafQsLtuH
SapKunlnbreBNuMdYYy0M+2MguD++bdFMfX60gXACaDo7AybLArY1igy1WaXMAV1wm72AMN3412A
rIZPR8bbAXlT5ajkbnR8rxjmOZKkY9n+MyRnAdI13q2sTvcljaElPb5zoOTfksJr4TItIW0Xj9Ob
t33G50Xtbko6DvwuSfcoZhwOdC4kSUHk0lKSA2mm41n0vcsR57fkD0NqdpgojWXhXSpJqkZhOkr6
/dVSy8VKcjuf0tGTsLjbhsUyIfcrl95Yyu0Pi2RpwWXNNY3WXa3qp02mj69Ix+kr/ZUdKgcPAr5T
nTu75p6TVtmsxe9ZTmy+YymZU5a3gev3TACrLQ0A0zaNnBYiXc1jX9gOx6BTTos0MmfKwlMse/4z
JLsC7uN1lxKStIRmNGsNrp8HSGsNoBPQCrfV7GGX+6FrXknaCxyRpD+pbRw9Zd3wSKnQrm5eI1pS
FHDZH4XkS5JUiYIkcz2u3myWOb+kq6koHiTpHO3/bZId589zAs8vvlswOCfm+p7t5/rMMfKBcyCs
CsBsdf+iY+DtZJKO6UJ3jR0lHZNllY56+D9crEtpmaPgzPBH6M5jQOXd2mAN9z8WWpg6WlNMmg0e
2gVAqUD/AvC5jp2xHNd6fd9enrBWjY0iI+xViv8Iyf54rW2XijFXqC5JC+hP1UczP8pAuSBNpXtC
r4z0xW7vTePW5mAyzGmTGWQc/JOPJUnpUhmfD5GOoVHPbsI5SaEp0r/HNAoHdcNXqTJLuprScPK1
fLzl3yW5ULOOPpM79HFqs2TkJJ+leUr5+PjM/mrau9YEnCsBSlh+AXzkrYOndDLrMDWeqmBtf7uS
Fg9Wb8ZoEdukRTgHHf3lFoDjQav8DzUVTMdDSkuPklNIy2yuccfNBQIfnVB7j5mOoRfX3jFBOw3T
WMC11l1/+/8IyedoLJ1O4XzN0TNUUj9WOBaRFFqP6XqYpUFCkxyWyxx6h3clqQE1WCBJpUwnjIMn
jTE5yJTX+LyF/NFzYHRhv6SRdG7EhdZ078jvypZSUk2y8LMkyw39qySbFvj4+Pj4LE/lmPXdFNV8
2uTz8alnFxlOUt1wxqwLjFAfrUn/kTublXe1SlU18YW6XAzywO10eO4dfpdDixfRcl85APygAlaf
0GLARHWQNJgKEf7Lq9XtkOYZecXvPXi0GfhAG7R98R83wxTW5Y3VLgIDJeluqBT45cRw7aWbpIF8
15jR0tnMznc9U0vSanpLJ+4lNMmqzi15pAqV/rTLvpFukhbZlAuFuKcNluRL0x4fGcpEN76KevIk
pkl/uiW/2IF9QQVNhTmifIRrIh/utysuhdUwbfp3x+TCFZaPzJ/NhVKLfZZ09Wnytk/vSITOBJsB
Dod7AIM1woiE+iPMfrs8gDVqpYfn/CX1/eVqNR2rqTGblBrgaLgbgJ3fOYBeGqo9AQ/TVbQmEoe8
CtOsNSoJOFmCNBWoogOSQo//OK7MG2u7OOiU4qF03jnTfc0Af01kqqRTVL+elqI13fhGXiy4oeBK
/KgjTu7bEprk5vyl5szTzUKsfujkGaqtLslO2w42Zbp0pATLGyS7oYBO5N9LP0lh1/f9eFSSrtiV
1yVPFmoQP2qfGY6oDKH3PZwuqhG/aRf/uhWuik9twGX+wkbj5vnkL+9TLxKhuzeN/24BLNJww5Xt
7H3+CAY4oIFS4MlNG/X92mA2aKd6/KjsQDVtN9wsDIflbpqgWQM1vaLNqRM26GCyKw+MCJUnGgQ0
1SmdVdhc9zfYnrwb9kp/eb4jzaartPOdy5JU3O7mX+9B9gXSzBSYsnpQzaIlsDShSe6Kr444Onkl
p7P0KfVamZ23Rhw845aso5eJhuE/UrxTNvJfO0DzuW2LOwJvS5KqUSMtQ6RR7JA6wEmVJHQmXaS1
dNJ+bBr3v0ZyK598QFGfZqRZNtnuo6h2jD/DkwGcC3IE0+mwVkZs9JkwhxPXAS74L1MOM2TS1pni
bYvUbLmyQwG/oCLW+pQ7DB/QURrvdDpocCTJLTWUx9cM7/s96gV8pT/Uvt55Xaz4Bq+MnNxnkRQW
JMk/cu8PLW5KfpfCJOnsgHKp0ra8L2np7ATXLrrjK23Pa8oz2SLdLg4FfKMc9XHDruZWydIOUra5
aYRgOhdr1HvkZknStQK4T5fkWzdYupc+xWN920kbqlyRnrhmk3x+sPzLJHf3yQRU8amb/FOf9/nS
J0oI3Th1gre+uqXaMFjrmqgRwPcqceYcwPmA368C2GvXFHnwk/T+fGWn2s0I582rAVkg5S2/TupN
BUtIZHh2A43idGhyYJP6qTeY/7JMV3OcB4YEN0/y6kyQFpPk7z+4LtlKmCl8/97ozmt3d1m9ic4e
DpZ0Y2jXSQfDohwPvxhlifJyVJvi2u2vfLexkFzf55ve3Umz2MfHx6eDadIyuyjrdU/C9/4t3Qp7
PONnXcvQ1Kja1EE9rv0BsEdPtgMk009DVIhGUun5en+2wiIyerbV76Wz79TgPvoU5ioy/0V79WC8
+kEVy4EC2mZisNa1V3eg6pOAzEkkJw6SE3OLhWSXIct9FkLhIcNaT5ifvHGbqIvNXscVvqttslb3
pcOFyPyFM0CGzbWuLAaYLsOs5vT44ICQ1Ji2nXKbI+nvCpFrLCslaadrf5WHZAetpYSBnmpDxgeP
27R+GFaVtfrrkO7kKhY6zkgRMDyJ5CSS/8nKCK6prWt6bulj6KhpnABSeeeJVusxZQaAT+43SwlA
7typ37bmf1s+t1FUvwm7RrMXtjaToaUZcC0TIaTAlNzQLFQKagkpF1vCNueAVCYAp365kkhOIvkf
kZxgLf/QgUa5KTenpPzJSSS/xiT/1/2TT/d5nERyEsmvP8nh+dwfJpGcRPLrT/JvNNBrRPLJqa8D
ycYszcn++Vg5Fcj0/ztUb/l0pb6qU1ctrpZEcozWy4iqeF1InlggMZNcbPLx07v3n/YLvZvb1PCg
RRdHuUG2z1q6QZqCZas16vbN8sVWA0e1ftNX/3LyrrTduWHfRhG2ulI7rDEhKa2BJ75Khlv1XFls
doqSByXJ0hLytstuuHoM750GoMjEXybFjH/K1hegaJNPy5nfbJLDczs/fJ1IHpAmEZOcL0ChjxRy
/9yBza4TdXPD5pvab+4eKl2pstF2yt+GXeGcJCnwyt7Ve25LWpE2tbF4ctlS1uBtSdCHANx5QOnb
klYbsFcO0KpKaYof6k7bUAX8VgnzMknX80OXMEn+BU3DbMla3LIbFR+y0OysJP2e840m+bAR8/na
kNzd/CDxklxUvxkeOykoYpnkCG4HNdByd+h0Hdk07zdN6/tJZU/rmt/7dStvuJgKSp2ybG9ezNdy
5Uo+oxyUNRUiI/SkGOBiOZT+TsDYXceMbIbZ7gc1sWbQCt//41k9fmuUZl1bp4Om6pY79bK2vF0n
h7ESMqWF3aGwhlBZm2kQptPfLzyrvxzfZJLHMvu1IrkHfydeku38npgAaqtIQzUwdIOQ0LLQviH0
1nvR3/vfP4Asl0+XB+ZoTOiJZNhvkI9Ni+gsnU0DJbRwqtpCZS03KpFYU8PZnzzrCk30S/BanNgp
7+PWag+NjR5Bk72ky3YM1KfufpYWgPNWa0bQN5TkDzj1WpH8BX8mYj15mzIaNDX2Uk+gikK1PsJf
rVV0kpcHQk1VAIdeYX7ugzWGSdrtEukUdEMbTHyswX7+TpBOvwN5w0852FIlNgC4KlUFumiLLUJq
qioCTlrxuZ6oDjNU/VNtM8pXN7F/k0nOmzz0tSK5P7sTMcmLNWDEzgG0UNdyGpG99DcBIVO0fvja
3Td6Q131jE7yYqXCW8t3H/XXoxrYHwweoT9TRxytpolb1ZseGqk1gCn0PNBKQ8F+9LQFC/+85wiw
X4/NwIcKUzuYej1o1EV/Z8BdG0dpsPYxU17r1fA/YLsIcCil14rkIUawdCIleaKkkJHU1lAvhUm6
+l4VSQp7FF6cmnqqSvUMvUV5BUqPp+QASlp0NYrbWnEty3QjpNAQTTSCnG49ANpqILj6SdJaAM7r
pJEMXyoJA/7UVS02nPKXj1LNnSo2XZWPqzB4bzj6c8c3meTzicua/HyShyfqMXmQ1lRxAS+N9pak
H1yhzRc10jg10+CYJA/Sh1TSuJnBmmQCOKXCUVO2aA8ttHmIBmgVYPfED/DSZsAudVYZXvbJLfIF
WKkrKg7Ut/6CFNCiUarZWFvHqeZv+gDqWyQ/uzeY5G10e93G5N8SMckdjRKRXhpSSxM2SutSkNkO
aKZBMUlurzaU1Gjy/KFeRiKWHFEPh5zH/KsOqvbNwMyQQ3sB8x9GKvymMlIAfCn9ATS2nPpCw8Hl
L/krA0ARTR+lqnaH9JOaDdJhM5A55NabPCavZMzrRnJiHpNbGeEd5TSxtrrjvVMH0j/pB3ytj2mk
p4Kcm6gf2TQb0voF5wVWKy1EVlK4dh8K+0s1e2hnMqyhTgUfhk31JM35sPMh3XO8M11Xz4Z5JhsQ
4lcw/eOwaa13aNaVy8YSjWaOUm3qK1x93Y5rODBQO99kkqcnMiPcc0nuy8FETPKnqmeUsJn2gQaD
eaGG/2RZXKbU/dBcNH/adlFJ3+Kq1UA3rQU2WuxIdW287fC5+0BPqaZprW5tDgs0Bux3Livs8F2N
fvuuJF0o3FWP7up+eXj3jqSfHK9ftEvZdO6WPPpxqcrBPmkyRYLC61I1PKRSsjeY5CFsfL1I7szh
RExyM1UDyKgfK2gGkPru4/yHJKkLdIkIx7O2gpoEPw0G7LZuBb5/BL100+assekGYP5FNXEc7a9z
tjSI7p8f0pPxZrJ+s3zpp86Y+54+NTUzQPLG/SrZsUyPQ6VrmXbN7HfQETpJk6GrQg+GamifoNpv
Lsk92Pp6kdyEC4mYZI/dRnaJ3lXtO+QCqDkMuwZz1tUBcszPGp1k+/bvRngc2QGVOkDO5RG+QenT
AyRvYAacMkWNMXH+P+5Baacd3TG4REScidOK+1WArk8U2Mf0x6P6bzLJP79eJJd0CknEJCfalqxA
Mkib7g3WLtoYyS9fH5Kz5FQSyUn+yTFbC/a9ViRfSVxu+P8+ySmLlS9RysvLK3fmt3J7ej6tR5iT
JZH8jPbRa0byz/Fb9STeSS4WPQ9f+P371y6cOXn+wl+HfXceOnH9YUi5JJJjb58lKvPs80n+hmWJ
n2THjzz+KWV9n3fFiTbkx//wRVpbFEkBt2eIM+cp4PAfIbkjO18rkutxOfGS/P4GY13jc30dtbTI
4qhcO7Z+OtopHak/HTDo887DPwdaS1LoqZ1Tun1Up8mEWK5oLSXSIETSrVT2zces3HE+NCI5i13U
IiMlTf3vS4988v0nSO7GlteJZEu6bEq8JM9QIYC8AY/eGbRlZg7wrgL00WCAglvzANRUNzIsP7+3
Ldhbs3s3vGPIuACUkqZHOHaWUcjlo9eCdPOhJPmPCpaszLrdDmyUcdj35m6S5P/HktLtD6xMj3la
sO+YNNBkmQtw73x9/Tnxu8u6m5Oiw3674PNmkzyaFa8TyQeiVZhMbCS31TozOB5Q18OSHpZk189A
lrBLAOU1AqCr2tr9psfh+rTRrUyA6VfLfG2uW+eT9o3yAM7SnooVvGm889jJoW1LOwMN5FQ0XFJP
ZknWFe+2GoZR56FdrhRgWqjg8FUMV7h07S3WqjG46sgueYL9MC1ZaVHYtUVvNsmTmfE6kfwlvyZi
ks2+6g5TtWCWZuS8rF+5uwlgnzyAtzQFYKAavqdtdnnD95XWMKC2Vs2M6gPnL0m7GKVHOcDx7Uat
mi96ACsk5XB/LKkHACNlJJMdLW+jgvu+dL025Ak9mrVOT8tph9/1I3jId2u4C1BA0vaGad907WIe
419c3qMrCUyyJUfqkERMMnkCH9VspGP5Qi67YL75xNGIWPJVciCXFhghHZW+1qfwm7L7XzXDT3rv
O207ecGWMPymJPkySuepcyjgifEiIvMDhdi/I0nfAjBWTUr2HlmSIVp14GjhFA/upAMmqRWwRE2u
KywHWeT7vTIC70na3i/bm07ywpepttcoa2jCkryP3krMJPPRY/+7AQU+Vy9gnYpqHcCNABOQWisz
lGkx+LTybVAB1wzj9MliVcQj/JTpKynobm0rZn9J0k+M0lHSpIEOkjaQiXqWU9SSZC0tUl0WSfp4
iCS/oh30JcClB65AI40JDdYyPLV2tooDX2neH1JI+zec5LXPt/so8MhPc4Z2H3Tme1slA8veLRH5
s3cM7TpyR3jMkx7O/O7CC9yiX+npL0Fyc+MP/bwWmmAkM0jqzxKVBObLSxuB5DoCkMN2TmprtZ/V
NTWDTzWCTzQ00ubwuyT5MEG/kr64d6URktYwoBLfrqWPJI21xqsentWunZZNV+vksE5vARm01fDn
XKGDO9TcUwtHqArYHQ9JR7aON0Lzvtkk//K8GqFXG2e0OmfN9Eh1VZJ0+12w8dcZgK8ky5Si/a9G
5odfnRbsvo4Us7umZ5nu352QfpndOH/2mtOtqehC3nv6nfD/SD5vX/kFvvWV2g75YsRf3558XpLk
v2d0i3mRu//aNn+X9TEcWvuZ9r3Zh16Y5OI64chP8gA2qYS2A8WMQKWSOr9mepdqf8ruir+v7/qJ
9/xTPLnjNFfVqK6vIl/9W4yBd4HWMMkqexv9rqd3+oI5irBdGLqM1vygQsB+OQGlNR+guZbJN8/D
wGZaNUgfQhsjVLWPOr3ZJJ97XvTTUYeM7k7DZ6w/cORYcp+TacZLIe+Qzuafv538Ow6saP2H1Aeg
wCXrWUvNLsMmZaspaWaOQcct2uCUq3pWoNJZcMqeGYosC5cU0pDKoS9Ochd+ef6XvpydvOYCYdLt
EZGm8kdvk9nv+NSmuU1AQ9veOx8AFDcYr045XRrnJ0n7mudrbatbcSBEwc9+2mOQnE7bYZPSg/OD
AHftB9oY9FXUSKPOY+q7xw1kc81R273KSPnI3PTgK0n92azVRoluST/TUcsws12y/s7kzQtU0qw1
ygH8qNxAXq0BXA6oijZRKyxAP3RXS8o9elIAYI4avdkk36fic4SEqT5+hlKhMaQ4pt406GTzoGvM
98bGQVNqJ6dM1DE+7XVw3yMFhlkN+UWvvY+/dGJOXvJS86F0611YJ9304t1HL64nX3Wu8vzvHPgO
wzWIU7qUFVPvH4MfFal1VWpLKlaVgNTeny/6O9jaNaikqVHbos4UeLL/3atSBQ6vY7lk6WVHFtr+
4OYvhXxhGqRQ+yovTLKHtsN4jctWYb1mcOeBmeR/GfaF9wxe16jA8esAC1S8QPCtJzehovol827d
b8rwNGAsuQ5mtzaTr/ZnvQeMGrVxHZ0VntY6GRxpFBzxrwAr1WSLMoJTD/l+kKdytauBuSjmqxVo
G7QM05zPNKFfQFj99ivMxaZazpWbnOFNJjmI8s8V05O91q3ekHyGg+fDEmarduDFnYc/zVwepHps
DDv9OIfJYP491hrHb7oXbulaj7dLZwySHo93SN6OEZL2JXeijOVGfhoHvMSMr53pBUL4JvGZdMb9
kqWkUbJ0JxR++LNd0Rb4pjZFT5Uxl8+ka+tbMnMgN6UhzN3CaGkYOQ/o48LjOKK73uQ+ozsvMSbj
fwnSXZOk35MxSzMHXtAPRnUzY4FknBpskyswXYWZIm2EarppPFz1gQOSNIRfdNPKHUUWMlmqjHGp
pdbYlPBNB3TSfpl2bLlmMaZ/mqegW9I6R+4vBfpaqo2U9LAeSyTpZLH7evdNJjnUMe9zxQxjnXWr
Hj0cYbI8bK6VDUgPUPK+c/pwSfWtIfzpkk1bubD9h6s1mJ8UKG94V1sbOpF6y2bG6e5wF9NSbxYU
oaPlJWwXR80vkPcr1NP1lvFaoFTlz4o0Ii/2vJOfzXnNj7Ok2ulzPErfqsb88Sh1K7hLmsuYnfTS
YXP2m9LhQYPYdi4Ptf2k888ueRyT5HOBdpBprM+C5s6Q+bwUNsFwey+rSQBVNXtsaHpglkqQ9o76
QOZAy/F5XRt4f2wH3JGkIwPuSsGnjx3evWnj6uPX5vlL25sZuv9ya8DgSVk25sH7ifTg90Of1Z+1
dEIlU+fTV3d+aoYyeY18ockHzeueDgot3TqvkVOK3X6ub3ReuEyZnytmCj7WrXzcW5K+hUVp3rLu
OF4ka6me0yvyCc0kqSlHJUktrH/WoUUyhEtaDJ7rIVPvG/qOL73scV+u404mWltewgpnKet09vlf
2ZfGtqnoIknhpeGTbiZq3DUVlStgH0WGRwZJUniy/OnzSFrGAF/6qYpRGlsT6ZveNMIi6Vd6vjjJ
m0Oi1nJ0/6i57YhzsxwA5i5vpc4CkL6hHRTv7QKkily2sPv/X8+i3baemdMDuHumfnFv+zc7V6dn
8ueKmcwa64BnzmJsFHMLitphN46MlKSK1tCk0JVLxo3/+YiTg1GW9wy4DaSTJI2hEXZf3jMUvnUv
Y09eaVihntO+sllVSnFLkr6B7qpud+AAbYNx/2JslEjycEobG8lzOL0taS4jVzPwBgWNvQsAw4Vn
RfRS2P+f5IypXsnj8rlxBOeSvDqf0UoQ9jwx39hIfkhxY6MTRg3gsPtWyNIySdJjt0xRhQWaoKsk
/QmcS0/vMOkrNr1PseuSgkqw5yVIDsmZ8e4LfOVWtuXsjB6yLol11+0jWsXgULyiKyKUl7Ty86t4
4yXpCxbOY8wKm2fHGiDzIUmab9i/XozkV2z2S7Yv27hxy+7de44ePXr0zCW/q38fPXr08M6Ny9du
3b1548/7hieR/IzmbUyJ/18bxQZj44HN0PGzdY1kaIonklSbdvSQNMo6bIZvDzDGsreMvDC9MHPy
VF7aS19wKLApua9L6h5rCOGzSJ7Byhf5yp+zy7r+YORZbg+tJGkW40Jtz6GtOeSQVMnlS3qbC0jy
5uhMpk+wYbuGTm1NaY5KOpF8RbyRnBT99I9JbsSJ54mZzdfrxn8z6TvdtJFseYcFxrt8bKCC+5Lj
bvKUfwfOc0hzW5J0gRmSLmc3rTC9ZVHA1+Z0vdmgO/nxUT+OyNKT4mFSdzbc8v7+BUkOzl7shcry
fsNYY6MIDyTdS2OiriQtZqicc0Tv+7a9nzQWTEfzON/VclMhzWD6RCYbR1cxSQtMnnckBSuJ5MRP
csvnZEI5NrZNEWv8gR68Y1NV/07mekvSBXecPd3J8bdGY7bHdYfVtJei9JHT89IyRHWo3DQTaQ5s
YYp03FzA0o9fJTVhk/QtSwdS7wVJnmOz6z1PjbTL4SfJN0dGvpWCa/MxJY11so7K4xT9YejPt1Jo
NTI/mkzOSuZke7SYkVv4SJJ0Zg6zpCF4B0hXNgQmkZzoSe7wnHWzduDwdrPRSzcujZbiZd8SSdIf
nbw983a4J8nn/bJtT9qO9gVwmSz5eUPKjtcUOOaSpMZsn8AiScuZLB0qdvjaUL8XI9mSL98LVkrv
TbbhE5s6OC5xduk/siglniTLIkm3KasPWfZd789bdbDFT12xT/u71riYeDegg5nC+6TdNAxM47Q6
NOxEa1N+lkph9fA6c8rtmSnHkkhOPCR3N9ZHn9meHDns/9IXDl/cpsFgwwX01sVI96Kddt/fHvJI
0o0eZ551auwk74hYun1eC+tjD+TYovUpgHoPlS2/JKmC3fGImjMRRhkTKXDZ2JSJenTdIik0Z2qt
ccDZEYp+zUJJwS0wDcfupySSEz3JXeIzdcuj53eJneQa7o9e+CIXVi3eFy7pjs+S45J2HZMkXVwY
2qN6+7m+B44evR/R9ccahT49oZAZEc/q+RPSXx28322xNPRYYcMK8tNg/3kHkvTk14HkxBWFESvJ
J019E+cvnERy4iG5p+FilbhJbu98I4nkJJL/P8ndXsRT8v+2oDgn+a5rWyWRnETy/yf581dN3bK5
SpyTPClR5UxOIjlxktzLtij2T9vHDeKc5JKFX03m/GVJJP8X7MmvWEupccu4JvkCg15NZr26CUdy
3szglSaJ5LgnueGrZtauXjeuSZ7AsVeTWapWwpF84gpp1TuJ5LgnuTzhr3aREs3imuQPs7+izEz1
E4zk9JZ9jp9oaa4kkuOc5KJur3iRXB/HMcmW1K1fTaSfuUsCkVxl1zmFS9KYJJLjnORcGV/1j9kq
jkk+82z34BdrS1gUryRn7//DqJTgtrtpo4thunxM13umSSI5zkn2yPuKF/GIa5KXPd/v9P+32m4P
4pFkh2Ehkg5ntxus0djfu+vQJVoO2iSS44jkIMq94kVSNYljkoc4hL2SxCNxp1zERvI8netYbL7C
/XQ9NxV0dslNPTieRHKck3w6Mo/JPx2T49p20TT/KwkMLeV6OR5Jdrp3PAWYOx0+vzS3+wI/SQq6
6mtKIjmuSV7Hq/rmZKoTxySXebULdOYrxeeYnCF9xGaah+FB7derSdLKSDyQPJK5r3iRbBXjmOSc
7V5BnH9b6gTHK8lRW4awrVx/6JhEcjyQ3ORVp1PyjGuSM/f5p7Ku+g5KQ4O4zNz5HJIHqENyHYjt
iJtDEsn/LslvO7zqiOVZLo5J9hz+jwRdHp4f7EqvtCjhSE7d0sE8+aMoO+qOGjb862Fjpi72KZ5E
8r9K8h1771e9iId3XJM84h+IudLczlx1yOqrcfwLv6wHUScfW0si+d8leSVDX/EawdSPa5K/fHkp
OzycvrwaD7/wy5Kcv2aVUrlyFyxYqGDqJJL/VZIHxpoH6GXaDdrFMclFX97N/leXAmfi5RdO8upM
LCSXzhH2itf4g35xbU9+72Vl3PfMdUtJJP+XSH5ifuWwopXMimOSh2V9WRmdXE4rieT/FMl7Wf+q
1xjCT3FM8s+cfTkR+80TlETyf4vk/UWDXp28S3FMckjKqS8lIbxEsdAkkv97vnCv2nJkVxyTrCYf
vpSEtf/yWyKJ5P8Eyb/TNs5JXpX8pcbYGuWVACQXvPQvtJOuEa5GhS/FVcufRHJsrf+/nfglFpLD
yl9/CQE3HZYlBMlF/xV5jSMGzrh7ixZ+EZKDL/y2cdaMG68byeFn/qliGZ7RIzTOSX65tt3JP5GQ
fO/33478tnHH8ulfD/98+ovJ+zvS2+huvJMcdP3X2YP6fPRh9RI5rLV1Pk/UJH9bpETFiu81b9Ww
WauPPqhUJKtn9uxOvP0Pr7DpX14X+Yckh++NpPeqt1/Cknx60ajOzWsUSWMXTTl9wQyNXSJOOBiv
JPt+UbOAXQyNum+iJjl9bJMAh4B/doXqT5W4SyCSP8Wl8eagBPiFYyH5jlPEr2qXu0qTFh0/Llkq
O24vGMF+3t528sL4JNnf+i5IVfy9hh069h40bfnPf94uSuvETHKgHTX7dfy8Y/O6DTp2/GLsvJXb
dh9bBCf/0QW20kiJgOTwzAD25RdfSwwkQ41eo+buuq8HjyPegzi+qMT6kR6h8UiyxY3Oc3efjV5y
piZVEjPJT+xjhirvhkP/6AJejicSA8lbcF/bvihgV3na6YQmWalZ9XS3RRD4wvqatX0Qr9pF+Vje
AV3Il6i1i4yMiIXko/9E/ko6Kz5JfparcVWaStr0DgDlht9KWJLLGLUJo7ZfrbVPX6CFZbKSnC9e
Se5B7xgdB5MlUZOcz6iRJ0l6fPnsod27Dy+GaRs33ntZ8dfTJL8ZbyQ/mt84nbt3l+/2hETff+io
5RjGonv4vj65AUyVZl5OQJKb0FrS+YO+vr6+O7evXbNmz5EJ4LN+3vgnLyJymJVkl3gleUmU+uW3
jvmunT9/zjAvnD//uGVwYiU5sCiFWjWsW6NUvuzpnKPN+rq9rPj6TFU8kbyimqvtNot+Ni8qzIVx
yopHhGfGgR7FzAC5ZyYQyeFHK1GwU+M8sc2rX8jQvdfW2y8+Sb7v5Pj7Vp8JAz+qWvSpIq8nEifJ
H7s+exGz+0tKn0KV8DgjOehhlH3BLQFyRNxptSiWFt+KzoC57vjfbIOH/5axnyTDI2FI9o3xA0cY
I9zyxFxECl7bIpuT8zt9LkbuCs1i7X8svki+Nm+gd62n7tslZdpsGSGf90d3EifJVQHSl6/ZoGXv
LweOXLhqx+69J67d/MmZ2g8exiriyeLPvIvk+6DLqhg2rrnmHHHxJa0k/5EnytStNxQvmHZwHjCX
cgH4NOodpsfTDOBS6esDNm/r+U/XQ40vkn8w20OZ7mOXHrx+4fr9R5IUcD9Qo2FHSMzTl2azZcno
Y3sOf+lVwbpvTXyRHLmw6FiweqshMzcdvRAoSQEusSjPEe3y+mlzfwtIMJJ1+4Aplpfc15hi9Wiz
TPKwfUmPgdG5nWTKcFxxR7JW5opYot5vwj5PPt6+WwJch0+rZMYUpbTqUtxuhfzav5IbgEfTudcl
qTrDE4ZkhU4nWczZQ1AGYiaCDGoFFB04ffHwvFDUOgTXjsBqfHyRPCiNZxHou+vK0/aVBlR+hoy7
w403pFvtVWEJRLIWEsv68l2IrTTznepg59158Nc9qjhCsolRbAdzTZn+VlySrNFFbFPQD8jTDzOc
flgUM+3Dzr9F6shMb3WtjIQeGVvBEXDoekon/6lZ8V+Y8ZWmTSwdOxMzk2RLeMuoZB861hXHxdap
V1wblGPTk/vhHMsi5AhSxy5iRSqANM4A+X9JIJL7xWrudqDftPwpyn62IOpjGfAO1LfGw90cnh4a
R8xjZ5syxVGgXOSMr2tZYwn6iol5f9sB9S13itplMNcLPmgXmVcoyIXlkXe8vjqA2QOX0AQi+bIp
1tDIufDthO92RFXhxkLNCHxOemM2BpOsERkF4pHkslSPpeeC2L0/wrqAudX2e7KcmV/FhLlbSIKQ
3DqakmmbuUZmru7pF6Uro6KYwTqaIszHP9vH0YgcleSw1mX8JOk7kvmrE0Bv+RXF067yoyJ4BERO
9aPZj0+Negsgy5zbCUPyIlLGNg9eavUKqLE5wkZholaUN3N4LzzuS9IE258idzySnDWmDVzSNvhT
D3du3nsy6leyNINSEU4Kh8tAeb+EIPn9KObkiLY9yqzVY4F15zqeUvh3Jjc1uCBJt7Kk+EtxTrIs
HXL/LqkNdaRzDgB17jz8AGiQB1Zb+8+L6f94tQ6Ae3iCkNyVqrF1HBuZHfy8sacG7tH06fDS1Fh8
XbprBuwKJ4eg+CPZTGzOekfgo+L2AOnHRY67o+HDKLcWNtqRFglBcs3YSB4UmwdUIYo99drY6EAX
SWpi3qi4Jrlt91X3tLbEI6kynSV1BCDj1Gv5geQwzNp/AAVjyJgFVZy9E2ZMrh67K2SURETpVt6S
dNmOp/J4HAIc2oSoNcDOx6MGhccfyY58G0vPw1GgeM+mGB1z5L3oXOx0oI8l/kluE5vTT+loJJuW
S9JPsO/pfl3wlKSpqxXnJJ/uX8Dx/XnhkvIxSdKjTBEeZm0LQERuglbEDBHZAOdvP0kYkovE+p62
ZI/4cTHhvlcah/3T5qLudsAgHXvazhgPJOeO9flbY73nrJXcIIfV7d6LDDdjYMG8hJjxvRuj30kT
rlGdU8tKUtNYcoRvhCNxzEYU7eJQvzwPJHkyTZK2mG2qfE4AW/h0U2IWdloP8RTtEJPkwlHnFhHt
N8g5d34jp2o5cS4J2QP1HjEzQ/b79AMKSh8AqYLjleRGlIil55RoA1z5UEnaEouZK6BA3Pk7/T8r
XLoY/QaCJ2B6p8si34WA/V0pPAUxY+0DnBgTfyRbWzYmSpLmmKP+rt9HmLJikjwGh6CEIrkgsaWx
a41rx7Tlek1tb00H98sDx1g1005kkX4F/uUsIs8jeRrmWKxwnxqrNu/17F/O3rbY/iFFY2oSHckb
/yQfgacdHx+mBMgz5Kyk+x/jAFuk/cRWcrIUjeOd5AK29/VPeY1ajV07RnEHaBxLXrrqvKeEIrk6
TWN2O2ZPq4EtDA0jZ2Zgxmq4GIu8bnhKqgDkssQnyVdNsT06OQD7Pvf9d4xpmRtoIOmGObaOQ/CM
f5KVnbFPdRuOnXePLX8e2f3juGYpAZgmTcc5LLantEK8k1yZ9rbp/S/jv111Tgr2InPkaFf6aRFn
7ZmcYCR3j214qm5T8UuNmJgPB1f6diWXYiU5j81kZ0Q/HQuJF5JViYIxJpgHwBUKlLG9C1Mb+uXl
xEJyd7JH/3VuuVN9R/vI93aawgyVesTqIFuLSvFFsu+MqX9bX11Pz+n8k0XaB78ig24viqa9tyHl
/Whz65XxSPIGTDFSJm01QGi97Pyi8pBsdTHavRt7itNO5NbNFn1TAs2lkwOyMiR+SN4OM2NXLqK0
+9IoUsYism9EDo0bE+7GG8mHTdEnmmE1ot9uxQst6SJ1jGVmKOUmso7S5cmP45Lkv07sMPLGLrGu
pEYGnU6INCcrPYxJwWdRJJx2jLbQu/kd7PbHH8mBKWKoFzczAaT8rLW3G5jqnFVueuYl1oJUVfHW
PuPvYL4pdyBjaLyQrPdxe0rrPOEA0esInpQ6Els6hI+tS4RXuzrEjfoZe5aABqS5Fn26lyNttrfz
pnMBU6HeB6QmdJJ6USYWfQ82WDcvtDHHQcRIbNrFRZgn/V3fzjZFCs0De63bD1KBa7RQIUtV3KKs
+m0GqGKJN5I1AlP0ZHuh7+FYzhb5W+83KcieKelidXF64MRg6UOsKt5MM7Ayfkg+60rOaD5i4d7G
fM+WSyYXHJOa4xWLyDzWEPscgN2xeCP5hCulIx1w5tnjZSjEgdfuGjP+snwjjYjNw7cTyWwjowfg
eD5OSfY3tKCeUCgsNAsRqQy+JNISNNBqt48EuXfkqokk3asDsDb+SA7Ki8vWqH0+g1E6N+MT70qf
TPaTpD9gl0esasMPsE+67AJu4CXNioun8Bn5LmZD0SgjnKU3eE9Z9lvA4bHtPny/cd/Nl+CM1DK2
AW4f1tDFWSkgVg+OuCFZ35l4yxpLHTrCDtPTKrwHq6Tvn/JnkKSLzpEG9AluEBdvkkiSf8000Gpn
g2GngSJGj+9NUcIvBjzlN3a8ItSIPg8YD6S9EW8k63hyXGZF4BfUBZpHh3EyTgHZY1tq1UdkM+Z9
dL2wYL8UkhHYEj8kazjkPBfxqTcQzcttHfhLw2Mx4qoOGaxWzysZ4sYd9Vk5iCaYcOnxe6Dl2vel
IcZq73U4Il2JJVinDskiiTiTEuKgIHQEyT+4Vjcu1ggwtQCrMrPDBYgoNjkOikARKyoBY5yg3lNu
tpZixvwpvkjWL2mhqtVV0PctKPOUP3olqqoKDWJKO+9IT0m64WazbIyBWF/ncUGyxkJ668vrYTMg
T7Sjg0gnaRvEcLk5YGK0bXsW4Hoyvkh+0KE0gMkal/P0wugy3IIlvU2zpw5shnFRPo4B0t2MK5J9
zP2tVsBvwQmg1FVJtyc6AZGFXX+GM2Wg5z3J8mfPNJBqWoy38X4z8EP8kay+gEOreb7bvykBNH9q
lea0HfP0BVnDYzHWOZ+z8mA6ZShYqYEd8UPywzYNnaHy1L1/7RqaCTCeqohWghqS/F3p9ZS8sHdJ
FzH3DysPlLHEE8lN4K227hEz0lYxfs/akjQKl+jqhX82ikf99YPfBmrEFcmNZkfscmaMN1BleO+m
Bazzj44RdgE4NApwKlgqLWDXMrZkiD2B9DfjjeQpkDtd5Ix/+VPntMP9kf62i5kVYwn0t75GPLEW
IZwElA+PF5IbwKoSUU0VP0c9eslsmOkaku0pa8qgaF/xtLuxHhEfJP8IvKegAwsXb0gJkDb62/h7
60rwLbenbBPNsNsbbcdvDsCSOLddhDswKnRoZIRnqqhx4Bfg6ChSGMZ7jx6xv9eCCoLxeMYHyeeS
wRD/iUXswb0Y0OGpIdmJXpKqUvgp14qHGchu00MG2+IQHycHNsYHyb7AivBlVR2B9GmBDNGIHYzz
HUna8zSna0zRTeOzAKfT8UFyeF7AyfjJhpbh6SfovAd5ja/QC9O2KAdGEWOSMhyias5xRLI8GCLZ
k9HTFexoXjhq6r2LsGsg3tend3Hh2fGT201gWhs/JFu8rVPhwEvXw+oAyaLNqANKkuqmpFOpn7LD
WZpChBv+kywmq6LdC/COD5KrAyMkhVw6f+2eG2B/KspRP3dbkrj3SRXVsX69G9miWe9CiwCVQuOB
5FUA/GpV8h2AdFHu7EIeHK3OnA+ykTwikMcy0ozX0245QbmAJnFOcgqmKQgWbzFDJhZ3zB1Fgw82
s/hrikgjgQX/V2rmgHgheTmAs+HNuwGAUlHS4AY3gKWSpJHYb4oKck+iBgB+Y3ssr7gBB+Ke5KNA
RFb3yfCU5tgZlysR06OGkfuXu+D+VMjkLtO/7wAVG8nhhQCYKylQCn8b4N2Ixd0z2aMkiZwOLmMN
1cO/JeSPmaVqI0SulcQhySN1H2amA5atvifLqShMZmPIZPJop/n/Jmq9kzxufH5jWeMz4vB8JEln
2yUHKB4xaIXUhh5WAxIk2xn5Z+kM+aJ8rTspc0X5QerHPcmdo+aiq2MocpEOOqttlAc3jmr0DOlp
InOM2N+OQMpLcU6yD+QD+kvD7T84dd8lOUAhq6vAMndMg209H5UCyDnx1yPbBqSHIrGpEQ2ADLfi
mORM9NN9yAJw4en+Zei3lmQPc0EsBvDINgJg8D++q49ylWzayLtwUS+v97y8vN9Z82ySp+OcMVI5
fmIo8Dmtz9jR0vCBYe3eYQ84zbBO5s54Q6Gob3PNtkWHHzCD6R/PRrYWzv9BixrFC5f2qlTBy7ts
vdBnkBzgDpDT+un39Ia7egfrI/hjcnIZo10rIAPUuyIp3PddKHQuxjVveQD5/rH/xZpCBWu2eL9Y
4TJelct7eZdp9iySy1G2HlBDs93+1953h0V1dV+vKQxVKYoFFcXEGjHYG2qMxhqV2BWxxl7e2Gvs
FRvGHkti7LHGJPZYYnujRmPBjmhEbFgQBCkz+/vj9joX8Ps9r8/D+Qtm7ty758665+yz99prA16H
1jK7Vbfhp2JOLw4B8gjJ0W5A09Z8ocYg1RqMG24AQrLNv1hbPrhVeOOQ4FqhDUND69cYoI7kQEyj
hywZQcE7roKx14CGAOCnE/zJDAYAtko1OfrQj8vXbjp42WjV6k0ZlaajJpIzAjHQH96omBjW/G+i
N+3yugKAtcehk6cWN7MAQ5jY4t38gFcIUHPHY6K48W5AI6meSOq3bLdiewkAPsz3djw8u2v58o0/
n7tjlCLXVmb5PQ0kb4PFC7BwP/JvYWyPiJGnH1zaFWaC+xluIYHr01aAW4exnf0AdFFrIPAjAExg
zX9wesfy5Zt2nI8x6jo3kpn8Qh3JV4Afy8KEQnesAFCDLo0Xf6qGoMay1wR8R6c6+wHI318rmR4J
gONQZsb++fPylZt2XYg1anRZqc3mDFUkl8BseiBtWyQOcw5PZ2WsP9e7VCHAYzJR8onZrcq78Fd0
/ajl+D23nYY/V8pu7hpNJG+C+Q+gLVwXAAUyiejZRQvMhfjS9U3s8Q0AVEzpCgDe3gBsi7VCbecA
uI4gurt+YA0htmf2q9VrxRmnQvn2wlLDy2jNyfXQsChEKiEzpZ8ryXjqR6wA6pF9JqsoWHyb6lUX
AqgbS45ba/tVE/pzm/OH9vn+L6ez3jtv6aWr2tWRPAZ5UgujItANANCSaAuA4aN9AHgIJexEiYUA
/EJEjrhYnfDEWACNH5P92sqvqwgmmAt+1u/HC053WI9l/QAaa83JMyiWOUIZSmuAPlQKMKsExsUj
Bi5lol+ta+mqpogX2HPaGd1qo6qyD5zUQrI9GG2vA/OBuoBHOhHZawLVE8YVAGCrvThRvC1sQnTw
MzMAWNpo6zmNRNGuj8+PKKHaiqDRiI263YR+lX2ghYaffBRY7ydW8Iqe3BVAgaZuAFByPjNXX83L
KWE+jmpVs9HowxozVgd4TM84M7SImsmuTUdu1XVHN8g+0EXdu8gohJ7khpFsdWRILFF9AN0oLebM
JcnKPRFwXrR3zh5q8Vye8Ue/QmpGe7Qas1NXCSFSRWNRBckfYQid02rhEoH6jjIoHgTgW62J6TXN
3IvpmTN15B3h/uVuzYqp8/KDT2gheQ8sN08DBwCAYXeuAlCb6N0/F+4KU/+7kgAY/tiTHYuXbNOe
Kvq+aOCRfreyXnu9UtO0o7dNZcc200ByTQQ9NgmrKxElNgNgTUyPPn6GA979ILjwm1nt8duTykXS
r1bQM7ncnFjNj9eUHdtJHck7gAvJQCXA1KwUmhLRQwuAporzxecBYHqlZ/HeW1Tqj6DyGf8tpWOz
OWRhnOYZSssOHqqO5GAMcHwJPuQiHZNRegNQB9Cs731Ko2jwHbcyz79w0nExXy8N2eVp8iN/00Jy
U4TRFmA9AOAAET3JBzC0IPFg1u6ZTvc+l6jsytlY8l8nhptqbVA/wVub7MhQdSQfBtafEzm3RM+Z
ZUgsp5TyKcxdACe9l0/R0si+2HjQicnmehps1WcmlWVEBcktUIX+BQD0o2KYTESToEpSGQQA+q3P
e0dQx++6mvdvdGK07Yvf1U9wV35kD3UkV8aAFewRyo3yPPj4AYAncEz9MuNP7t6w+v5SS4nyTruH
un8Ta2Sa0ERyggX7aA5gAWoxK1pvAIBst3yZWRw2OfFxKaap/dtar4Jd6jvve1p5u5qzvwfGkNwW
pTJ2AiV5WlNKdbibpW170prBtLqlymMpGifSqPXtuDnxJSyNTE5NrqXaM309DCH5phUraDUAKzq9
sGADUVoAAFRRgMwNgF7DlPuN3h7L+2bV4DsBbo2c3+cGh9TO8Z1BJFdFfQ8AMMNd+XMtZz/bBFBd
QVbfS+uQeIro4U/duud1bqlHV2WfimcW+VF7NZC8Dj7p1IPx6IELRBeZH1S6LDmqMWc5o4/ka99S
+6XxexyHp9TyM9DDN2SJ0jvqKz+otiqSn9iwkJYDg1GMiQSm1IX7QkDM9HZEAEMSbOCLKtXG1DG0
YwJR5r4pbWoZMLnaaqWj3REqDpESyX2Q/98IAMu6ov424DbRT+oSZMxiPlDb5rSgydRztj09fe/Y
tlUMGF17Q6ZTJw7d1JEcAsACoLpa28C1AGoCrt01mmKk9Hr02kGvhnvVftXHUFdncwe5duNqxTF/
aSC5MzrQBleY+jREZeAoUSv4lQRwSxmkAsy6O+ITD2joxJQndLAGjhwtZMjy4qtk9zhD0bHuK1Uk
T4L7K5qMAgcBtIgnoqGw7F8HFBKYWjQVaJm5AlASn/jxw683vK+Rg/ZUMp/b52fI5FLrZXNTSl61
6U2B5FduaBgIABdnIagzgomoHkoBKCSz6SRzktUaJo9rmU5r8zxNeeHYWMZ6a0seQ0aXlytbv1DE
EUarI7khYBoNVGmiJl/5O9B9NhD+pboe22NKekLJ4/I23eGV32iHco/h0iBpPcURSepIdgRg3lQT
0JdJxByjI8AsC2RiTklMWZ8TBcPfCk5KO0EnavrM6ulvMmp58CnJOfYpDpigiuSP0JVoLAIdA6yA
39fnD5kQSQPxaTPhl9hrQeUUqqPq4HHjjH/yczv9EeK/oEMBw+3gq16QnGOL4oBINSRvBABLBBCz
GhYfTCW6CXQEFKt2UxS2ArigYfLvpgGOtLabaXf5wBXNDQME9aUc7cWKAzaoI7kF0PsgEFlGQY0l
opTKXnEj4XorWM45Y8b0+ZS6uECFA9uLIgujwDJRdDdB4Vz4a2SrTwLFAGAO7QWA/2YE49MzCmLn
eCAQ+arqqsq+uJ8Yf4QuNrONutXdnAXDTWHRsrSxdHyvhuS/gFYHbw9DINHTJV6AxQ+eR+lT9PuG
XwRjvFDhOcWb9NJPV648IDrzhdukmx1MWTDZ3DFGluKVjm1qSG4LoPCBk8CDA2BmiknwnCtKfrEj
1oTuAKwaiSSHY6VlFdGx2p6zrzTNCkAsvcXhps8U759SR3Iv+Cf9APNNq3pdTSZ1wCDyV9VjI8rc
UrTY5pg2yOIovkvuDIjnEQ0kDwSA1sBWJiUZtwbYtQxWD0lh4U1XfBWEvhV1+yR3gmX/o86WnnGr
/bJouLU7v12wBynePaCG5KMcpoKqtBs240tfAHBP9cDcWfDl7cHfRDsAWIdrpaGX5tkSG2btE7c4
bxZNtvXn17g05WR+Vg3JBYCOL+gkEPsIgL+d0gqg52goyFbz4DpXNaHGjP71r9+8f7OJbdjTue5Z
NNptGE+GfqZMUjxUR/KDvofpUaP+MZo+WsLGZ3azRhBujc+ypFluyPpo8JdW5EJknxTJFQHLunvA
30S+AJJLoB71QLmKPMeM2dJ4nQR2N8JnehnnZ/GOAi2vXq6WDcPzT2OzPAeU711UQ3LGxCpiR9wG
AC73gD2/A2zNcRPAO+LgdKYjwzcaaZzMZJ8Oty5WyobJBSMzRBkj6YhRQ3I/cx8H0Z/AFSoIdCfa
Cfy3MwJE8hFERI4gtBsMAF3V6yz+bV885qlnj9jjpbNhdPGV7MK9SPleOul1ljyvIh7Kj5caNYVJ
i722lkO2hiXsHyKif5VLZVt1JL80wX0b/Q3EEtUCApYCx6k8InqLpUPirQgaBLxcBYTq8cXSUj37
DbVkz/ISUanMWqYY0VpcuEdnGiEoclDLsmxJXPU9QHQyL50V38EKwIJq2762AqZmh9R8jHevXUb0
MWfP5FIr04lEvY6FEacau7ATEZ0F/qR6wFai5gih6mhnlTbWe2dF3gIoUxLIO0EjV/A23jypqyl7
RpffaFffR1lIF8nH1ZuhMCMOOKzy8kX3Pdm8tQBg6p9ENEf5eg91JDvCCpwjugicI5oOlCuI5pRs
wqJZcBOiTXctAOC6P2m4CV7j7mt9n+SI7h2zbziCTrLlJrLxQJPVSaOYKtj0k0UAoNMK2N5RFbTm
F8YpQQBga7p8eH4AIT8oAmiJYUOb5sDkMueJntuUr7/W4sIR3QZ20DjgPj0zYwkFo/fHsl58kQBQ
8+VwG2CbohalfVl/Zq0cGF35KtF9JcK89ZG8Sq+U+5pa7unZIlspE3Iyih5R2YOIYlMqTPunwBFi
RJDNF+kYcEpaj7zjm7omAC6fFARgqr1ClfBxsXT5nBluGpB4WeUMz7SRPJbVdrtoQkS9ejHDUIRo
BmzCsmzfz/56o5v6AShcq7nE2btYKIcmW0Ykn1T5Iu+0kfwU2EQ3qwwlWgXzEyqLAfXQUjb9NbIA
nv85Fu4J+C+W3+vMMz5lc2a0dXr6b8pXC+kj+UugqyaSL/AuHRERPUwicixtlQ85Hp1iJrrIX5ui
h+Q0pk1caaYCfDG80l9ZZFyFGvDl+00iYIZ4Yk6LIaJrMy05tzxg++kQBSxIG8ljWCR3QplMIgpB
S6JYs1gb9xmwelE18S+/kYgo9T4R/T3F7Jdjk4N+O6LwBJk9pzqSX/N50s9RjygQoyNQTkn7BeDS
43hHG1Caj0m9iSOi08Mtvjk2utwfuwPlr1XURbLDF+r6luzvABH34KA1756kyfkKak9ZRQxbmnf5
KXkMYJoeklPdMIWIdvnDdIFoACoSleLqRFg6hRvmJO8bG8YFBs3h/LqXGYqwt5s76wC5gKthyxtf
l4fhzDpIbsroW8S7MjfSBdOIKFSszfoYOE0U/11NBsz+Ie53iehdCMLT1rVx9da0o5DVsMlhN7rJ
XvHTQXIiE1YkemzGGqIATJkNmyw7tBfoMtwVsLRb1xhAaaZmOfkj9MuMauGnbVkx43N19ztyFvin
uki+qEdpsgeKM5IvAgCXGWZtW9rdtotIGK3HuOhaWn1fT5siKquJ5JOMKuG7QviKiBqgOVEHTj6J
GcO5IMzdKUEMY+TjJ0ImDfWaaRsesC5NREzwXPqpruGe0zd+LP24DpKLI286EQ1B0XQiessohKyB
SYj2ngbq24mILk8pAwA17rHfBs3qa5tcYku6iJbgs0I/TJB3/tri0l2VDpKvMkRgohWwJhCZEbkL
eKTgW3rFPR2XBzDV+LwQ4LmBWDpMRDlto0v/Yp8u/Fd4WaB+tGhpVEEFPVkbyRPgYoIG54sOAcjD
e0HTAUDbTPNcCQ22ZCKd1w9vWAadkvDoNukhOQrAz0RbmdBmJfQiWgibODPdRuBZ27fUrZG00Ks4
G5t87alveeVYonQBnKsoY4T+hrb03kjx+llaB8muwDGiODemei+Wqc9MchfF8M8LzNnEM3srMMVk
8S76Jtd5xGU1weSI3vXXn+0qHJwq3qpW00HyfsDykIioMWoSEbAiRlSfzowZDBf4dWRJAJ9OsCGI
iG6Z9Y1umkD0wof/+Y/Ri3D9abnmn2PFAekwXSQ3R8NgFfopM/qJg+IZxfRnqq1ERPaPODtPEVFi
PScuxuzNZYT/DuohuR+A6kT1mG4tRTCY6L9SHaGeACTN2G9d10x7SsbnSZJMTTMHEW12EptptK+P
VUIg0kByBoBJRDPg/ZqI6AzwJ5NMExQNrwCw8VP0k9PqjFfpaJkq+V6diIhWOlm4vzwQIXyp5jpI
/p5lVCS6YiERWbDJ4Stv8zIeQN63RJT+Uzkg8nL73zmmp/bo4SAiGsf9O0yNSy8f7feFCV+rtx6S
k1yxaDw8Nbokl0ETQURmp/41l0npg4zsz7OCTizNM2El77xe1kNyU3gADy8wfnuaK2YQZeSXFMOO
ADQE1T/RDxOzmqpszZg/w4Cf4syNC93ZkvuzjTaSHwNoQRnFWV3nHWy8ZSdMvBrOLQBy2We7/rpb
gd1fsXNLIKMRO8yZyY128QTLXjpIHg0gnMmnxBBRcaygqvKyosEoYGU75jhmshq1b7x0r85G/+NY
z78cw8kNd5Z1D9vFt1Abr4fk48C5jfK0Op+rAdYH8tuqZrpXLMkGQlnyfrlUzeySnIN4cjjrLsfq
Ibk4wk34sT0C0ojoGBPoaCVB7mhAtYkL/aVvAKc4ecIMEZkjw2nix232vrKiQLg6kmMB+GZGM8kT
ojWwpBIRpecXIrRXgErII2VXOWHUc8x0JlJlYlf+1OLOTPZcvJvdZn+jg+QIMDTOHqhMRBkBmEfD
ECS9p6NRtTP/m3OkPf2Ln5IcZmH/TXBKlPNZt5HdxM/WQ/JCuKWfA46qInkXcOtrToAqQX+95UNo
Vz0B2LgCr7QAp1B2WXSvFQBxM2clku1WRIWgtSsjXzmXmUkXwlUEgLkI8OZ03yRjqP6qkC7h8EZw
/y1wvruu/nC1L8CKSKkj+QYAXKIZcznueFFuTsvH8Z7vAQuFRojM6KYfCcyUMHj5+XySAbpZ/Hde
YDweTSR/CQvwhKgsxjDJsSr0HSzSmtfpKH5Grs2nP9XxcTx7HQCCUlRf50a3ejLDBoBJNGohuSMq
01sXtqGZfHyNUrQBpnjee9IZfDXpOjOAgjzfb4iBeMtnB6I8IW7Dq0TyNeDgZJhgfsg498D3RNdE
tZ5Ei+A5CAFKknmGPg2ZvysvygPAQC5hHGcgXuQ/5c9aYFGojuQzgE2U8ongkHzNlO8hl+YDDnaG
i5jR+NZb97K81m98SUD0EFw3Eg+fdawSwOrZqiO5HMJM2E7pHtjNnvT4Gbl+zxQEUm1pn6cEfRoO
vwbdLQJAQNwpA0YHRh0pD7YOTwvJZdGHqJKGDnIdRNAVrpFBmH5ShqMwPfHIG7m5qbl4smQBdBq8
rwFYr2VqI3kLcPccUH8M67+bUIGIgsQCSHNh/YepW5cOJ+V604RnrtmWafkF9niwEcNdWrkAg55r
Inkz0BtuPFexEl8e8BdfuvUQOPTYWyLG4MS5WCxM3W22TvQVpH8DjZjs2soCjHmpjeSCGB2AcUTR
i+wsG3tM2sziUuX02ShAG4Abopd+1r8sb2Qj9Ng2Ko/pAJfQ8DFitEdLEzA3SRvJbz3xA1FbdVns
DHfMIYcvuxL5617JN4VfLHYT0RRewSxG7eDAwMDAwKDg4ODg4ODiAp+nnjaSJ8GHyJcVms6wIQy4
RDSKCeVyEQpXqq/SMtCJm8A1+rzrXiuT6FEA7xB2UAFBQGBgYGBgueDg4ODgcsLEaT2uheT1wA0X
oW9oPpWOFv8Ch2mphG7mxE3gCoj/cWlMRDF+/GaiiYo3z5hcPjg4ODi4rOCUuv6theRMK5bUFZTG
VgMIVtzU7wFK8pL0UPqPvtFcGO+gqT0RXXSpT5qkSLgXCQwMDAz8JDg4ODi4jKfg59/URPIx4DzR
UJnOurCpPkLUlOmWGuds/1Omab+RY7+NKuFNRPQE7e7GP46NvX93nQoepJGSzKT4e6c2L5u3qoiX
NpLboybRZ6zC2hngqBemEMX3FpVfLoGZDqgwniKc8BL8a3QdNmrCjEFMBrkXTsc9+ff+vXsxKls+
WSfbtPgnN7ZsWrCkL5ZoIXkBrNQGH7HZ3HdQKbiPBg6QvRJChDKEVk5MLliz2/BRE2b2YkLwbS2X
4p4+uB8Te1tlyyfTRHz36PH1zT8tWBqBn7SQfB3YOwmenHzNRI4iJd/dEXWU4MZJga9LQJ2eI0ZN
nNmeKTyu6XEj7un9BzGxN1RqSmSaiKlxj6I3rVu4rBV+0UTyfNjeEs2Fq5pUzwy4JhOtYBQydxlN
MnqzWNMZV2937DsrMmrRopnjevXo0CmsVdsBQzp0aeIToI3k0hhMNAGuaQxmXd61U7SE3gC8cdRR
doouY9Ty5WxNmPYId8zuPDpyUdSiOTMG9YgI69jiq/Ax3Tq3qIb1Wkgeh0J0gJ+Q7gBwkUc8rwAn
iXaICnKpkFGTNxEpBXAkY2Dm5M7jIhdFLZozfUCPrmEdv2zdbUxE52aVsEcLyYeBy9eFkE47lIQy
dfYDbESbxBwuu6dRo38jpYqTdExNG9l1wryoRYvmTOvXI/yrji1bfD0mPLxJeRzTRHJbNCCiPVKe
EDc+RnsiemLFOpK3DdcbcURsIbTWiG6mG3FU04WzYTnRKVayYAhCaAXcZBSsrUAC/SJrSENEXkYN
H0xEt2x6R0Sc0HjjTy0kd0QtygziqNd/MJEMhR9/gcheWvCUMw3f6wlEdEGXbj14r8Ybl7WQvALW
VPqE36bWRkSwUqTqB1iI3niIWk0kGDY60un2afoGjTfuayHZUQCTiOi5ShsfoutslOULtCa+r5kR
FtPPx5xwgNsqnfwvzOKFV4HkGOAwUYYXE7ypil50QVEP+SNMmUSKQr4k4/zHUcdW6WdySilzlgWr
sOhUR3JldCSazTbxpCjYzIqWyYeZh/97AeNxhk22TTy+RJ93Vl7phharADDED1UkD0NJor58ZW8x
TPoO1tdq3gUNRr63IraGweEx8+R8/Qm8ilJX4OOSACzPtZB8jl33CqsVpa4CHhMRTUZ+O0u6MGRn
fr3UuwaG+tYAqm58ronkHUwNV318QURJFiykFBu/V2PHciCVaCNcpHIEGYatKGbKsuH4ZLgFmHxO
M3aRD9OJYk0sp2QIqn0kqdli9/xviCjVny/+NT69BWb9XqPKYBMQeUkzdtEWjYn+YEq6iN4A2y4r
2wtFwUJEd82CfNWd/69G1+8DuEZFa8YuhiNfGkMVUelb2IHtQHsEuEY03+g1Z2Qc2Xl2QZbM9FlW
ChDvgxVIngJvOxGNQwE70V/Ar0Sfy1PTK2CyEyX5YpR03bEZtKLA2/u79t4pkiXL239jAtzeacaT
45mwYG12uWmMjl0VPSSXAxkMlSEvG7p8axgTmXd2/n4ja6Trnn3B0ZPVkRyCgUT2oqzjcAK4leEv
5c8S0SJmP9RM2JY8MmpARce1nfsv2LJk9NBO4Apg1ZFciW1iOVJUoc9vzPOwDXRS3fCdU9aFEBlO
ISJ6F5AFMz+dZpNS4ZRI7sGwdE4BZ4kWwe0N0XIhhs3dXF+GfuEjXQqNlkWuZcuGjQ/rxJrgY1Sq
SP4NeE5Ey+DygogyvTA9Et6yGz0NTMzmoQvfza6gQQN2CNEFg8N1WgjA67ypITnJBWuIaDAKZTJZ
yTwOGgJfWZ3vbKYV5m7BJco0uOMzn2azbsaH56xSQg5LFcnJLizFaSugUP88yRS/E1EjtCZOn9bp
+J2yCoiGoy38llYLyVWYNdmeD98ShaGBxD6en1qUiOiJTcbb6mTMjNpMdjDReI2G50yGdxGqjeQZ
DG4fMM1abgK/HwdkGqBDuERuOxR8ZyTvy48mTD7yqYfx1W92SXFiUw3JZxgFgePs1nk4ShEdVVQt
D2SorGkFBFWG2sZMYKs1Y4xPyoXnFRGlNlWRvJ/b6F9U7vhpFqwsAf9beGaS3ZjcAgfBNKPKM6YR
X4uCMxpIzvBk81rt8Dkle2I6EaXmkXV5GoBPiIioD0cVE1FpDYxj/Pc2OMqu94MzJLdHDfZJ7EJE
vwJxyTamAZ843l2LZzr9xJWxGprd/hYqe4yNkHV54AzJ7KyWWZSJXjREK6Jki7wXYAs20DQSnonG
CRQAXLi60P5Gja65zAZnSJ7EeJ9ESWYl86Ilr8T4CxCjmvJSGbsEXBnbfC/kA7hntZF8m8P5JPg4
DrHNGavK+nuHsYg4JmPEXTRkSHUSaBCGRp0lHM+gtTaSA9lg1nh4pRJNRAGiFvKEaigvj1AbFRzc
VzAwvhAznI2MxvO4ibCbNpJnsCJw3Znez95YRETVZPf6pTvL1Yox8aU+ewzZ0E605hvbikznooxD
tJFcgwfrx+JWX5zbEylwbHcwSjnOKTV82P+EsXDBkhL831e0kXyAK/H+Dbi0HLZkJjv6kYyKxXyb
jMIyZfOyRiwRcmH1DFnev6+M0KOG5BfACq4u5ABROKoQrYVZWmxWkg/K7uFIiXZD2wyBiVbR0Oo3
TMh2DtBGcigLtpUwvyaKZfzFwTKNw2UwsY1FPuP3fO8Mrdp87YmjmKGo1kQh3zlWE8nvvDCDn8/q
Kuv7zgt7pn5EyUYcyNI80TnZiI5S7SjRUdHaSF4IC7PjSHTH/AFs+d6Psq5UX3DdwkbCP0XqJxkY
K42SHhgXOVKUC4zQRPJ3wE2u7ncyURM0JboJ7JOxW7j1MDOQaxo02ojJWwzyVpmRd1EtaRJIHckJ
VvahvgNsJ/oRuM/4ofektQuNBeYv5+T0MGL0viztX/IvCZamgdSRfFoQVx+r0BWNgkuqUFcUTEQj
DeWqwxYeenw/euegIAPP28gJ4rjiTW0kt+Yf/EaoH8qujY+kkHDwnTFvmqVsg6dGdtWmcuM3/xMf
eyqysYGtSOlNJRV7GBUkOwL5vF0b1CaqgR5c4l2SgOLnqXlwYSo/77kY8ZMrTtx67dG9E3PrG6iy
rrhRLEb5H00kL4U5nvOM+hH1ZBoUJVslqmpXBDptRjG+ZvmsoUm22rTtN+Nijk6rY0D7J3SDWNBu
kiaSl8OFowSvBeLl0eTPRXR8awrRqyC811F0t3SLru1dpHnyZU6z4OXJZcmKShoxHBXaeYShvENW
CPw+h2nABkkGXNO7OCv84IvhlvGvC5YS0UCpfv182BJ5b8Sda7g84f2abB75vYQ9rO1dVOebIPdF
acrMzz52wUwRHXE7Us9k4W+eB9n7/RptnTxPkojX9i76CDz+a7IW90RBIpX184xQ/Fn392impXuU
LLyhjeTdgqfzN8Cr1rfCV5LSan/O/zxngqRFgaP5+7zBHy1rJp1ONJE8CvnShVt4dgEsD5gNtKh5
nf1jcXp9IHyZ9G9GvfdpcrmVDaRZNU0kRwM/CHWbj45yMZ2vUUbklvqJlpVoE++wp1Z5n0ZXXSOL
62kjWSR2mSinOz0T74HSPJgag72292Zm8JnJ8pdiNJHcGR9zc2ymD+D2lstQivT2b1vY1YdxmWtJ
vk1K3fdmuG1CtHxt6qOF5E8QznvDeTG3DkOeTvEQqPJEhyU+0lUTJ9KX+P5Q4T7jknwHOVILyWOR
j5thn5qxpQOKZLKxOdNjMcNF1E6gpsAvel7mvRmdZ+Gf8h3kdC0kvzQhSlQnMEjy5hGJ6FptNpu9
7j2Z6TUn/Y3ixXtaSM7wFmWym/CVxbRLvEvsBlehrHaTvHPDm3LvyfK60TRc/lpfDSTfEVew1EM1
brqrKTSDJQpHMbsk68pR3BNKvCeTG99RyouO0kCyvZgoiFUBbdy4LFO0OFQSLKlniIQbn1SN9X1P
RreLp5YKHoQWkg+KCYZtZbPYeOTJFBtre8MWIr6P8dl1ogzFd9b0Lg6KqgRpCfh7fV+0fY82ix/F
N25ydfP7Rd6H4T7fZarUoGh5Fwvhlkrieka2ufAIFBZWC2+hxRlzgz25Ittb+d6Hyfm/dxB9K39V
y7s4Kq7XGwTA8i/vbQ4V1aqL+2vFmNhQIxHR33neh9EBG9UyLZrexQTks4siVR6SN7+UxO//4bLQ
me1ybqb/WvbZMYrkofDJFEcHuY2dw0sQ3O8Bd3EnqM5SoS0iOuaZY8NNYXGq9EUtJNcQ14es590Q
OiRaS7bLsqsxItbnr245Ntnc+alqFkILyb3EbVo2A3ywjdoJEYA6KC5hYTQWV6ZusebYaEvfN2yy
0SCSGzFlTXzGRdKzpbxkr5rhxuk9ZeQUyuY+r/msqHRc1UJyqHA/idaKSovr8Nmx++7SpsRHJSlD
BsreObT848MalKRwdSQ/hlg/+zJ4zL7z5eP41BalpOrfrVCaf+E3jxyaXJ4VlVBUaAxUR3JKHrHM
wi2ItPpmwTONL4ZeKytxMItU49bnFMpcs583ctHJcRpITrCICb6pVsmWLwFiXVSi+vz04ojyyomZ
LXm8vrIYnJOfmsWmfSXqR9KP1xNpzeecWCuDhM0WN27WydEqvZwvURlsbE7+UVL+lmFBEQ6i3fkV
44mLaJ/KlZUIkmCXc7TtC1iXoVXLqDEnHxD7cUSBsNwXuRSnOacjQEqMS3bHXNG/ZyrkxOgS2/gH
uaHBOfmAqMU9EVUS/CBmM3VRGpF15b36B22yrfJc4RdJOYV0aOX4doolkl66Aq4pwoqdwH2XIfLE
jsdLRYXM6oLZNdw26JmEGi8ZGjm+MIno8Fs3VouNOcE9zkxXWasOeymOFEFElLk42/LJbiNEN0De
+1AjxzcYhcXbzxCR8lCmG7fClJJLf1F7VvqTe2inZ9uTyzvljXCembI3tXJ8U+AlVjkJ57SG2Gyp
h6RKLlbSVudc9pT3fZZK6l7Hyd6+rYHkIeJQ5mqYBSnEO2wtg70y/N/IAjOuWKMsHkienL2QeFNJ
qlbe7KCXKpIfWiQz1W4IigYP+KDt58p2h1PgLe7K+npM9kKfbSQNZB/IJp9hqkh2FOKSf1wS1SQ8
DXXRnoiILilKn2mnvBLt+dBsuRimLs9JujuTjCkaSG4n0QqhObClidnpMjn+UmKNFHKc7J1lSPjO
keoxKeqvNbyLzHziqEQTNCjJrx7prsz2aD2kHV0Yf0NV7TAhKutrXyN5/+D6RryLYbDGS5h6AP9P
Cdarv2biyfViDoFUEuNpZNajtF9elJ01xIh3cU566Z8gDiOOQEl2zQ+Qd9dK81eUdD2YmuWMsKnT
TZJn5wx5F+VEIk9MZlXouXZXLMtNTDGfp7R/c8zQLO2grH0VIorpboZ2fH+LUwfPbFgsahVQH+2J
KLEoSitEtH6B6YoalCljQ40s3eDSyo7mEwzs+OwBYi+BHpkhCnr2Q4CdIdH4JSrOXk5RyZy+JiRr
aSdll6PBRnZ8XZE/VeJwittp72dWzXNM0YB0DEMBRe+41GXls2R0tZOK00YY2/GFShnfKa7Chpp2
wyJz324qtKApZeeQT4zi+Gu1Zlcyj/6IOpKnwiNFFMsy/bsOJs6V+AYlGF66QtiH0n1luR7RuBrZ
xmiAq+x2FSWQw7I5Ww3Jf0o1ZObDwypwwHcxqL5rlsjm8nsSF2WXyX9mtzLqZXz6q0ovNBknt60a
kp+7Yqo42AMEssULRESvrFhOREOQX9mM6LTaL0COv6Y2N+plVD+k8jutVfPiVKJwT6S/USURmXo8
TyMRkXzDVa51oqsB5z5Io2nbculh69WRXFsgZxM1Rx2KEW7bargRXXIRmoJJWC6+qaQ5XkYZmDFc
Wu9RFZbOkCqLlVND8hj4iaap1NL4qqbQmSg1L+YQ0Ri4q8j9xplVZj0iehZZysA+r/1+NREeSpb+
TjXVkDwRnk8kHLJKa2B6Kkr09iJK9VZtQhuiig4iejjVQKrSs8sfqs2OE6TPQRNtpr2UkDw9VuC7
mpfL3/5WovAq8pEuzGugN8H59z9u17jifWmg+aQqkmPNPKWF6K0NC4mK87PtH8Bzagi/Rypnj5YI
eaqMx7v76CkCWkJXPtX6aHdpZFENyZXRXfSBBcCBUSKaSBfUI3rpJ2riIhp1UVWjz2vczz30OPjW
BusStExuLTmyqxqSAyU9yh97YNstk2iuHYWCdvpVEmPix7w8EzXv871NXfU0BV2bbHqj9VFp3HSQ
QSRLxivFM3LR4paodXTGte3jWlRRlgzka7Hgit5VpvXr2Duia69Bozccv3dbXLEkQvJkuAvX3Q7E
EA1EEPvSQ+DPJya1KAUR1UWI0+/59OjSbnU+VjBlTZUH707S+djlfl3De3fu0W/o3F+jY0+lqiD5
irTLYSmE0X6BDk7bi/YiWgDbVbWz/4RiKdrXjj+yOLxWSUUg1Fz9m191PkVn+4V37d25R7//zP/9
+r1T6WpInuwjjndcdqmcRh+JJBe2AX/RPyXUUZSmf5//PTS/Y03lvGGrPeKQ3icP9O/SrVfnHv2/
Wbz/ZszJzOwgWWUkOj0i4crBeX1b163auEVoaJMOk/ZEZ2b3WiIkfyZWxJ2Ohkz0mIvDff1NgmPV
SNWlif7xCE03drl3MX+tH9OpUZ0aLRqGftay76qTL7NhsxTJi/h6XiKitCC/G5SUX1bWeb28+qLh
2HLX6dVS755dN6pDw9q1Wnwe2qDVgNWnE7Nzn+VROOmilkDUQeQ1JE3YmUGUkZF9CKXcOv39yHaf
1wpt0SC0YashP5xNysZJco7k/8shQnJdcc21fesDInJ829fQWdL/T22WIvlGo3HiN5MeEdHtI/9r
91ldP1nszpxJ+V+z+YNF8vNDmR+GzRpdzP6nh1Mk/w+ODxbJH4zNuUjORXIuknORnIvkXCTnIjkX
yblIzkVyLpJzkZyL5Fwk5yI5F8m5SM5Fci6Sc5Gci+RcJOci+X8LyQFdPpThySPZ94Ox2YdHsucH
Y3NBHskuH4zNJYCK+HAGV6rQ/AOymav06PYB2cx1QBxt+nBsrvD/ANIP5RKCEJXDAAAAAElFTkSu
QmCC
--f46d040a62c49bb1c804f027e8cc--"""
class PyzorPreDigestTest(PyzorTestBase):
# we don't need the pyzord server to test this
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
# no argument necessary
self.client_args = {}
def test_predigest_email(self):
"""Test email removal in the predigest process"""
emails = ["t@abc.ro",
"t1@abc.ro",
"t+@abc.ro",
"t.@abc.ro",
]
message = "Test %s Test2"
expected = b"TestTest2\n"
for email in emails:
msg = message % email
res = self.check_pyzor("predigest", None, input=TEXT % msg)
self.assertEqual(res, expected)
def test_predigest_long(self):
"""Test long "words" removal in the predigest process"""
strings = ["0A2D3f%a#S",
"3sddkf9jdkd9",
"@@#@@@@@@@@@"]
message = "Test %s Test2"
expected = b"TestTest2\n"
for s in strings:
msg = message % s
res = self.check_pyzor("predigest", None, input=TEXT % msg)
self.assertEqual(res, expected)
def test_predigest_line_length(self):
"""Test small lines removal in the predigest process"""
msg = "This line is included\n"\
"not this\n"\
"This also"
expected = b"Thislineisincluded\nThisalso\n"
res = self.check_pyzor("predigest", None, input=TEXT % msg)
self.assertEqual(res, expected)
def test_predigest_atomic(self):
"""Test atomic messages (lines <= 4) in the predigest process"""
msg = "All this message\nShould be included\nIn the predigest"
expected = b"Allthismessage\nShouldbeincluded\nInthepredigest\n"
res = self.check_pyzor("predigest", None, input=TEXT % msg)
self.assertEqual(res, expected)
def test_predigest_pieced(self):
"""Test pieced messages (lines > 4) in the predigest process"""
msg = ""
for i in range(100):
msg += "Line%d test test test\n" % i
expected = b""
for i in [20, 21, 22, 60, 61, 62]:
expected += ("Line%dtesttesttest\n" % i).encode("utf8")
res = self.check_pyzor("predigest", None, input=TEXT % msg)
self.assertEqual(res, expected)
def test_predigest_html(self):
expected = """Emailspam,alsoknownasjunkemailorbulkemail,isasubset
ofspaminvolvingnearlyidenticalmessagessenttonumerous
byemail.Clickingonlinksinspamemailmaysendusersto
byemail.Clickingonlinksinspamemailmaysendusersto
phishingwebsitesorsitesthatarehostingmalware.
Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,isasubsetofspaminvolvingnearlyidenticalmessagessenttonumerousbyemail.Clickingonlinksinspamemailmaysenduserstophishingwebsitesorsitesthatarehostingmalware.
""".encode("utf8")
res = self.check_pyzor("predigest", None, input=HTML_TEXT)
self.assertEqual(res, expected)
def test_predigest_attachemnt(self):
expected = b"Thisisatestmailing\n"
res = self.check_pyzor("predigest", None, input=TEXT_ATTACHMENT)
self.assertEqual(res, expected)
class PyzorDigestTest(PyzorTestBase):
# we don't need the pyzord server to test this
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
# no argument necessary
self.client_args = {}
def test_digest_email(self):
"""Test email removal in the digest process"""
emails = ["t@abc.ro",
"t1@abc.ro",
"t+@abc.ro",
"t.@abc.ro",
]
message = "Test %s Test2"
expected = b"TestTest2"
for email in emails:
msg = message % email
res = self.check_pyzor("digest", None, input=TEXT % msg)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_long(self):
"""Test long "words" removal in the digest process"""
strings = ["0A2D3f%a#S",
"3sddkf9jdkd9",
"@@#@@@@@@@@@"]
message = "Test %s Test2"
expected = b"TestTest2"
for s in strings:
msg = message % s
res = self.check_pyzor("digest", None, input=TEXT % msg)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_line_length(self):
"""Test small lines removal in the digest process"""
msg = "This line is included\n"\
"not this\n"\
"This also"
expected = b"ThislineisincludedThisalso"
res = self.check_pyzor("digest", None, input=TEXT % msg)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_atomic(self):
"""Test atomic messages (lines <= 4) in the digest process"""
msg = "All this message\nShould be included\nIn the digest"
expected = b"AllthismessageShouldbeincludedInthedigest"
res = self.check_pyzor("digest", None, input=TEXT % msg)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_pieced(self):
"""Test pieced messages (lines > 4) in the digest process"""
msg = ""
for i in range(100):
msg += "Line%d test test test\n" % i
expected = b""
for i in [20, 21, 22, 60, 61, 62]:
expected += ("Line%dtesttesttest" % i).encode("utf8")
res = self.check_pyzor("digest", None, input=TEXT % msg)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_html(self):
expected = """Emailspam,alsoknownasjunkemailorbulkemail,isasubset
ofspaminvolvingnearlyidenticalmessagessenttonumerous
byemail.Clickingonlinksinspamemailmaysendusersto
byemail.Clickingonlinksinspamemailmaysendusersto
phishingwebsitesorsitesthatarehostingmalware.
Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,isasubsetofspaminvolvingnearlyidenticalmessagessenttonumerousbyemail.Clickingonlinksinspamemailmaysenduserstophishingwebsitesorsitesthatarehostingmalware.
""".replace("\n", "").encode("utf8")
res = self.check_pyzor("digest", None, input=HTML_TEXT)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
def test_digest_attachemnt(self):
expected = b"Thisisatestmailing"
res = self.check_pyzor("digest", None, input=TEXT_ATTACHMENT)
self.assertEqual(res.decode("utf8"),
hashlib.sha1(expected).hexdigest().lower() + "\n")
ENCODING_TEST_EMAIL = """From nobody Tue Apr 1 13:18:54 2014
Content-Type: multipart/related;
boundary="===============0632694142025794937=="
MIME-Version: 1.0
This is a multi-part message in MIME format.
--===============0632694142025794937==
Content-Type: text/plain; charset="iso-8859-1"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable
Thist is a t=E9st
--===============0632694142025794937==
MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: base64
VGhpcyBpcyBhIHRlc3Qg5r+A5YWJ6YCZ
--===============0632694142025794937==
MIME-Version: 1.0
Content-Type: text/plain; charset="cp1258"
Content-Transfer-Encoding: base64
VGhpcyBpcyBhIHTpc3Qg4qXG
--===============0632694142025794937==--
"""
BAD_ENCODING = """From nobody Tue Apr 1 13:18:54 2014
Content-Type: multipart/related;
boundary="===============0632694142025794937=="
MIME-Version: 1.0
This is a multi-part message in MIME format.
--===============0632694142025794937==
Content-Type: text/plain; charset=ISO-8859-1Content-Transfer-Encoding: quoted-printable
This is a test
--===============0632694142025794937==
Content-Type: text/plain; charset=us-asciia
Content-Transfer-Encoding: quoted-printable
This is a test
--===============0632694142025794937==
"""
class PyzorEncodingTest(PyzorTestBase):
# we don't need the pyzord server to test this
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
# no argument necessary
self.client_args = {}
def test_encodings(self):
expected = "47a83cd0e5cc9bd2c64c06c00e3853f79e63014f\n"
res = self.check_pyzor("digest", None, input=ENCODING_TEST_EMAIL)
self.assertEqual(res.decode("utf8"), expected)
def test_bad_encoding(self):
expected = "2b4dbf2fb521edd21d997f3f04b1c7155ba91fff\n"
res = self.check_pyzor("digest", None, input=BAD_ENCODING)
self.assertEqual(res.decode("utf8"), expected)
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(PyzorDigestTest))
test_suite.addTest(unittest.makeSuite(PyzorPreDigestTest))
test_suite.addTest(unittest.makeSuite(PyzorEncodingTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,30 @@
import unittest
from tests.util import *
class GdbmPyzorTest(PyzorTest, PyzorTestBase):
"""Test the gdbm engine"""
dsn = "pyzord.db"
engine = "gdbm"
class ThreadsGdbmPyzorTest(GdbmPyzorTest):
"""Test the gdbm engine with threads activated."""
threads = "True"
max_threads = "0"
class MaxThreadsGdbmPyzorTest(GdbmPyzorTest):
"""Test the gdbm engine with with maximum threads."""
threads = "True"
max_threads = "10"
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(GdbmPyzorTest))
test_suite.addTest(unittest.makeSuite(ThreadsGdbmPyzorTest))
test_suite.addTest(unittest.makeSuite(MaxThreadsGdbmPyzorTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,108 @@
import unittest
import ConfigParser
from tests.util import *
try:
import MySQLdb
except ImportError:
MySQLdb = None
schema = """
CREATE TABLE IF NOT EXISTS `%s` (
`digest` char(40) default NULL,
`r_count` int(11) default NULL,
`wl_count` int(11) default NULL,
`r_entered` datetime default NULL,
`wl_entered` datetime default NULL,
`r_updated` datetime default NULL,
`wl_updated` datetime default NULL,
PRIMARY KEY (`digest`)
)
"""
@unittest.skipIf(not os.path.exists("./test.conf"),
"test.conf is not available")
@unittest.skipIf(MySQLdb == None, "MySQLdb library not available")
class MySQLdbPyzorTest(PyzorTest, PyzorTestBase):
"""Test the mysql engine."""
dsn = None
engine = "mysql"
@classmethod
def setUpClass(cls):
conf = ConfigParser.ConfigParser()
conf.read("./test.conf")
table = conf.get("test", "table")
db = MySQLdb.Connect(host=conf.get("test", "host"),
user=conf.get("test", "user"),
passwd=conf.get("test", "passwd"),
db=conf.get("test", "db"))
c = db.cursor()
c.execute(schema % table)
c.close()
db.close()
cls.dsn = "%s,%s,%s,%s,%s" % (conf.get("test", "host"),
conf.get("test", "user"),
conf.get("test", "passwd"),
conf.get("test", "db"),
conf.get("test", "table"))
super(MySQLdbPyzorTest, cls).setUpClass()
@classmethod
def tearDownClass(cls):
super(MySQLdbPyzorTest, cls).tearDownClass()
try:
conf = ConfigParser.ConfigParser()
conf.read("./test.conf")
table = conf.get("test", "table")
db = MySQLdb.Connect(host=conf.get("test", "host"),
user=conf.get("test", "user"),
passwd=conf.get("test", "passwd"),
db=conf.get("test", "db"))
c = db.cursor()
c.execute("DROP TABLE %s" % table)
c.close()
db.close()
except:
pass
class ThreadsMySQLdbPyzorTest(MySQLdbPyzorTest):
"""Test the mysql engine with threads activated."""
threads = "True"
max_threads = "0"
class BoundedThreadsMySQLdbPyzorTest(MySQLdbPyzorTest):
"""Test the mysql engine with threads and DBConnections set."""
threads = "True"
max_threads = "0"
db_connections = "10"
class MaxThreadsMySQLdbPyzorTest(MySQLdbPyzorTest):
"""Test the mysql engine with threads and MaxThreads set."""
threads = "True"
max_threads = "10"
class BoundedMaxThreadsMySQLdbPyzorTest(MySQLdbPyzorTest):
"""Test the mysql engine with threads, MaxThreads and DBConnections set."""
threads = "True"
max_threads = "10"
db_connections = "10"
class ProcessesMySQLdbPyzorTest(MySQLdbPyzorTest):
processes = "True"
max_processes = "10"
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(MySQLdbPyzorTest))
test_suite.addTest(unittest.makeSuite(ThreadsMySQLdbPyzorTest))
test_suite.addTest(unittest.makeSuite(BoundedThreadsMySQLdbPyzorTest))
test_suite.addTest(unittest.makeSuite(MaxThreadsMySQLdbPyzorTest))
test_suite.addTest(unittest.makeSuite(BoundedMaxThreadsMySQLdbPyzorTest))
test_suite.addTest(unittest.makeSuite(ProcessesMySQLdbPyzorTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,164 @@
import sys
import unittest
from tests.util import *
class PyzorScriptTest(PyzorTestBase):
password_file = None
access = """ALL : anonymous : allow
"""
def test_report_threshold(self):
input = "Test1 report threshold 1 Test2"
self.client_args["-r"] = "2"
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(1, 0))
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(2, 0))
# Exit code will be success now, since the report count exceeds the
# threshold
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(3, 0))
def test_whitelist_threshold(self):
input = "Test1 white list threshold 1 Test2"
self.client_args["-w"] = "2"
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(1, 0))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(1, 1))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(1, 2))
# Exit code will be failure now, since the whitelist count exceeds the
# threshold
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(1, 3))
def test_report_whitelist_threshold(self):
input = "Test1 report white list threshold 1 Test2"
self.client_args["-w"] = "2"
self.client_args["-r"] = "1"
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(1, 0))
# Exit code will be success now, since the report count exceeds the
# threshold
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(2, 0))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(2, 1))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(2, 2))
# Exit code will be failure now, since the whitelist count exceeds the
# threshold
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(2, 3))
def test_digest_style(self):
input = "da39a3ee5e6b4b0d3255bfef95601890afd80700"
self.client_args["-s"] = "digests"
self.check_pyzor("pong", None, input=input, code=200, exit_code=0,
counts=(sys.maxint, 0))
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(0, 0))
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(1, 0))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(1, 1))
r = self.get_record(input, None)
self.assertEqual(r["Count"], "1")
self.assertEqual(r["WL-Count"], "1")
def test_digest_style_multiple(self):
input2 = "da39a3ee5e6b4b0d3255bfef95601890afd80705\n"\
"da39a3ee5e6b4b0d3255bfef95601890afd80706\n"
input3 = "da39a3ee5e6b4b0d3255bfef95601890afd80705\n"\
"da39a3ee5e6b4b0d3255bfef95601890afd80706\n"\
"da39a3ee5e6b4b0d3255bfef95601890afd80707\n"
self.client_args["-s"] = "digests"
self.check_pyzor_multiple("pong", None, input=input3, exit_code=0,
code=[200, 200, 200],
counts=[(sys.maxint, 0),
(sys.maxint, 0),
(sys.maxint, 0)])
self.check_pyzor_multiple("check", None, input=input3, exit_code=1,
code=[200, 200, 200],
counts=[(0, 0), (0, 0), (0, 0)])
self.check_pyzor_multiple("report", None, input=input2, exit_code=0)
self.check_pyzor_multiple("check", None, input=input3, exit_code=0,
code=[200, 200, 200],
counts=[(1, 0), (1, 0), (0, 0)])
self.check_pyzor_multiple("whitelist", None, input=input3, exit_code=0)
self.check_pyzor_multiple("check", None, input=input3, exit_code=1,
code=[200, 200, 200],
counts=[(1, 1), (1, 1), (0, 1)])
def test_mbox_style(self):
input = "From MAILER-DAEMON Mon Jan 6 15:13:33 2014\n\nTest1 message 0 Test2\n\n"
self.client_args["-s"] = "mbox"
self.check_pyzor("pong", None, input=input, code=200, exit_code=0,
counts=(sys.maxint, 0))
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(0, 0))
self.check_pyzor("report", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=0,
counts=(1, 0))
self.check_pyzor("whitelist", None, input=input, code=200, exit_code=0)
self.check_pyzor("check", None, input=input, code=200, exit_code=1,
counts=(1, 1))
r = self.get_record(input, None)
self.assertEqual(r["Count"], "1")
self.assertEqual(r["WL-Count"], "1")
def test_mbox_style_multiple(self):
input2 = "From MAILER-DAEMON Mon Jan 6 15:08:02 2014\n\nTest1 message 1 Test2\n\n"\
"From MAILER-DAEMON Mon Jan 6 15:08:05 2014\n\nTest1 message 2 Test2\n\n"
input3 = "From MAILER-DAEMON Mon Jan 6 15:08:02 2014\n\nTest1 message 1 Test2\n\n"\
"From MAILER-DAEMON Mon Jan 6 15:08:05 2014\n\nTest1 message 2 Test2\n\n"\
"From MAILER-DAEMON Mon Jan 6 15:08:08 2014\n\nTest1 message 3 Test2\n\n"
self.client_args["-s"] = "mbox"
self.check_pyzor_multiple("pong", None, input=input3, exit_code=0,
code=[200, 200, 200],
counts=[(sys.maxint, 0),
(sys.maxint, 0),
(sys.maxint, 0)])
self.check_pyzor_multiple("check", None, input=input3, exit_code=1,
code=[200, 200, 200],
counts=[(0, 0), (0, 0), (0, 0)])
self.check_pyzor_multiple("report", None, input=input2, exit_code=0)
self.check_pyzor_multiple("check", None, input=input3, exit_code=0,
code=[200, 200, 200],
counts=[(1, 0), (1, 0), (0, 0)])
self.check_pyzor_multiple("whitelist", None, input=input3, exit_code=0)
self.check_pyzor_multiple("check", None, input=input3, exit_code=1,
code=[200, 200, 200],
counts=[(1, 1), (1, 1), (0, 1)])
def test_predigest(self):
out = self.check_pyzor("predigest", None, input=msg).strip()
self.assertEqual(out.decode("utf8"), "TestEmail")
def test_digest(self):
out = self.check_pyzor("digest", None, input=msg).strip()
self.assertEqual(out.decode("utf8"), digest)
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(PyzorScriptTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,40 @@
import unittest
try:
import redis
except ImportError:
redis = None
from tests.util import *
@unittest.skipIf(redis == None, "redis library not available")
class RedisPyzorTest(PyzorTest, PyzorTestBase):
"""Test the redis engine"""
dsn = "localhost,,,10"
engine = "redis"
@classmethod
def tearDownClass(cls):
super(RedisPyzorTest, cls).tearDownClass()
redis.StrictRedis(db=10).flushdb()
class ThreadsRedisPyzorTest(RedisPyzorTest):
"""Test the redis engine with threads activated."""
threads = "True"
class MaxThreadsRedisPyzorTest(RedisPyzorTest):
"""Test the gdbm engine with with maximum threads."""
threads = "True"
max_threads = "10"
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(RedisPyzorTest))
test_suite.addTest(unittest.makeSuite(ThreadsRedisPyzorTest))
test_suite.addTest(unittest.makeSuite(MaxThreadsRedisPyzorTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,32 @@
"""A suite of unit tests that verifies the correct behaviour of various
functions/methods in the pyzord code.
Note these tests the source of pyzor, not the version currently installed.
"""
import unittest
import test_gdbm
import test_mysql
import test_redis
import test_client
import test_digest
import test_server
import test_account
def suite():
"""Gather all the tests from this package in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(test_gdbm.suite())
test_suite.addTest(test_mysql.suite())
test_suite.addTest(test_redis.suite())
test_suite.addTest(test_client.suite())
test_suite.addTest(test_digest.suite())
test_suite.addTest(test_server.suite())
test_suite.addTest(test_account.suite())
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest='suite')

View File

@@ -0,0 +1,139 @@
"""Test the pyzor.account module
"""
import os
import sys
import time
import email
import hashlib
import unittest
import StringIO
import pyzor
import pyzor.config
import pyzor.account
class AccountTest(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.timestamp = 1381219396
self.msg = email.message_from_string("")
self.msg["Op"] = "ping"
self.msg["Thread"] = "14941"
self.msg["PV"] = "2.1"
self.msg["User"] = "anonymous"
self.msg["Time"] = str(self.timestamp)
def tearDown(self):
unittest.TestCase.tearDown(self)
def test_sign_msg(self):
"""Test the sign message function"""
hashed_key = hashlib.sha1(b"test_key").hexdigest()
expected = "2ab1bad2aae6fd80c656a896c82eef0ec1ec38a0"
result = pyzor.account.sign_msg(hashed_key, self.timestamp, self.msg)
self.assertEqual(result, expected)
def test_hash_key(self):
"""Test the hash key function"""
user = "testuser"
key = "testkey"
expected = "0957bd79b58263657127a39762879098286d8477"
result = pyzor.account.hash_key(key, user)
self.assertEqual(result, expected)
def test_verify_signature(self):
"""Test the verify signature function"""
def mock_sm(h, t, m):
return "testsig"
real_sm = pyzor.account.sign_msg
pyzor.account.sign_msg = mock_sm
try:
self.msg["Sig"] = "testsig"
del self.msg["Time"]
self.msg["Time"] = str(int(time.time()))
pyzor.account.verify_signature(self.msg, "testkey")
finally:
pyzor.account.sign_msg = real_sm
def test_verify_signature_old_timestamp(self):
"""Test the verify signature with old timestamp"""
def mock_sm(h, t, m):
return "testsig"
real_sm = pyzor.account.sign_msg
pyzor.account.sign_msg = mock_sm
try:
self.msg["Sig"] = "testsig"
self.assertRaises(pyzor.SignatureError, pyzor.account.verify_signature, self.msg, "testkey")
finally:
pyzor.account.sign_msg = real_sm
def test_verify_signature_bad_signature(self):
"""Test the verify signature with invalid signature"""
def mock_sm(h, t, m):
return "testsig"
real_sm = pyzor.account.sign_msg
pyzor.account.sign_msg = mock_sm
try:
self.msg["Sig"] = "testsig-bad"
del self.msg["Time"]
self.msg["Time"] = str(int(time.time()))
self.assertRaises(pyzor.SignatureError, pyzor.account.verify_signature, self.msg, "testkey")
finally:
pyzor.account.sign_msg = real_sm
class LoadAccountTest(unittest.TestCase):
"""Tests for the load_accounts function"""
def setUp(self):
unittest.TestCase.setUp(self)
self.real_exists = os.path.exists
os.path.exists = lambda p: True
self.mock_file = StringIO.StringIO()
self.real_open = pyzor.account.__builtins__["open"]
def mock_open(path, mode="r", buffering=-1):
if path == "test_file":
self.mock_file.seek(0)
return self.mock_file
else:
return self.real_open(path, mode, buffering)
pyzor.account.__builtins__["open"] = mock_open
def tearDown(self):
unittest.TestCase.tearDown(self)
os.path.exists = self.real_exists
pyzor.account.__builtins__["open"] = self.real_open
def test_load_accounts(self):
"""Test loading the account file"""
self.mock_file.write("public.pyzor.org : 24441 : test : 123abc,cba321\n"
"public2.pyzor.org : 24441 : test2 : 123abc,cba321")
result = pyzor.config.load_accounts("test_file")
self.assertIn(("public.pyzor.org", 24441), result)
self.assertIn(("public2.pyzor.org", 24441), result)
account = result[("public.pyzor.org", 24441)]
self.assertEqual((account.username, account.salt, account.key),
("test", "123abc", "cba321"))
account = result[("public2.pyzor.org", 24441)]
self.assertEqual((account.username, account.salt, account.key),
("test2", "123abc", "cba321"))
def test_load_accounts_comment(self):
"""Test skipping commented lines"""
self.mock_file.write("#public1.pyzor.org : 24441 : test : 123abc,cba321")
result = pyzor.config.load_accounts("test_file")
self.assertNotIn(("public.pyzor.org", 24441), result)
self.assertFalse(result)
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(AccountTest))
test_suite.addTest(unittest.makeSuite(LoadAccountTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,168 @@
import sys
import time
import socket
import unittest
import pyzor
import pyzor.client
import pyzor.account
import pyzor.message
def make_MockSocket(response, request):
"""Create a MockSocket class that will append requests to
the specified `request` list and return the specified `response`
"""
class MockSocket():
def __init__(self, *args, **kwargs):
pass
def settimeout(self, timeout):
pass
def recvfrom(self, packetsize):
return response, ("127.0.0.1", 24441)
def sendto(self, data, flag, address):
request.append(data)
return MockSocket
def make_MockThreadId(thread):
"""Creates a MockThreadId class that will generate
the specified thread number.
"""
class MockThreadId(int):
def __new__(cls, i):
return int.__new__(cls, i)
@classmethod
def generate(cls):
return thread
def in_ok_range(self):
return True
return MockThreadId
def mock_sign_msg(hash_key, timestamp, msg):
return "TestSig"
def mock_hash_key(user_key, user):
return None
class ClientTest(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.real_sg = pyzor.account.sign_msg
pyzor.account.sign_msg = mock_sign_msg
self.real_hk = pyzor.account.hash_key
pyzor.account.hash_key = mock_hash_key
self.thread = 33715
# the response the mock socket will send
self.response = "Code: 200\nDiag: OK\nPV: 2.1\nThread: 33715\n\n"
# the requests send by the client will be stored here
self.request = []
# the expected request that the client should send
self.expected = {"Thread": str(self.thread),
"PV": str(pyzor.proto_version),
"User": "anonymous",
"Time": str(int(time.time())),
"Sig": "TestSig"}
def tearDown(self):
unittest.TestCase.tearDown(self)
pyzor.account.sign_msg = self.real_sg
pyzor.account.hash_key = self.real_hk
def check_request(self, request):
"""Check if the request sent by the client is equal
to the expected one.
"""
req = {}
request = request.decode("utf8").replace("\n\n", "\n")
for line in request.splitlines():
key = line.split(":")[0].strip()
value = line.split(":")[1].strip()
req[key] = value
self.assertEqual(req, self.expected)
def check_client(self, accounts, method, *args, **kwargs):
"""Tests if the request and response are sent
and read correctly by the client.
"""
real_socket = socket.socket
socket.socket = make_MockSocket(self.response.encode("utf8"),
self.request)
real_ThreadId = pyzor.message.ThreadId
pyzor.message.ThreadId = make_MockThreadId(self.thread)
client = pyzor.client.Client(accounts)
try:
response = getattr(client, method)(*args, **kwargs)
self.assertEqual(str(response), self.response)
self.check_request(self.request[0])
finally:
socket.socket = real_socket
pyzor.message.ThreadId = real_ThreadId
return client
def test_ping(self):
"""Test the client ping request"""
self.expected["Op"] = "ping"
self.check_client(None, "ping")
def test_pong(self):
"""Test the client pong request"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op"] = "pong"
self.expected["Op-Digest"] = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.check_client(None, "pong", digest)
def test_check(self):
"""Test the client check request"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op"] = "check"
self.expected["Op-Digest"] = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.check_client(None, "check", digest)
def test_info(self):
"""Test the client info request"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op"] = "info"
self.expected["Op-Digest"] = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.check_client(None, "info", digest)
def test_report(self):
"""Test the client report request"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op"] = "report"
self.expected["Op-Digest"] = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op-Spec"] = "20,3,60,3"
self.check_client(None, "report", digest)
def test_whitelist(self):
"""Test the client whitelist request"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op"] = "whitelist"
self.expected["Op-Digest"] = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.expected["Op-Spec"] = "20,3,60,3"
self.check_client(None, "whitelist", digest)
def test_handle_account(self):
"""Test client handling accounts"""
test_account = pyzor.account.Account("TestUser", "TestKey", "TestSalt")
self.expected["Op"] = "ping"
self.expected["User"] = "TestUser"
self.check_client({("public.pyzor.org", 24441): test_account}, "ping")
def test_handle_invalid_thread(self):
"""Test invalid thread id"""
self.thread += 20
self.expected["Op"] = "ping"
self.assertRaises(pyzor.ProtocolError, self.check_client, None, "ping")
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(ClientTest))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,200 @@
"""The the pyzor.digest module
"""
import sys
import hashlib
import unittest
from pyzor.digest import *
HTML_TEXT = """<html><head><title>Email spam</title></head><body>
<p><b>Email spam</b>, also known as <b>junk email</b>
or <b>unsolicited bulk email</b> (<i>UBE</i>), is a subset of
<a href="/wiki/Spam_(electronic)" title="Spam (electronic)">electronic spam</a>
involving nearly identical messages sent to numerous recipients by <a href="/wiki/Email" title="Email">
email</a>. Clicking on <a href="/wiki/Html_email#Security_vulnerabilities" title="Html email" class="mw-redirect">
links in spam email</a> may send users to <a href="/wiki/Phishing" title="Phishing">phishing</a>
web sites or sites that are hosting <a href="/wiki/Malware" title="Malware">malware</a>.</body></html>"""
HTML_TEXT_STRIPED = 'Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE ),'\
' is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email'\
' . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware .'
class HTMLStripperTests(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.data = []
def tearDown(self):
unittest.TestCase.tearDown(self)
def test_HTMLStripper(self):
stripper = HTMLStripper(self.data)
stripper.feed(HTML_TEXT)
res = " ".join(self.data)
self.assertEqual(res, HTML_TEXT_STRIPED)
class PreDigestTests(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.lines = []
def mock_digest_paylods(c, message):
yield message.decode("utf8")
def mock_handle_line(s, line):
self.lines.append(line.decode("utf8"))
self.real_digest_payloads = DataDigester.digest_payloads
self.real_handle_line = DataDigester.handle_line
DataDigester.digest_payloads = mock_digest_paylods
DataDigester.handle_line = mock_handle_line
def tearDown(self):
unittest.TestCase.tearDown(self)
DataDigester.digest_payloads = self.real_digest_payloads
DataDigester.handle_line = self.real_handle_line
def test_predigest_emails(self):
"""Test email removal in the predigest process"""
real_longstr = DataDigester.longstr_ptrn
DataDigester.longstr_ptrn = re.compile(r'\S{100,}')
emails = ["test@example.com",
"test123@example.com",
"test+abc@example.com",
"test.test2@example.com",
"test.test2+abc@example.com", ]
message = "Test %s Test2"
expected = "TestTest2"
try:
for email in emails:
self.lines = []
DataDigester((message % email).encode("utf8"))
self.assertEqual(self.lines[0], expected)
finally:
DataDigester.longstr_ptrn = real_longstr
# XXX This fails
# def test_predigest_emails_whitespace(self):
# real_longstr = DataDigester.longstr_ptrn
# DataDigester.longstr_ptrn = re.compile(r'\S{100,}')
# emails = ["chirila@example. com",
# "chirila@example . com",
# "chirila @example. com",
# "chirila@ example. com",
# "chirila @example . com",
# "chirila @ example. com",
# "chirila @ example . com",]
# message = "Test %s Test2"
# expected = "TestTest2"
# try:
# for email in emails:
# self.lines = []
# DataDigester(message % email)
# self.assertEqual(self.lines[0], expected)
# finally:
# DataDigester.longstr_ptrn = real_longstr
def test_predigest_urls(self):
"""Test url removal in the predigest process"""
real_longstr = DataDigester.longstr_ptrn
DataDigester.longstr_ptrn = re.compile(r'\S{100,}')
urls = ["http://www.example.com",
# "www.example.com", # XXX This also fail
"http://example.com",
# "example.com", # XXX This also fails
"http://www.example.com/test/"
"http://www.example.com/test/test2", ]
message = "Test %s Test2"
expected = "TestTest2"
try:
for url in urls:
self.lines = []
DataDigester((message % url).encode("utf8"))
self.assertEqual(self.lines[0], expected)
finally:
DataDigester.longstr_ptrn = real_longstr
def test_predigest_long(self):
"""Test long "words" removal in the predigest process"""
strings = ["0A2D3f%a#S",
"3sddkf9jdkd9",
"@@#@@@@@@@@@"]
message = "Test %s Test2"
expected = "TestTest2"
for string in strings:
self.lines = []
DataDigester((message % string).encode("utf8"))
self.assertEqual(self.lines[0], expected)
def test_predigest_min_line_lenght(self):
"""Test small lines removal in the predigest process"""
message = "This line is included\n"\
"not this\n"\
"This also"
expected = ["Thislineisincluded", "Thisalso"]
DataDigester(message.encode("utf8"))
self.assertEqual(self.lines, expected)
def test_predigest_atomic(self):
"""Test atomic messages (lines <= 4) in the predigest process"""
message = "All this message\nShould be included\nIn the predigest"
expected = ["Allthismessage", "Shouldbeincluded", "Inthepredigest"]
DataDigester(message.encode("utf8"))
self.assertEqual(self.lines, expected)
def test_predigest_pieced(self):
"""Test pieced messages (lines > 4) in the predigest process"""
message = ""
for i in range(100):
message += "Line%d test test test\n" % i
expected = []
for i in [20, 21, 22, 60, 61, 62]:
expected.append("Line%dtesttesttest" % i)
DataDigester(message.encode("utf8"))
self.assertEqual(self.lines, expected)
class DigestTests(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.lines = []
def mock_digest_paylods(c, message):
yield message.decode("utf8")
self.real_digest_payloads = DataDigester.digest_payloads
DataDigester.digest_payloads = mock_digest_paylods
def tearDown(self):
unittest.TestCase.tearDown(self)
DataDigester.digest_payloads = self.real_digest_payloads
def test_digest(self):
message = b"That's some good ham right there"
predigested = b"That'ssomegoodhamrightthere"
digest = hashlib.sha1()
digest.update(predigested)
expected = digest.hexdigest()
result = DataDigester(message).value
self.assertEqual(result, expected)
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(HTMLStripperTests))
test_suite.addTest(unittest.makeSuite(PreDigestTests))
test_suite.addTest(unittest.makeSuite(DigestTests))
return test_suite
if __name__ == '__main__':
unittest.main()

View File

@@ -0,0 +1,153 @@
"""Test the pyzor.engines.gdbm_ module."""
import gdbm
import unittest
import threading
from datetime import datetime, timedelta
import pyzor.engines
import pyzor.engines.gdbm_
import pyzor.engines.common
class MockTimer():
def __init__(self, *args, **kwargs):
pass
def start(self):
pass
def setDaemon(self, daemon):
pass
class MockGdbm(dict):
"""Mock a gdbm database"""
def firstkey(self):
if not self.keys():
return None
self.key_index = 1
return self.keys()[0]
def nextkey(self, key):
if len(self.keys()) <= self.key_index:
return None
else:
self.key_index += 1
return self.keys()[self.key_index]
def sync(self):
pass
def reorganize(self):
pass
class GdbmTest(unittest.TestCase):
"""Test the GdbmDBHandle class"""
max_age = 60 * 60 * 24 * 30 * 4
r_count = 24
wl_count = 42
entered = datetime.now() - timedelta(days=10)
updated = datetime.now() - timedelta(days=2)
wl_entered = datetime.now() - timedelta(days=20)
wl_updated = datetime.now() - timedelta(days=3)
def setUp(self):
unittest.TestCase.setUp(self)
self.real_timer = threading.Timer
threading.Timer = MockTimer
self.db = MockGdbm()
def mock_open(fn, mode):
return self.db
self.real_open = gdbm.open
gdbm.open = mock_open
self.record = pyzor.engines.common.Record(self.r_count, self.wl_count,
self.entered, self.updated,
self.wl_entered, self.wl_updated)
def tearDown(self):
unittest.TestCase.tearDown(self)
threading.Timer = self.real_timer
gdbm.open = self.real_open
def record_as_str(self, record=None):
if not record:
record = self.record
return ("1,%s,%s,%s,%s,%s,%s" % (record.r_count, record.r_entered,
record.r_updated, record.wl_count,
record.wl_entered, record.wl_updated)).encode("utf8")
def test_set_item(self):
"""Test GdbmDBHandle.__setitem__"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=self.max_age)
handle[digest] = self.record
self.assertEqual(self.db[digest], self.record_as_str().decode("utf8"))
def test_get_item(self):
"""Test GdbmDBHandle.__getitem__"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=self.max_age)
self.db[digest] = self.record_as_str()
result = handle[digest]
self.assertEqual(self.record_as_str(result), self.record_as_str())
def test_del_item(self):
"""Test GdbmDBHandle.__delitem__"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=self.max_age)
self.db[digest] = self.record_as_str()
del handle[digest]
self.assertFalse(self.db.get(digest))
def test_reorganize_older(self):
"""Test GdbmDBHandle.start_reorganizing with older records"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.db[digest] = self.record_as_str()
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=3600 * 24)
self.assertFalse(self.db.get(digest))
def test_reorganize_older_no_max_age(self):
"""Test GdbmDBHandle.start_reorganizing with older records, but no
max_age set.
"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.db[digest] = self.record_as_str()
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=None)
self.assertEqual(self.db[digest], self.record_as_str())
def test_reorganize_fresh(self):
"""Test GdbmDBHandle.start_reorganizing with newer records"""
digest = "2aedaac999d71421c9ee49b9d81f627a7bc570aa"
self.db[digest] = self.record_as_str()
handle = pyzor.engines.gdbm_.GdbmDBHandle(None, None,
max_age=3600 * 24 * 3)
self.assertEqual(self.db[digest], self.record_as_str())
def suite():
"""Gather all the tests from this module in a test suite."""
test_suite = unittest.TestSuite()
test_suite.addTest(unittest.makeSuite(GdbmTest))
return test_suite
if __name__ == '__main__':
unittest.main()

Some files were not shown because too many files have changed in this diff Show More