Initial commit.

mail/spamassassin/pyzor-0.7.0/pyzor/.cvsignore

@@ -0,0 +1 @@
+*.pyc

mail/spamassassin/pyzor-0.7.0/pyzor/__init__.py

@@ -0,0 +1,60 @@
+"""Networked spam-signature detection."""
+
+__author__ = "Frank J. Tobin, ftobin@neverending.org"
+__credits__ = "Tony Meyer, Dreas von Donselaar, all the Pyzor contributors."
+__version__ = "0.7.0"
+
+import hashlib
+
+proto_name = 'pyzor'
+proto_version = 2.1
+anonymous_user = 'anonymous'
+
+# We would like to use sha512, but that would mean that all the digests
+# changed, so for now, we stick with sha1 (which is the same as the old
+# sha module).
+sha = hashlib.sha1
+
+# This is the maximum time between a client signing a Pyzor request and the
+# server checking the signature.
+MAX_TIMESTAMP_DIFFERENCE = 300  # seconds
+
+
+class CommError(Exception):
+    """Something in general went wrong with the transaction."""
+    pass
+
+
+class ProtocolError(CommError):
+    """Something is wrong with talking the protocol."""
+    pass
+
+
+class TimeoutError(CommError):
+    """The connection timed out."""
+    pass
+
+
+class IncompleteMessageError(ProtocolError):
+    """A complete requested was not received."""
+    pass
+
+
+class UnsupportedVersionError(ProtocolError):
+    """Client is using an unsupported protocol version."""
+    pass
+
+
+class SignatureError(CommError):
+    """Unknown user, signature on msg invalid, or not within allowed time
+    range."""
+    pass
+
+
+class AuthorizationError(CommError):
+    """The signature was valid, but the user is not permitted to do the
+    requested action."""
+    pass
+
+
+

mail/spamassassin/pyzor-0.7.0/pyzor/account.py

@@ -0,0 +1,75 @@
+"""A collection of utilities that facilitate working with Pyzor accounts.
+
+Note that accounts are not necessary (on the client or server), as an
+"anonymous" account always exists."""
+
+import time
+import hashlib
+
+import pyzor
+
+
+def sign_msg(hashed_key, timestamp, msg, hash_=hashlib.sha1):
+    """Converts the key, timestamp (epoch seconds), and msg into a digest.
+
+    lower(H(H(M) + ':' T + ':' + K))
+    M is message
+    T is integer epoch timestamp
+    K is hashed_key
+    H is the hash function (currently SHA1)
+    """
+    M = msg.as_string().strip().encode("utf8")
+    digest = hash_()
+    digest.update(hash_(M).digest())
+    digest.update((":%d:%s" % (timestamp, hashed_key)).encode("utf8"))
+    return digest.hexdigest().lower()
+
+def hash_key(key, user, hash_=hashlib.sha1):
+    """Returns the hash key for this username and password.
+
+    lower(H(U + ':' + lower(K)))
+    K is key (hex string)
+    U is username
+    H is the hash function (currently SHA1)
+    """
+    S = ("%s:%s" % (user, key.lower())).encode("utf8")
+    return hash_(S).hexdigest().lower()
+
+def verify_signature(msg, user_key):
+    """Verify that the provided message is correctly signed.
+
+    The message must have "User", "Time", and "Sig" headers.
+
+    If the signature is valid, then the function returns normally.
+    If the signature is not valid, then a pyzor.SignatureError() exception
+    is raised."""
+    timestamp = int(msg["Time"])
+    user = msg["User"]
+    provided_signature = msg["Sig"]
+    # Check that this signature is not too old.
+    if abs(time.time() - timestamp) > pyzor.MAX_TIMESTAMP_DIFFERENCE:
+        raise pyzor.SignatureError("Timestamp not within allowed range.")
+    # Calculate what the correct signature is.
+    hashed_user_key = hash_key(user_key, user)
+    # The signature is not part of the message that is signed.
+    del msg["Sig"]
+    correct_signature = sign_msg(hashed_user_key, timestamp, msg)
+    if correct_signature != provided_signature:
+        raise pyzor.SignatureError("Invalid signature.")
+
+class Account(object):
+    def __init__(self, username, salt, key):
+        self.username = username
+        self.salt = salt
+        self.key = key
+
+def key_from_hexstr(s):
+    try:
+        salt, key = s.split(",")
+    except ValueError:
+        raise ValueError("Invalid number of parts for key; perhaps you "
+                         "forgot the comma at the beginning for the "
+                         "salt divider?")
+    return salt, key
+
+AnonymousAccount = Account(pyzor.anonymous_user, None, "")

mail/spamassassin/pyzor-0.7.0/pyzor/client.py

@@ -0,0 +1,255 @@
+"""Networked spam-signature detection client.
+
+>>> import pyzor
+>>> import pyzor.client
+>>> import pyzor.digest
+>>> import pyzor.config
+
+To load the accounts file:
+
+>>> accounts = pyzor.config.load_accounts(filename)
+
+To create a client (to then issue commands):
+
+>>> client = pyzor.client.Client(accounts)
+
+To create a client, using the anonymous user:
+
+>>> client = pyzor.client.Client()
+
+To get a digest (of an email.message.Message object, or similar):
+
+>>> digest = pyzor.digest.get_digest(msg)
+
+To query a server (where address is a (host, port) pair):
+
+>>> client.ping(address)
+>>> client.info(digest, address)
+>>> client.report(digest, address)
+>>> client.whitelist(digest, address)
+>>> client.check(digest, address)
+
+To query the default server (public.pyzor.org):
+
+>>> client.ping()
+>>> client.info(digest)
+>>> client.report(digest)
+>>> client.whitelist(digest)
+>>> client.check(digest)
+
+Response will contain, depending on the type of request, some 
+of the following keys (e.g. client.ping()['Code']): 
+
+All responses will have:
+- 'Diag' 'OK' or error message
+- 'Code' '200' if OK
+- 'PV' Protocol Version
+- 'Thread'
+
+`info` and `check` responses will also contain:
+- '[WL-]Count' Whitelist/Blacklist count
+
+`info` responses will also have:
+- '[WL-]Entered' timestamp when message was first whitelisted/blacklisted
+- '[WL-]Updated' timestamp when message was last whitelisted/blacklisted
+"""
+
+import sys
+import time
+import email
+import socket
+import logging
+
+import pyzor.digest
+import pyzor.account
+import pyzor.message
+
+import pyzor.hacks.py26
+pyzor.hacks.py26.hack_email()
+
+class Client(object):
+    timeout = 5
+    max_packet_size = 8192
+
+    def __init__(self, accounts=None, timeout=None):
+        if accounts:
+            self.accounts = accounts
+        else:
+            self.accounts = {}
+        if timeout is not None:
+            self.timeout = timeout
+        self.log = logging.getLogger("pyzor")
+
+    def ping(self, address=("public.pyzor.org", 24441)):
+        msg = pyzor.message.PingRequest()
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def pong(self, digest, address=("public.pyzor.org", 24441)):
+        msg = pyzor.message.PongRequest(digest)
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def info(self, digest, address=("public.pyzor.org", 24441)):
+        msg = pyzor.message.InfoRequest(digest)
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def report(self, digest, address=("public.pyzor.org", 24441),
+               spec=pyzor.digest.digest_spec):
+        msg = pyzor.message.ReportRequest(digest, spec)
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def whitelist(self, digest, address=("public.pyzor.org", 24441),
+                  spec=pyzor.digest.digest_spec):
+        msg = pyzor.message.WhitelistRequest(digest, spec)
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def check(self, digest, address=("public.pyzor.org", 24441)):
+        msg = pyzor.message.CheckRequest(digest)
+        sock = self.send(msg, address)
+        return self.read_response(sock, msg.get_thread())
+
+    def send(self, msg, address=("public.pyzor.org", 24441)):
+        msg.init_for_sending()
+        try:
+            account = self.accounts[address]
+        except KeyError:
+            account = pyzor.account.AnonymousAccount
+        timestamp = int(time.time())
+        msg["User"] = account.username
+        msg["Time"] = str(timestamp)
+        msg["Sig"] = pyzor.account.sign_msg(pyzor.account.hash_key(
+            account.key, account.username), timestamp, msg)
+        self.log.debug("sending: %r", msg.as_string())
+        return self._send(msg, address)
+
+    def _send(self, msg, addr):
+        sock = None
+        for res in socket.getaddrinfo(addr[0], addr[1], 0, socket.SOCK_DGRAM,
+                                      socket.IPPROTO_UDP):
+            af, socktype, proto, _, sa = res
+            try:
+                sock = socket.socket(af, socktype, proto)
+            except socket.error:
+                sock = None
+                continue
+            try:
+                sock.sendto(msg.as_string().encode("utf8"), 0, sa)
+            except socket.timeout:
+                sock.close()
+                raise pyzor.TimeoutError("Sending to %s time-outed" % sa)
+            except socket.error:
+                sock.close()
+                sock = None
+                continue
+            break
+        if sock is None:
+            raise pyzor.CommError("Unable to send to %s" % addr)
+        return sock
+
+    def read_response(self, sock, expected_id):
+        sock.settimeout(self.timeout)
+        try:
+            packet, address = sock.recvfrom(self.max_packet_size)
+        except socket.timeout as e:
+            sock.close()
+            raise pyzor.TimeoutError("Reading response timed-out.")
+        except socket.error as e:
+            sock.close()
+            raise pyzor.CommError("Socket error while reading response: %s"
+                                  % e)
+
+        self.log.debug("received: %r/%r", packet, address)
+        msg = email.message_from_bytes(packet, _class=pyzor.message.Response)
+        msg.ensure_complete()
+        try:
+            thread_id = msg.get_thread()
+            if thread_id != expected_id:
+                if thread_id.in_ok_range():
+                    raise pyzor.ProtocolError(
+                        "received unexpected thread id %d (expected %d)" %
+                        (thread_id, expected_id))
+                self.log.warn("received error thread id %d (expected %d)",
+                              thread_id, expected_id)
+        except KeyError:
+            self.log.warn("no thread id received")
+        return msg
+
+
+class ClientRunner(object):
+    __slots__ = ['routine', 'all_ok', 'log']
+
+    def __init__(self, routine):
+        self.log = logging.getLogger("pyzor")
+        self.routine = routine
+        self.all_ok = True
+
+    def run(self, server, args, kwargs=None):
+        if kwargs is None:
+            kwargs = {}
+        message = "%s:%s\t" % server
+        response = None
+        try:
+            response = self.routine(*args, **kwargs)
+            self.handle_response(response, message)
+        except (pyzor.CommError, KeyError, ValueError), e:
+            self.log.error("%s\t%s: %s", server, e.__class__.__name__, e)
+            self.all_ok = False
+
+    def handle_response(self, response, message):
+        """mesaage is a string we've built up so far"""
+        if not response.is_ok():
+            self.all_ok = False
+        sys.stdout.write("%s%s\n" % (message, response.head_tuple()))
+
+
+class CheckClientRunner(ClientRunner):
+
+    def __init__(self, routine, r_count=0, wl_count=0):
+        ClientRunner.__init__(self, routine)
+        self.found_hit = False
+        self.whitelisted = False
+        self.hit_count = 0
+        self.whitelist_count = 0
+        self.r_count_found = r_count
+        self.wl_count_clears = wl_count
+
+    def handle_response(self, response, message):
+        message += "%s\t" % str(response.head_tuple())
+        if response.is_ok():
+            self.hit_count = int(response['Count'])
+            self.whitelist_count = int(response['WL-Count'])
+            if self.whitelist_count > self.wl_count_clears:
+                self.whitelisted = True
+            elif self.hit_count > self.r_count_found:
+                self.found_hit = True
+            message += "%d\t%d" % (self.hit_count, self.whitelist_count)
+            sys.stdout.write(message + '\n')
+        else:
+            self.all_ok = False
+            sys.stdout.write(message + '\n')
+
+class InfoClientRunner(ClientRunner):
+
+    def handle_response(self, response, message):
+        message += "%s\n" % str(response.head_tuple())
+
+        if response.is_ok():
+            for f in ('Count', 'Entered', 'Updated',
+                      'WL-Count', 'WL-Entered', 'WL-Updated'):
+                if response.has_key(f):
+                    val = int(response[f])
+                    if 'Count' in f:
+                        stringed = str(val)
+                    elif val == -1:
+                        stringed = 'Never'
+                    else:
+                        stringed = time.ctime(val)
+                    message += ("\t%s: %s\n" % (f, stringed))
+            sys.stdout.write(message + "\n")
+        else:
+            self.all_ok = False
+            sys.stdout.write(message + "\n")

mail/spamassassin/pyzor-0.7.0/pyzor/config.py

@@ -0,0 +1,220 @@
+"""Functions that handle parsing pyzor configuration files."""
+
+import os
+import re
+import logging
+import collections
+
+import pyzor.account
+
+# Configuration files for the Pyzor Server
+
+def load_access_file(access_fn, accounts):
+    """Load the ACL from the specified file, if it exists, and return an
+    ACL dictionary, where each key is a username and each value is a set
+    of allowed permissions (if the permission is not in the set, then it
+    is not allowed).
+
+    'accounts' is a dictionary of accounts that exist on the server - only
+    the keys are used, which must be the usernames (these are the users
+    that are granted permission when the 'all' keyword is used, as
+    described below).
+
+    Each line of the file should be in the following format:
+        operation : user : allow|deny
+    where 'operation' is a space-separated list of pyzor commands or the
+    keyword 'all' (meaning all commands), 'username' is a space-separated
+    list of usernames or the keyword 'all' (meaning all users) - the
+    anonymous user is called "anonymous", and "allow|deny" indicates whether
+    or not the specified user(s) may execute the specified operations.
+
+    The file is processed from top to bottom, with the final match for
+    user/operation being the value taken.  Every file has the following
+    implicit final rule:
+        all : all : deny
+
+    If the file does not exist, then the following default is used:
+        check report ping info : anonymous : allow
+    """
+    log = logging.getLogger("pyzord")
+    # A defaultdict is safe, because if we get a non-existant user, we get
+    # the empty set, which is the same as a deny, which is the final
+    # implicit rule.
+    acl = collections.defaultdict(set)
+    if not os.path.exists(access_fn):
+        log.info("Using default ACL: the anonymous user may use the check, "
+                 "report, ping and info commands.")
+        acl[pyzor.anonymous_user] = set(("check", "report", "ping", "pong",
+                                         "info"))
+        return acl
+    for line in open(access_fn):
+        if not line.strip() or line[0] == "#":
+            continue
+        try:
+            operations, users, allowed = [part.lower().strip()
+                                          for part in line.split(":")]
+        except ValueError:
+            log.warn("Invalid ACL line: %r", line)
+            continue
+        try:
+            allowed = {"allow": True, "deny" : False}[allowed]
+        except KeyError:
+            log.warn("Invalid ACL line: %r", line)
+            continue
+        if operations == "all":
+            operations = ("check", "report", "ping", "pong", "info",
+                          "whitelist")
+        else:
+            operations = [operation.strip()
+                          for operation in operations.split()]
+        if users == "all":
+            users = accounts
+        else:
+            users = [user.strip() for user in users.split()]
+        for user in users:
+            if allowed:
+                log.debug("Granting %s to %s.", ",".join(operations), user)
+                # If these operations are already allowed, this will have
+                # no effect.
+                acl[user].update(operations)
+            else:
+                log.debug("Revoking %s from %s.", ",".join(operations), user)
+                # If these operations are not allowed yet, this will have
+                # no effect.
+                acl[user].difference_update(operations)
+    log.info("ACL: %r", acl)
+    return acl
+
+def load_passwd_file(passwd_fn):
+    """Load the accounts from the specified file.
+
+    Each line of the file should be in the format:
+        username : key
+
+    If the file does not exist, then an empty dictionary is returned;
+    otherwise, a dictionary of (username, key) items is returned.
+    """
+    log = logging.getLogger("pyzord")
+    accounts = {}
+    if not os.path.exists(passwd_fn):
+        log.info("Accounts file does not exist - only the anonymous user "
+                 "will be available.")
+        return accounts
+    for line in open(passwd_fn):
+        if not line.strip() or line[0] == "#":
+            continue
+        try:
+            user, key = line.split(":")
+        except ValueError:
+            log.warn("Invalid accounts line: %r", line)
+            continue
+        user = user.strip()
+        key = key.strip()
+        log.debug("Creating an account for %s with key %s.", user, key)
+        accounts[user] = key
+    # Don't log the keys at 'info' level, just ther usernames.
+    log.info("Accounts: %s", ",".join(accounts))
+    return accounts
+
+# Configuration files for the Pyzor Client
+
+def load_accounts(filepath):
+    """Layout of file is: host : port : username : salt,key"""
+    accounts = {}
+    log = logging.getLogger("pyzor")
+    if os.path.exists(filepath):
+        for lineno, orig_line in enumerate(open(filepath)):
+            line = orig_line.strip()
+            if not line or line.startswith('#'):
+                continue
+            try:
+                host, port, username, key = [x.strip()
+                                             for x in line.split(":")]
+            except ValueError:
+                log.warn("account file: invalid line %d: wrong number of "
+                         "parts", lineno)
+                continue
+            try:
+                port = int(port)
+            except ValueError, e:
+                log.warn("account file: invalid line %d: %s", lineno, e)
+            address = (host, port)
+            salt, key = pyzor.account.key_from_hexstr(key)
+            if not salt and not key:
+                log.warn("account file: invalid line %d: keystuff can't be "
+                         "all None's", lineno)
+                continue
+            try:
+                accounts[address] = pyzor.account.Account(username, salt, key)
+            except ValueError, e:
+                log.warn("account file: invalid line %d: %s", lineno, e)
+    else:
+        log.warn("No accounts are setup.  All commands will be executed by "
+                 "the anonymous user.")
+    return accounts
+
+
+def load_servers(filepath):
+    """Load the servers file."""
+    logger = logging.getLogger("pyzor")
+    if not os.path.exists(filepath):
+        servers = []
+    else:
+        servers = []
+        with open(filepath) as f:
+            for line in f:
+                line = line.strip()
+                if re.match("[^#][a-zA-Z0-9.-]+:[0-9]+", line):
+                    address, port = line.rsplit(":", 1)
+                    servers.append((address, int(port)))
+
+    if not servers:
+        logger.info("No servers specified, defaulting to public.pyzor.org.")
+        servers = [("public.pyzor.org", 24441)]
+    return servers
+
+# Common configurations
+
+def setup_logging(log_name, filepath, debug):
+    """Setup logging according to the specified options. Return the Logger 
+    object.
+    """
+    fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+
+    stream_handler = logging.StreamHandler()
+    file_handler = None
+
+    if debug:
+        stream_log_level = logging.DEBUG
+        file_log_level = logging.DEBUG
+    else:
+        stream_log_level = logging.CRITICAL
+        file_log_level = logging.INFO
+
+    logger = logging.getLogger(log_name)
+    logger.setLevel(file_log_level)
+
+    stream_handler.setLevel(stream_log_level)
+    stream_handler.setFormatter(fmt)
+    logger.addHandler(stream_handler)
+
+    if filepath:
+        file_handler = logging.FileHandler(filepath)
+        file_handler.setLevel(file_log_level)
+        file_handler.setFormatter(fmt)
+        logger.addHandler(file_handler)
+
+    return logger
+
+def expand_homefiles(homefiles, category, homedir, config):
+    """Set the full file path for these configuration files."""
+    for filename in homefiles:
+        filepath = config.get(category, filename)
+        if not filepath:
+            continue
+        filepath = os.path.expanduser(filepath)
+        if not os.path.isabs(filepath):
+            filepath = os.path.join(homedir, filepath)
+        config.set(category, filename, filepath)
+
+

mail/spamassassin/pyzor-0.7.0/pyzor/digest.py

@@ -0,0 +1,149 @@
+import re
+import hashlib
+import HTMLParser
+
+# Hard-coded for the moment.
+digest_spec = ([(20, 3), (60, 3)])
+
+class HTMLStripper(HTMLParser.HTMLParser):
+    """Strip all tags from the HTML."""
+    def __init__(self, collector):
+        HTMLParser.HTMLParser.__init__(self)
+        self.reset()
+        self.collector = collector
+    def handle_data(self, data):
+        """Keep track of the data."""
+        data = data.strip()
+        if data:
+            self.collector.append(data)
+
+class DataDigester(object):
+    """The major workhouse class."""
+    __slots__ = ['value', 'digest']
+
+    # Minimum line length for it to be included as part of the digest.
+    min_line_length = 8
+
+    # If a message is this many lines or less, then we digest the whole
+    # message.
+    atomic_num_lines = 4
+
+    # We're not going to try to match email addresses as per the spec
+    # because it's too difficult.  Plus, regular expressions don't work well
+    # for them. (BNF is better at balanced parens and such).
+    email_ptrn = re.compile(r'\S+@\S+')
+
+    # Same goes for URLs.
+    url_ptrn = re.compile(r'[a-z]+:\S+', re.IGNORECASE)
+
+    # We also want to remove anything that is so long it looks like possibly
+    # a unique identifier.
+    longstr_ptrn = re.compile(r'\S{10,}')
+
+    ws_ptrn = re.compile(r'\s')
+
+    # String that the above patterns will be replaced with.
+    # Note that an empty string will always be used to remove whitespace.
+    unwanted_txt_repl = ''
+
+    def __init__(self, msg, spec=digest_spec):
+        self.value = None
+        self.digest = hashlib.sha1()
+
+        # Need to know the total number of lines in the content.
+        lines = []
+        for payload in self.digest_payloads(msg):
+            for line in payload.splitlines():
+                norm = self.normalize(line)
+                if self.should_handle_line(norm):
+                    lines.append(norm.encode("utf8"))
+
+        if len(lines) <= self.atomic_num_lines:
+            self.handle_atomic(lines)
+        else:
+            self.handle_pieced(lines, spec)
+
+        self.value = self.digest.hexdigest()
+
+        assert len(self.value) == len(hashlib.sha1(b"").hexdigest())
+        assert self.value is not None
+
+    def handle_atomic(self, lines):
+        """We digest everything."""
+        for line in lines:
+            self.handle_line(line)
+
+    def handle_pieced(self, lines, spec):
+        """Digest stuff according to the spec."""
+        for offset, length in spec:
+            for i in xrange(length):
+                try:
+                    line = lines[int(offset * len(lines) // 100) + i]
+                except IndexError:
+                    pass
+                else:
+                    self.handle_line(line)
+
+    def handle_line(self, line):
+        self.digest.update(line.rstrip())
+
+    @classmethod
+    def normalize(cls, s):
+        repl = cls.unwanted_txt_repl
+        s = cls.longstr_ptrn.sub(repl, s)
+        s = cls.email_ptrn.sub(repl, s)
+        s = cls.url_ptrn.sub(repl, s)
+        # Make sure we do the whitespace last because some of the previous
+        # patterns rely on whitespace.
+        return cls.ws_ptrn.sub('', s).strip()
+
+    @staticmethod
+    def normalize_html_part(s):
+        data = []
+        stripper = HTMLStripper(data)
+        try:
+            stripper.feed(s)
+        except (UnicodeDecodeError, HTMLParser.HTMLParseError):
+            # We can't parse the HTML, so just strip it.  This is still
+            # better than including generic HTML/CSS text.
+            pass
+        return " ".join(data)
+
+    @classmethod
+    def should_handle_line(cls, s):
+        return len(s) and cls.min_line_length <= len(s)
+
+    @classmethod
+    def digest_payloads(cls, msg):
+        for part in msg.walk():
+            if part.get_content_maintype() == "text":
+                payload = part.get_payload(decode=True)
+                charset = part.get_content_charset()
+                if not charset:
+                    charset = "ascii"
+                try:
+                    payload = payload.decode(charset, "ignore")
+                except LookupError:
+                    payload = payload.decode("ascii", "ignore")
+                if part.get_content_subtype() == "html":
+                    yield cls.normalize_html_part(payload)
+                else:
+                    yield payload
+            elif part.is_multipart():
+                # Skip, because walk() will give us the payload next.
+                pass
+            else:
+                # Non-text parts are passed through as-is.
+                yield part.get_payload()
+
+
+class PrintingDataDigester(DataDigester):
+    """Extends DataDigester: prints out what we're digesting."""
+    def handle_line(self, line):
+        print line.decode("utf8")
+        super(PrintingDataDigester, self).handle_line(line)
+
+
+# Convenience function.
+def get_digest(msg):
+    return DataDigester(msg).value

mail/spamassassin/pyzor-0.7.0/pyzor/engines/__init__.py

@@ -0,0 +1,27 @@
+"""Database backends for pyzord.
+
+The database class must expose a dictionary-like interface, allowing access
+via __getitem__, __setitem__, and __delitem__.  The key will be a forty
+character string, and the value should be an instance of the Record class.
+
+If the database backend cannot store the Record objects natively, then it
+must transparently take care of translating to/from Record objects in
+__setitem__ and __getitem__.
+
+The database class should take care of expiring old values at the
+appropriate interval.
+"""
+
+from pyzor.engines import gdbm_
+from pyzor.engines import mysql
+from pyzor.engines import redis_
+
+
+__all__ = ["database_classes"]
+
+database_classes = {"gdbm": gdbm_.handle,
+                    "mysql": mysql.handle,
+                    "redis": redis_.handle,
+                    }
+
+

mail/spamassassin/pyzor-0.7.0/pyzor/engines/common.py

@@ -0,0 +1,51 @@
+"""Common library shared by different engines."""
+
+import sys
+import datetime
+
+from collections import namedtuple
+
+__all__ = ["DBHandle", "DatabaseError", "Record"]
+
+DBHandle = namedtuple("DBHandle", ["single_threaded", "multi_threaded",
+                                   "multi_processing"])
+
+class DatabaseError(Exception):
+    pass
+
+class Record(object):
+    """Prefix conventions used in this class:
+    r = report (spam)
+    wl = whitelist
+    """
+    def __init__(self, r_count=0, wl_count=0, r_entered=None,
+                 r_updated=None, wl_entered=None, wl_updated=None):
+        self.r_count = r_count
+        self.wl_count = wl_count
+        self.r_entered = r_entered
+        self.r_updated = r_updated
+        self.wl_entered = wl_entered
+        self.wl_updated = wl_updated
+
+    def wl_increment(self):
+        # overflow prevention
+        if self.wl_count < sys.maxint:
+            self.wl_count += 1
+        if self.wl_entered is None:
+            self.wl_entered = datetime.datetime.now()
+        self.wl_update()
+
+    def r_increment(self):
+        # overflow prevention
+        if self.r_count < sys.maxint:
+            self.r_count += 1
+        if self.r_entered is None:
+            self.r_entered = datetime.datetime.now()
+        self.r_update()
+
+    def r_update(self):
+        self.r_updated = datetime.datetime.now()
+
+    def wl_update(self):
+        self.wl_updated = datetime.datetime.now()
+

mail/spamassassin/pyzor-0.7.0/pyzor/engines/gdbm_.py

@@ -0,0 +1,174 @@
+"""Gdbm database engine."""
+
+try:
+    import gdbm
+except ImportError:
+    gdbm = None
+
+import sys
+import time
+import logging
+import datetime
+import threading
+
+from pyzor.engines.common import *
+
+class GdbmDBHandle(object):
+    absolute_source = True
+    sync_period = 60
+    reorganize_period = 3600 * 24  # 1 day
+    _dt_decode = lambda x: None if x == 'None' else datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f")
+    fields = (
+        'r_count', 'r_entered', 'r_updated',
+        'wl_count', 'wl_entered', 'wl_updated',
+        )
+    _fields = [('r_count', int),
+                ('r_entered', _dt_decode),
+                ('r_updated', _dt_decode),
+                ('wl_count', int),
+                ('wl_entered', _dt_decode),
+                ('wl_updated', _dt_decode)]
+    this_version = '1'
+    log = logging.getLogger("pyzord")
+
+    def __init__(self, fn, mode, max_age=None):
+        self.max_age = max_age
+        self.db = gdbm.open(fn, mode)
+        self.start_reorganizing()
+        self.start_syncing()
+
+    def apply_method(self, method, varargs=(), kwargs=None):
+        if kwargs is None:
+            kwargs = {}
+        return apply(method, varargs, kwargs)
+
+    def __getitem__(self, key):
+        return self.apply_method(self._really_getitem, (key,))
+
+    def _really_getitem(self, key):
+        return GdbmDBHandle.decode_record(self.db[key])
+
+    def __setitem__(self, key, value):
+        self.apply_method(self._really_setitem, (key, value))
+
+    def _really_setitem(self, key, value):
+        self.db[key] = GdbmDBHandle.encode_record(value)
+
+    def __delitem__(self, key):
+        self.apply_method(self._really_delitem, (key,))
+
+    def _really_delitem(self, key):
+        del self.db[key]
+
+    def start_syncing(self):
+        if self.db:
+            self.apply_method(self._really_sync)
+        self.sync_timer = threading.Timer(self.sync_period,
+                                          self.start_syncing)
+        self.sync_timer.setDaemon(True)
+        self.sync_timer.start()
+
+    def _really_sync(self):
+        self.db.sync()
+
+    def start_reorganizing(self):
+        if not self.max_age:
+            return
+        if self.db:
+            self.apply_method(self._really_reorganize)
+        self.reorganize_timer = threading.Timer(self.reorganize_period,
+                                                self.start_reorganizing)
+        self.reorganize_timer.setDaemon(True)
+        self.reorganize_timer.start()
+
+    def _really_reorganize(self):
+        self.log.debug("reorganizing the database")
+        key = self.db.firstkey()
+        breakpoint = time.time() - self.max_age
+        while key is not None:
+            rec = self._really_getitem(key)
+            delkey = None
+            if int(time.mktime(rec.r_updated.timetuple())) < breakpoint:
+                self.log.debug("deleting key %s", key)
+                delkey = key
+            key = self.db.nextkey(key)
+            if delkey:
+                self._really_delitem(delkey)
+        self.db.reorganize()
+
+    @classmethod
+    def encode_record(cls, value):
+        values = [cls.this_version]
+        values.extend(["%s" % getattr(value, x) for x in cls.fields])
+        return ",".join(values)
+
+    @classmethod
+    def decode_record(cls, s):
+        try:
+            s = s.decode("utf8")
+        except UnicodeError:
+            raise StandardError("don't know how to handle db value %s" %
+                                repr(s))
+        parts = s.split(',')
+        dispatch = None
+        version = parts[0]
+        if len(parts) == 3:
+            dispatch = cls.decode_record_0
+        elif version == '1':
+            dispatch = cls.decode_record_1
+        else:
+            raise StandardError("don't know how to handle db value %s" %
+                                repr(s))
+        return dispatch(s)
+
+    @staticmethod
+    def decode_record_0(s):
+        r = Record()
+        parts = s.split(',')
+        fields = ('r_count', 'r_entered', 'r_updated')
+        assert len(parts) == len(fields)
+        for i in range(len(parts)):
+            setattr(r, fields[i], int(parts[i]))
+        return r
+
+    @classmethod
+    def decode_record_1(cls, s):
+        r = Record()
+        parts = s.split(',')[1:]
+        assert len(parts) == len(cls.fields)
+        for part, field in zip(parts, cls._fields):
+            f, decode = field
+            setattr(r, f, decode(part))
+        return r
+
+class ThreadedGdbmDBHandle(GdbmDBHandle):
+    """Like GdbmDBHandle, but handles multi-threaded access."""
+
+    def __init__(self, fn, mode, max_age=None, bound=None):
+        self.db_lock = threading.Lock()
+        GdbmDBHandle.__init__(self, fn, mode, max_age=max_age)
+
+    def apply_method(self, method, varargs=(), kwargs=None):
+        if kwargs is None:
+            kwargs = {}
+        with self.db_lock:
+            return GdbmDBHandle.apply_method(self, method, varargs=varargs,
+                                             kwargs=kwargs)
+# This won't work because the gdbm object needs to be in shared memory of the
+# spawned processes.
+# class ProcessGdbmDBHandle(ThreadedGdbmDBHandle):
+#     def __init__(self, fn, mode, max_age=None, bound=None):
+#         ThreadedGdbmDBHandle.__init__(self, fn, mode, max_age=max_age,
+#                                       bound=bound)
+#         self.db_lock = multiprocessing.Lock()
+
+if sys.version_info[0] != 3 and gdbm is None:
+    handle = DBHandle(single_threaded=None,
+                      multi_threaded=None,
+                      multi_processing=None)
+else:
+    handle = DBHandle(single_threaded=GdbmDBHandle,
+                      multi_threaded=ThreadedGdbmDBHandle,
+                      multi_processing=None)
+
+

mail/spamassassin/pyzor-0.7.0/pyzor/engines/mysql.py

@@ -0,0 +1,268 @@
+"""MySQLdb database engine."""
+
+import time
+import Queue
+import logging
+import datetime
+import threading
+
+try:
+    import MySQLdb
+except ImportError:
+    # The SQL database backend will not work.
+    MySQLdb = None
+
+from pyzor.engines.common import *
+
+
+class MySQLDBHandle(object):
+    absolute_source = False
+    # The table must already exist, and have this schema:
+    #   CREATE TABLE `public` (
+    #   `digest` char(40) default NULL,
+    #   `r_count` int(11) default NULL,
+    #   `wl_count` int(11) default NULL,
+    #   `r_entered` datetime default NULL,
+    #   `wl_entered` datetime default NULL,
+    #   `r_updated` datetime default NULL,
+    #   `wl_updated` datetime default NULL,
+    #   PRIMARY KEY  (`digest`)
+    #   )
+    # XXX Re-organising might be faster with a r_updated index.  However,
+    # XXX the re-organisation time isn't that important, and that would
+    # XXX (slightly) slow down all inserts, so we leave it for now.
+    reorganize_period = 3600 * 24  # 1 day
+    reconnect_period = 60  # seconds
+    log = logging.getLogger("pyzord")
+
+    def __init__(self, fn, mode, max_age=None):
+        self.max_age = max_age
+        self.db = None
+        # The 'fn' is host,user,password,db,table.  We ignore mode.
+        # We store the authentication details so that we can reconnect if
+        # necessary.
+        self.host, self.user, self.passwd, self.db_name, \
+            self.table_name = fn.split(",")
+        self.last_connect_attempt = 0  # We have never connected.
+        self.reconnect()
+        self.start_reorganizing()
+
+    def _get_new_connection(self):
+        """Returns a new db connection."""
+        db = MySQLdb.connect(host=self.host, user=self.user,
+                               db=self.db_name, passwd=self.passwd)
+        db.autocommit(True)
+        return db
+
+    def _check_reconnect_time(self):
+        if time.time() - self.last_connect_attempt < self.reconnect_period:
+            # Too soon to reconnect.
+            self.log.debug("Can't reconnect until %s",
+                           (time.ctime(self.last_connect_attempt +
+                                       self.reconnect_period)))
+            return False
+        return True
+
+    def reconnect(self):
+        if not self._check_reconnect_time():
+            return
+        if self.db:
+            try:
+                self.db.close()
+            except MySQLdb.Error:
+                pass
+        try:
+            self.db = self._get_new_connection()
+        except MySQLdb.Error, e:
+            self.log.error("Unable to connect to database: %s", e)
+            self.db = None
+        # Keep track of when we connected, so that we don't retry too often.
+        self.last_connect_attempt = time.time()
+
+    def __del__(self):
+        """Close the database when the object is no longer needed."""
+        try:
+            if self.db:
+                self.db.close()
+        except MySQLdb.Error:
+            pass
+
+    def _safe_call(self, name, method, args):
+        try:
+            return method(*args, db=self.db)
+        except (MySQLdb.Error, AttributeError), e:
+            self.log.error("%s failed: %s", name, e)
+            self.reconnect()
+            # Retrying just complicates the logic - we don't really care if
+            # a single query fails (and it's possible that it would fail)
+            # on the second attempt anyway.  Any exceptions are caught by
+            # the server, and a 'nice' message provided to the caller.
+            raise DatabaseError("Database temporarily unavailable.")
+
+    def __getitem__(self, key):
+        return self._safe_call("getitem", self._really__getitem__, (key,))
+
+    def __setitem__(self, key, value):
+        return self._safe_call("setitem", self._really__setitem__,
+                               (key, value))
+
+    def __delitem__(self, key):
+        return self._safe_call("delitem", self._really__delitem__, (key,))
+
+    def _really__getitem__(self, key, db=None):
+        """__getitem__ without the exception handling."""
+        c = db.cursor()
+        # The order here must match the order of the arguments to the
+        # Record constructor.
+        c.execute("SELECT r_count, wl_count, r_entered, r_updated, "
+                  "wl_entered, wl_updated FROM %s WHERE digest=%%s" %
+                  self.table_name, (key,))
+        try:
+            try:
+                return Record(*c.fetchone())
+            except TypeError:
+                # fetchone() returned None, i.e. there is no such record
+                raise KeyError()
+        finally:
+            c.close()
+
+    def _really__setitem__(self, key, value, db=None):
+        """__setitem__ without the exception handling."""
+        c = db.cursor()
+        try:
+            c.execute("INSERT INTO %s (digest, r_count, wl_count, "
+                      "r_entered, r_updated, wl_entered, wl_updated) "
+                      "VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s) ON "
+                      "DUPLICATE KEY UPDATE r_count=%%s, wl_count=%%s, "
+                      "r_entered=%%s, r_updated=%%s, wl_entered=%%s, "
+                      "wl_updated=%%s" % self.table_name,
+                      (key, value.r_count, value.wl_count, value.r_entered,
+                       value.r_updated, value.wl_entered, value.wl_updated,
+                       value.r_count, value.wl_count, value.r_entered,
+                       value.r_updated, value.wl_entered, value.wl_updated))
+        finally:
+            c.close()
+
+    def _really__delitem__(self, key, db=None):
+        """__delitem__ without the exception handling."""
+        c = db.cursor()
+        try:
+            c.execute("DELETE FROM %s WHERE digest=%%s" % self.table_name,
+                      (key,))
+        finally:
+            c.close()
+
+    def start_reorganizing(self):
+        if not self.max_age:
+            return
+        self.log.debug("reorganizing the database")
+        breakpoint = (datetime.datetime.now() -
+                      datetime.timedelta(seconds=self.max_age))
+        db = self._get_new_connection()
+        c = db.cursor()
+        try:
+            c.execute("DELETE FROM %s WHERE r_updated<%%s" %
+                      self.table_name, (breakpoint,))
+        except (MySQLdb.Error, AttributeError), e:
+            self.log.warn("Unable to reorganise: %s", e)
+        finally:
+            c.close()
+            db.close()
+        self.reorganize_timer = threading.Timer(self.reorganize_period,
+                                                self.start_reorganizing)
+        self.reorganize_timer.setDaemon(True)
+        self.reorganize_timer.start()
+
+class ThreadedMySQLDBHandle(MySQLDBHandle):
+
+    def __init__(self, fn, mode, max_age=None, bound=None):
+        self.bound = bound
+        if self.bound:
+            self.db_queue = Queue.Queue()
+        MySQLDBHandle.__init__(self, fn, mode, max_age=max_age)
+
+    def _get_connection(self):
+        if self.bound:
+            return self.db_queue.get()
+        else:
+            return self._get_new_connection()
+
+    def _release_connection(self, db):
+        if self.bound:
+            self.db_queue.put(db)
+        else:
+            db.close()
+
+    def _safe_call(self, name, method, args):
+        db = self._get_connection()
+        try:
+            return method(*args, db=db)
+        except (MySQLdb.Error, AttributeError) as e:
+            self.log.error("%s failed: %s", name, e)
+            if not self.bound:
+                raise DatabaseError("Database temporarily unavailable.")
+            try:
+                # Connection might be timeout, ping and retry
+                db.ping(True)
+                return method(*args, db=db)
+            except (MySQLdb.Error, AttributeError) as e:
+                # attempt a new connection, if we can retry
+                db = self._reconnect(db)
+                raise DatabaseError("Database temporarily unavailable.")
+        finally:
+            self._release_connection(db)
+
+    def reconnect(self):
+        if not self.bound:
+            return
+        for _ in xrange(self.bound):
+            self.db_queue.put(self._get_new_connection())
+
+    def _reconnect(self, db):
+        if not self._check_reconnect_time():
+            return db
+        else:
+            self.last_connect_attempt = time.time()
+            return self._get_new_connection()
+
+    def __del__(self):
+        if not self.bound:
+            return
+        for db in iter(self.db_queue.get_nowait):
+            try:
+                db.close()
+            except MySQLdb.Error:
+                continue
+            except Queue.Empty:
+                break
+
+class ProcessMySQLDBHandle(MySQLDBHandle):
+    def __init__(self, fn, mode, max_age=None):
+        MySQLDBHandle.__init__(self, fn, mode, max_age=max_age)
+
+    def reconnect(self):
+        pass
+
+    def __del__(self):
+        pass
+
+    def _safe_call(self, name, method, args):
+        db = None
+        try:
+            db = self._get_new_connection()
+            return method(*args, db=db)
+        except (MySQLdb.Error, AttributeError) as e:
+            self.log.error("%s failed: %s", name, e)
+            raise DatabaseError("Database temporarily unavailable.")
+        finally:
+            if db is not None:
+                db.close()
+
+if MySQLdb is None:
+    handle = DBHandle(single_threaded=None,
+                      multi_threaded=None,
+                      multi_processing=None)
+else:
+    handle = DBHandle(single_threaded=MySQLDBHandle,
+                      multi_threaded=ThreadedMySQLDBHandle,
+                      multi_processing=ProcessMySQLDBHandle)

mail/spamassassin/pyzor-0.7.0/pyzor/engines/redis_.py

@@ -0,0 +1,110 @@
+"""Redis database engine."""
+
+import logging
+import datetime
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+from pyzor.engines.common import *
+
+NAMESPACE = "pyzord.digest"
+
+encode_date = lambda d: "" if d is None else d.strftime("%Y-%m-%d %H:%M:%S")
+decode_date = lambda x: None if x == "" else datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+
+
+def safe_call(f):
+    """Decorator that wraps a method for handling database operations."""
+    def wrapped_f(self, *args, **kwargs):
+        # This only logs the error and raise the usual Error for consistency,
+        # the redis library takes care of reconnecting and everything else.
+        try:
+            return f(self, *args, **kwargs)
+        except redis.exceptions.RedisError as e:
+            self.log.error("Redis error while calling %s: %s",
+                           f.__name__, e)
+            raise DatabaseError("Database temporarily unavailable.")
+    return wrapped_f
+
+
+class RedisDBHandle(object):
+    absolute_source = False
+
+    log = logging.getLogger("pyzord")
+
+    def __init__(self, fn, mode, max_age=None):
+        self.max_age = max_age
+        # The 'fn' is host,port,password,db.  We ignore mode.
+        # We store the authentication details so that we can reconnect if
+        # necessary.
+        fn = fn.split(",")
+        self.host = fn[0] or "localhost"
+        self.port = fn[1] or "6379"
+        self.passwd = fn[2] or None
+        self.db_name = fn[3] or "0"
+        self.db = self._get_new_connection()
+
+    @staticmethod
+    def _encode_record(r):
+        return ("%s,%s,%s,%s,%s,%s" %
+                (r.r_count,
+                 encode_date(r.r_entered),
+                 encode_date(r.r_updated),
+                 r.wl_count,
+                 encode_date(r.wl_entered),
+                 encode_date(r.wl_updated))).encode()
+
+    @staticmethod
+    def _decode_record(r):
+        if r is None:
+            return Record()
+        fields = r.decode().split(",")
+        return Record(r_count=int(fields[0]),
+                      r_entered=decode_date(fields[1]),
+                      r_updated=decode_date(fields[2]),
+                      wl_count=int(fields[3]),
+                      wl_entered=decode_date(fields[4]),
+                      wl_updated=decode_date(fields[5]))
+
+    @staticmethod
+    def _real_key(key):
+        return "%s.%s" % (NAMESPACE, key)
+
+    @safe_call
+    def _get_new_connection(self):
+        return redis.StrictRedis(host=self.host, port=int(self.port),
+                                 db=int(self.db_name), password=self.passwd)
+
+    @safe_call
+    def __getitem__(self, key):
+        return self._decode_record(self.db.get(self._real_key(key)))
+
+    @safe_call
+    def __setitem__(self, key, value):
+        if self.max_age is None:
+            self.db.set(self._real_key(key), self._encode_record(value))
+        else:
+            self.db.setex(self._real_key(key), self.max_age,
+                          self._encode_record(value))
+
+    @safe_call
+    def __delitem__(self, key):
+        self.db.delete(self._real_key(key))
+
+class ThreadedRedisDBHandle(RedisDBHandle):
+
+    def __init__(self, fn, mode, max_age=None, bound=None):
+        RedisDBHandle.__init__(self, fn, mode, max_age=max_age)
+
+
+if redis is None:
+    handle = DBHandle(single_threaded=None,
+                      multi_threaded=None,
+                      multi_processing=None)
+else:
+    handle = DBHandle(single_threaded=RedisDBHandle,
+                      multi_threaded=ThreadedRedisDBHandle,
+                      multi_processing=None)

mail/spamassassin/pyzor-0.7.0/pyzor/hacks/__init__.py

@@ -0,0 +1 @@
+"""Various hack to make pyzor compatible with different Python versions."""

mail/spamassassin/pyzor-0.7.0/pyzor/hacks/py26.py

@@ -0,0 +1,42 @@
+"""Hacks for Python 2.6"""
+
+__all__ = ["hack_all", "hack_email", "hack_select"]
+
+def hack_all(email=True, select=True): 
+    if email:
+        hack_email()
+    if select:
+        hack_select()
+
+def hack_email():
+    """The python2.6 version of email.message_from_string, doesn't work with 
+    unicode strings. And in python3 it will only work with a decoded. 
+    
+    So switch to using only message_from_bytes.
+    """
+    import email
+    if not hasattr(email, "message_from_bytes"):
+        email.message_from_bytes = email.message_from_string
+
+    
+def hack_select():
+    """The python2.6 version of SocketServer does not handle interrupt calls 
+    from signals. Patch the select call if necessary. 
+    """
+    import sys
+    if sys.version_info[0] == 2 and sys.version_info[1] == 6:
+        import select
+        import errno
+
+        real_select = select.select
+        def _eintr_retry(*args):
+            """restart a system call interrupted by EINTR"""
+            while True:
+                try:
+                    return real_select(*args)
+                except (OSError, select.error) as e:
+                    if e.args[0] != errno.EINTR:
+                        raise
+        select.select = _eintr_retry
+
+

mail/spamassassin/pyzor-0.7.0/pyzor/message.py

@@ -0,0 +1,152 @@
+"""This modules contains the various messages used in the pyzor client server 
+communication.
+"""
+
+import random
+import email.message
+
+import pyzor
+
+class Message(email.message.Message):
+    def __init__(self):
+        email.message.Message.__init__(self)
+        self.setup()
+
+    def setup(self):
+        pass
+
+    def init_for_sending(self):
+        self.ensure_complete()
+
+    def __str__(self):
+        # The parent class adds the unix From header.
+        return self.as_string()
+
+    def ensure_complete(self):
+        pass
+
+
+class ThreadedMessage(Message):
+    def init_for_sending(self):
+        if not self.has_key('Thread'):
+            self.set_thread(ThreadId.generate())
+        assert self.has_key('Thread')
+        self["PV"] = str(pyzor.proto_version)
+        Message.init_for_sending(self)
+
+    def ensure_complete(self):
+        if not (self.has_key('PV') and self.has_key('Thread')):
+            raise pyzor.IncompleteMessageError("Doesn't have fields for a "
+                                               "ThreadedMessage.")
+        Message.ensure_complete(self)
+
+    def get_protocol_version(self):
+        return float(self['PV'])
+
+    def get_thread(self):
+        return ThreadId(self['Thread'])
+
+    def set_thread(self, i):
+        self['Thread'] = str(i)
+
+
+class Response(ThreadedMessage):
+    ok_code = 200
+
+    def ensure_complete(self):
+        if not (self.has_key('Code') and self.has_key('Diag')):
+            raise pyzor.IncompleteMessageError("doesn't have fields for a "
+                                               "Response")
+        ThreadedMessage.ensure_complete(self)
+
+    def is_ok(self):
+        return self.get_code() == self.ok_code
+
+    def get_code(self):
+        return int(self['Code'])
+
+    def get_diag(self):
+        return self['Diag']
+
+    def head_tuple(self):
+        return self.get_code(), self.get_diag()
+
+
+class Request(ThreadedMessage):
+    """This is the class that should be used to read in Requests of any type.
+    Subclasses are responsible for setting 'Op' if they are generating a
+    message,"""
+
+    def get_op(self):
+        return self['Op']
+
+    def ensure_complete(self):
+        if not self.has_key('Op'):
+            raise pyzor.IncompleteMessageError("doesn't have fields for a "
+                                               "Request")
+        ThreadedMessage.ensure_complete(self)
+
+
+class ClientSideRequest(Request):
+    op = None
+    def setup(self):
+        Request.setup(self)
+        self["Op"] = self.op
+
+
+class SimpleDigestBasedRequest(ClientSideRequest):
+    def __init__(self, digest):
+        ClientSideRequest.__init__(self)
+        self["Op-Digest"] = digest
+
+
+class SimpleDigestSpecBasedRequest(SimpleDigestBasedRequest):
+    def __init__(self, digest, spec):
+        SimpleDigestBasedRequest.__init__(self, digest)
+        flat_spec = [item for sublist in spec for item in sublist]
+        self["Op-Spec"] = ",".join(str(part) for part in flat_spec)
+
+
+class PingRequest(ClientSideRequest):
+    op = "ping"
+
+
+class PongRequest(SimpleDigestBasedRequest):
+    op = "pong"
+
+
+class CheckRequest(SimpleDigestBasedRequest):
+    op = "check"
+
+
+class InfoRequest(SimpleDigestBasedRequest):
+    op = "info"
+
+
+class ReportRequest(SimpleDigestSpecBasedRequest):
+    op = "report"
+
+
+class WhitelistRequest(SimpleDigestSpecBasedRequest):
+    op = "whitelist"
+
+
+class ThreadId(int):
+    # (0, 1024) is reserved
+    full_range = (0, 2 ** 16)
+    ok_range = (1024, full_range[1])
+    error_value = 0
+
+    def __new__(cls, i):
+        self = int.__new__(cls, i)
+        if not (cls.full_range[0] <= self < cls.full_range[1]):
+            raise ValueError("value outside of range")
+        return self
+
+    @classmethod
+    def generate(cls):
+        return cls(random.randrange(*cls.ok_range))
+
+    def in_ok_range(self):
+        return self.ok_range[0] <= self < self.ok_range[1]
+

mail/spamassassin/pyzor-0.7.0/pyzor/server.py

@@ -0,0 +1,299 @@
+"""Networked spam-signature detection server.
+
+The server receives the request in the form of a RFC5321 message, and
+responds with another RFC5321 message.  Neither of these messages has a
+body - all of the data is encapsulated in the headers.
+
+The response headers will always include a "Code" header, which is a
+HTTP-style response code, and a "Diag" header, which is a human-readable
+message explaining the response code (typically this will be "OK").
+
+Both the request and response headers always include a "PV" header, which
+indicates the protocol version that is being used (in a major.minor format).
+Both the requestion and response headers also always include a "Thread",
+which uniquely identifies the request (this is a requirement of using UDP).
+Responses to requests may arrive in any order, but the "Thread" header of
+a response will always match the "Thread" header of the appropriate request.
+
+Authenticated requests must also have "User", "Time" (timestamp), and "Sig"
+(signature) headers.
+"""
+
+import sys
+import time
+import socket
+import signal
+import logging
+import StringIO
+import threading
+import traceback
+import SocketServer
+import email.message
+
+import pyzor.config
+import pyzor.account
+import pyzor.engines.common
+
+import pyzor.hacks.py26
+pyzor.hacks.py26.hack_all()
+
+class Server(SocketServer.UDPServer):
+    """The pyzord server.  Handles incoming UDP connections in a single
+    thread and single process."""
+    max_packet_size = 8192
+    time_diff_allowance = 180
+
+    def __init__(self, address, database, passwd_fn, access_fn):
+        if ":" in address[0]:
+            Server.address_family = socket.AF_INET6
+        else:
+            Server.address_family = socket.AF_INET
+        self.log = logging.getLogger("pyzord")
+        self.usage_log = logging.getLogger("pyzord-usage")
+        self.database = database
+
+        # Handle configuration files
+        self.passwd_fn = passwd_fn
+        self.access_fn = access_fn
+        self.load_config()
+
+        self.log.debug("Listening on %s", address)
+        SocketServer.UDPServer.__init__(self, address, RequestHandler,
+                                        bind_and_activate=False)
+        try:
+            self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+        except (AttributeError, socket.error) as e:
+            self.log.debug("Unable to set IPV6_V6ONLY to false %s", e)
+        self.server_bind()
+        self.server_activate()
+
+        # Finally, set signals
+        signal.signal(signal.SIGUSR1, self.reload_handler)
+        signal.signal(signal.SIGTERM, self.shutdown_handler)
+
+    def load_config(self):
+        """Reads the configuration files and loads the accounts and ACLs."""
+        self.accounts = pyzor.config.load_passwd_file(self.passwd_fn)
+        self.acl = pyzor.config.load_access_file(self.access_fn, self.accounts)
+
+    def shutdown_handler(self, *args, **kwargs):
+        """Handler for the SIGTERM signal. This should be used to kill the 
+        daemon and ensure proper clean-up. 
+        """
+        self.log.info("SIGTERM received. Shutting down.")
+        t = threading.Thread(target=self.shutdown)
+        t.start()
+
+    def reload_handler(self, *args, **kwargs):
+        """Handler for the SIGUSR1 signal. This should be used to reload
+        the configuration files.
+        """
+        self.log.info("SIGUSR1 received. Reloading configuration.")
+        t = threading.Thread(target=self.load_config)
+        t.start()
+
+
+class ThreadingServer(SocketServer.ThreadingMixIn, Server):
+    """A threaded version of the pyzord server.  Each connection is served
+    in a new thread.  This may not be suitable for all database types."""
+    pass
+
+
+class BoundedThreadingServer(ThreadingServer):
+    """Same as ThreadingServer but this also accepts a limited number of 
+    concurrent threads.
+    """
+    def __init__(self, address, database, passwd_fn, access_fn, max_threads):
+        ThreadingServer.__init__(self, address, database, passwd_fn, access_fn)
+        self.semaphore = threading.Semaphore(max_threads)
+
+    def process_request(self, request, client_address):
+        self.semaphore.acquire()
+        ThreadingServer.process_request(self, request, client_address)
+
+    def process_request_thread(self, request, client_address):
+        ThreadingServer.process_request_thread(self, request, client_address)
+        self.semaphore.release()
+
+
+class ProcessServer(SocketServer.ForkingMixIn, Server):
+    """A multi-processing version of the pyzord server.  Each connection is 
+    served in a new process. This may not be suitable for all database types.
+    """
+    def __init__(self, address, database, passwd_fn, access_fn,
+                 max_children=40):
+        ProcessServer.max_children = max_children
+        Server.__init__(self, address, database, passwd_fn, access_fn)
+
+
+class RequestHandler(SocketServer.DatagramRequestHandler):
+    """Handle a single pyzord request."""
+
+    def __init__(self, *args, **kwargs):
+        self.response = email.message.Message()
+        SocketServer.DatagramRequestHandler.__init__(self, *args, **kwargs)
+
+    def handle(self):
+        """Handle a pyzord operation, cleanly handling any errors."""
+        self.response["Code"] = "200"
+        self.response["Diag"] = "OK"
+        self.response["PV"] = "%s" % pyzor.proto_version
+        try:
+            self._really_handle()
+        except NotImplementedError, e:
+            self.handle_error(501, "Not implemented: %s" % e)
+        except pyzor.UnsupportedVersionError, e:
+            self.handle_error(505, "Version Not Supported: %s" % e)
+        except pyzor.ProtocolError, e:
+            self.handle_error(400, "Bad request: %s" % e)
+        except pyzor.SignatureError, e:
+            self.handle_error(401, "Unauthorized: Signature Error: %s" % e)
+        except pyzor.AuthorizationError, e:
+            self.handle_error(403, "Forbidden: %s" % e)
+        except Exception, e:
+            self.handle_error(500, "Internal Server Error: %s" % e)
+            trace = StringIO.StringIO()
+            traceback.print_exc(file=trace)
+            trace.seek(0)
+            self.server.log.error(trace.read())
+        self.server.log.debug("Sending: %r", self.response.as_string())
+        self.wfile.write(self.response.as_string().encode("utf8"))
+
+    def _really_handle(self):
+        """handle() without the exception handling."""
+        self.server.log.debug("Received: %r", self.packet)
+
+        # Read the request.
+        # Old versions of the client sent a double \n after the signature,
+        # which screws up the RFC5321 format.  Specifically handle that
+        # here - this could be removed in time.
+        request = email.message_from_bytes(
+            self.rfile.read().replace(b"\n\n", b"\n") + b"\n")
+
+        # Ensure that the response can be paired with the request.
+        self.response["Thread"] = request["Thread"]
+
+        # If this is an authenticated request, then check the authentication
+        # details.
+        user = request["User"] or pyzor.anonymous_user
+        if user != pyzor.anonymous_user:
+            try:
+                pyzor.account.verify_signature(request,
+                                               self.server.accounts[user])
+            except KeyError:
+                raise pyzor.SignatureError("Unknown user.")
+
+        if "PV" not in request:
+            raise pyzor.ProtocolError("Protocol Version not specified in request")
+
+        # The protocol version is compatible if the major number is
+        # identical (changes in the minor number are unimportant).
+        if int(float(request["PV"])) != int(pyzor.proto_version):
+            raise pyzor.UnsupportedVersionError()
+
+        # Check that the user has permission to execute the requested
+        # operation.
+        opcode = request["Op"]
+        if opcode not in self.server.acl[user]:
+            raise pyzor.AuthorizationError(
+                "User is not authorized to request the operation.")
+        self.server.log.debug("Got a %s command from %s", opcode,
+                              self.client_address[0])
+
+        # Get a handle to the appropriate method to execute this operation.
+        try:
+            dispatch = self.dispatches[opcode]
+        except KeyError:
+            raise NotImplementedError("Requested operation is not "
+                                      "implemented.")
+        # Get the existing record from the database (or a blank one if
+        # there is no matching record).
+        digest = request["Op-Digest"]
+        # Do the requested operation, log what we have done, and return.
+        if dispatch:
+            try:
+                record = self.server.database[digest]
+            except KeyError:
+                record = pyzor.engines.common.Record()
+            dispatch(self, digest, record)
+        self.server.usage_log.info("%s,%s,%s,%r,%s", user,
+                                   self.client_address[0], opcode, digest,
+                                   self.response["Code"])
+
+    def handle_error(self, code, message):
+        """Create an appropriate response for an error."""
+        self.server.usage_log.error("%s: %s", code, message)
+        self.response.replace_header("Code", "%d" % code)
+        self.response.replace_header("Diag", message)
+
+    def handle_pong(self, digest, _):
+        """Handle the 'pong' command.
+
+        This command returns maxint for report counts and 0 whitelist.
+        """
+        self.server.log.debug("Request pong for %s", digest)
+        self.response["Count"] = "%d" % sys.maxint
+        self.response["WL-Count"] = "%d" % 0
+
+    def handle_check(self, digest, record):
+        """Handle the 'check' command.
+
+        This command returns the spam/ham counts for the specified digest.
+        """
+        self.server.log.debug("Request to check digest %s", digest)
+        self.response["Count"] = "%d" % record.r_count
+        self.response["WL-Count"] = "%d" % record.wl_count
+
+    def handle_report(self, digest, record):
+        """Handle the 'report' command.
+
+        This command increases the spam count for the specified digest."""
+        self.server.log.debug("Request to report digest %s", digest)
+        # Increase the count, and store the altered record back in the
+        # database.
+        record.r_increment()
+        self.server.database[digest] = record
+
+    def handle_whitelist(self, digest, record):
+        """Handle the 'whitelist' command.
+
+        This command increases the ham count for the specified digest."""
+        self.server.log.debug("Request to whitelist digest %s", digest)
+        # Increase the count, and store the altered record back in the
+        # database.
+        record.wl_increment()
+        self.server.database[digest] = record
+
+    def handle_info(self, digest, record):
+        """Handle the 'info' command.
+
+        This command returns diagnostic data about a digest (timestamps for
+        when the digest was first/last seen as spam/ham, and spam/ham
+        counts).
+        """
+        self.server.log.debug("Request for information about digest %s",
+                              digest)
+        def time_output(time_obj):
+            """Convert a datetime object to a POSIX timestamp.
+
+            If the object is None, then return 0.
+            """
+            if not time_obj:
+                return 0
+            return time.mktime(time_obj.timetuple())
+        self.response["Entered"] = "%d" % time_output(record.r_entered)
+        self.response["Updated"] = "%d" % time_output(record.r_updated)
+        self.response["WL-Entered"] = "%d" % time_output(record.wl_entered)
+        self.response["WL-Updated"] = "%d" % time_output(record.wl_updated)
+        self.response["Count"] = "%d" % record.r_count
+        self.response["WL-Count"] = "%d" % record.wl_count
+
+    dispatches = {
+        'ping' : None,
+        'pong' : handle_pong,
+        'info' : handle_info,
+        'check' : handle_check,
+        'report' : handle_report,
+        'whitelist' : handle_whitelist,
+        }
+