areguera / rpms / mailman

Forked from rpms/mailman 4 years ago
Clone

Blame SOURCES/mailman-2.1.12-dmarc.patch

49da8b
diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py
49da8b
index a6a952a..0901b50 100644
49da8b
--- a/Mailman/Bouncers/SimpleMatch.py
49da8b
+++ b/Mailman/Bouncers/SimpleMatch.py
49da8b
@@ -42,7 +42,7 @@ PATTERNS = [
49da8b
     # sz-sb.de, corridor.com, nfg.nl
49da8b
     (_c('the following addresses had'),
49da8b
      _c('transcript of session follows'),
49da8b
-     _c(r'<(?P<fulladdr>[^>]*)>|\(expanded from: [^>)]*)>?\)')),
49da8b
+     _c(r'^ *(\(expanded from: )?[^\s@]+@[^\s@>]+?)>?\)?\s*$')),
49da8b
     # robanal.demon.co.uk
49da8b
     (_c('this message was created automatically by mail delivery software'),
49da8b
      _c('original message follows'),
49da8b
diff --git a/Mailman/Bouncers/Yahoo.py b/Mailman/Bouncers/Yahoo.py
49da8b
index b3edf4f..08ede54 100644
49da8b
--- a/Mailman/Bouncers/Yahoo.py
49da8b
+++ b/Mailman/Bouncers/Yahoo.py
49da8b
@@ -20,9 +20,15 @@ import re
49da8b
 import email
49da8b
 from email.Utils import parseaddr
49da8b
 
49da8b
-tcre = re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE)
49da8b
+tcre = (re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE),
49da8b
+        re.compile(r'Sorry, we were unable to deliver your message to '
49da8b
+                   r'the following address(\(es\))?\.',
49da8b
+                   re.IGNORECASE),
49da8b
+        )
49da8b
 acre = re.compile(r'<(?P<addr>[^>]*)>:')
49da8b
-ecre = re.compile(r'--- Original message follows')
49da8b
+ecre = (re.compile(r'--- Original message follows'),
49da8b
+        re.compile(r'--- Below this line is a copy of the message'),
49da8b
+        )
49da8b
 
49da8b
 
49da8b
 
49da8b
@@ -36,18 +42,26 @@ def process(msg):
49da8b
     # simple state machine
49da8b
     #     0 == nothing seen
49da8b
     #     1 == tag line seen
49da8b
+    #     2 == end line seen
49da8b
     state = 0
49da8b
     for line in email.Iterators.body_line_iterator(msg):
49da8b
         line = line.strip()
49da8b
-        if state == 0 and tcre.match(line):
49da8b
-            state = 1
49da8b
+        if state == 0:
49da8b
+            for cre in tcre:
49da8b
+                if cre.match(line):
49da8b
+                    state = 1
49da8b
+                    break
49da8b
         elif state == 1:
49da8b
             mo = acre.match(line)
49da8b
             if mo:
49da8b
                 addrs.append(mo.group('addr'))
49da8b
                 continue
49da8b
-            mo = ecre.match(line)
49da8b
-            if mo:
49da8b
-                # we're at the end of the error response
49da8b
-                break
49da8b
+            for cre in ecre:
49da8b
+                mo = cre.match(line)
49da8b
+                if mo:
49da8b
+                    # we're at the end of the error response
49da8b
+                    state = 2
49da8b
+                    break
49da8b
+        elif state == 2:
49da8b
+            break
49da8b
     return addrs
49da8b
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
49da8b
old mode 100644
49da8b
new mode 100755
49da8b
index 4fe63db..8e42f54
49da8b
--- a/Mailman/Defaults.py.in
49da8b
+++ b/Mailman/Defaults.py.in
49da8b
@@ -505,6 +505,7 @@ GLOBAL_PIPELINE = [
49da8b
     # (outgoing) path, finally leaving the message in the outgoing queue.
49da8b
     'AfterDelivery',
49da8b
     'Acknowledge',
49da8b
+    'WrapMessage',
49da8b
     'ToOutgoing',
49da8b
     ]
49da8b
 
49da8b
@@ -914,6 +915,29 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No
49da8b
 # moderators?
49da8b
 DEFAULT_FORWARD_AUTO_DISCARDS = Yes
49da8b
 
49da8b
+# Shall dmarc_moderation_action be applied to messages From: domains with
49da8b
+# a DMARC policy of quarantine as well as reject?  This sets the default for
49da8b
+# the list setting that controls it.
49da8b
+DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION = Yes
49da8b
+
49da8b
+# Default action for posts whose From: address domain has a DMARC policy of
49da8b
+# reject or quarantine.  See DEFAULT_FROM_IS_LIST below.  Whatever is set as
49da8b
+# the default here precludes the list owner from setting a lower value.
49da8b
+# 0 = Accept
49da8b
+# 1 = Munge From
49da8b
+# 2 = Wrap Message
49da8b
+# 3 = Reject
49da8b
+# 4 = Discard
49da8b
+DEFAULT_DMARC_MODERATION_ACTION = 0
49da8b
+
49da8b
+# Parameters for DMARC DNS lookups. If you are seeing 'DNSException:
49da8b
+# Unable to query DMARC policy ...' entries in your error log, you may need
49da8b
+# to adjust these.
49da8b
+# The time to wait for a response from a name server before timeout.
49da8b
+DMARC_RESOLVER_TIMEOUT = seconds(3)
49da8b
+# The total time to spend trying to get an answer to the question.
49da8b
+DMARC_RESOLVER_LIFETIME = seconds(5)
49da8b
+
49da8b
 # What shold happen to non-member posts which are do not match explicit
49da8b
 # non-member actions?
49da8b
 # 0 = Accept
49da8b
@@ -950,6 +974,25 @@ DEFAULT_SEND_WELCOME_MSG = Yes
49da8b
 # Send goodbye messages to unsubscribed members?
49da8b
 DEFAULT_SEND_GOODBYE_MSG = Yes
49da8b
 
49da8b
+# Some list posts and mail to the -owner address may contain DomainKey or
49da8b
+# DomainKeys Identified Mail (DKIM) signature headers <http://www.dkim.org/>.
49da8b
+# Various list transformations to the message such as adding a list header or
49da8b
+# footer or scrubbing attachments or even reply-to munging can break these
49da8b
+# signatures.  It is generally felt that these signatures have value, even if
49da8b
+# broken and even if the outgoing message is resigned.  However, some sites
49da8b
+# may wish to remove these headers by setting this to Yes.
49da8b
+REMOVE_DKIM_HEADERS = No
49da8b
+
49da8b
+# The following is a three way setting.  It sets the default for the list's
49da8b
+# from_is_list policy which is applied to all posts except those for which a
49da8b
+# dmarc_moderation_action other than accept applies.
49da8b
+# 0 -> Do not rewrite the From: or wrap the message.
49da8b
+# 1 -> Rewrite the From: header of posts replacing the posters address with
49da8b
+#      that of the list.  Also see REMOVE_DKIM_HEADERS above.
49da8b
+# 2 -> Do not modify the From: of the message, but wrap the message in an outer
49da8b
+#      message From the list address.
49da8b
+DEFAULT_FROM_IS_LIST = 0
49da8b
+
49da8b
 # Wipe sender information, and make it look like the list-admin
49da8b
 # address sends all messages
49da8b
 DEFAULT_ANONYMOUS_LIST = No
49da8b
diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py
49da8b
index 8271a30..05dc9ba 100644
49da8b
--- a/Mailman/Gui/General.py
49da8b
+++ b/Mailman/Gui/General.py
49da8b
@@ -153,6 +153,72 @@ class General(GUIBase):
49da8b
                             (listname %%05d) -> (listname 00123)
49da8b
              """)),
49da8b
 
49da8b
+            ('from_is_list', mm_cfg.Radio,
49da8b
+             (_('No'), _('Munge From'), _('Wrap Message')), 0,
49da8b
+             _("""Replace the From: header address with the list's posting
49da8b
+             address to mitigate issues stemming from the original From:
49da8b
+             domain's DMARC or similar policies."""),
49da8b
+             _("""Several protocols now in wide use attempt to ensure that use
49da8b
+             of the domain in the author's address (ie, in the From: header
49da8b
+             field) is authorized by that domain.  These protocols may be
49da8b
+             incompatible with common list features such as footers, causing
49da8b
+             participating email services to bounce list traffic merely
49da8b
+             because of the address in the From: field.  This has resulted
49da8b
+             in members being unsubscribed despite being perfectly able to
49da8b
+             receive mail.
49da8b
+             

49da8b
+             The following actions are applied to all list messages when
49da8b
+             selected here.  To apply these actions only to messages where the
49da8b
+             domain in the From: header is determined to use such a protocol,
49da8b
+             see the 
49da8b
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
49da8b
+             dmarc_moderation_action settings under Privacy options...
49da8b
+             -> Sender filters.
49da8b
+             

Settings:

49da8b
+             
49da8b
+             
No
49da8b
+             
Do nothing special. This is appropriate for anonymous lists.
49da8b
+             It is appropriate for dedicated announcement lists, unless the
49da8b
+             From: address of authorized posters might be in a domain with a
49da8b
+             DMARC or similar policy. It is also appropriate if you choose to
49da8b
+             use dmarc_moderation_action other than Accept for this list.
49da8b
+             
Munge From
49da8b
+             
This action replaces the poster's address in the From: header
49da8b
+             with the list's posting address and adds the poster's address to
49da8b
+             the addresses in the original Reply-To: header.
49da8b
+             
Wrap Message
49da8b
+             
Just wrap the message in an outer message with the From:
49da8b
+             header containing the list's posting address and with the original
49da8b
+             From: address added to the addresses in the original Reply-To:
49da8b
+             header and with Content-Type: message/rfc822.  This is effectively
49da8b
+             a one message MIME format digest.
49da8b
+             
49da8b
+             

The transformations for anonymous_list are applied before

49da8b
+             any of these actions. It is not useful to apply actions other
49da8b
+             than No to an anonymous list, and if you do so, the result may
49da8b
+             be surprising.
49da8b
+             

The Reply-To: header munging actions below interact with these

49da8b
+             actions as follows:
49da8b
+             

first_strip_reply_to = Yes will remove all the incoming

49da8b
+             Reply-To: addresses but will still add the poster's address to
49da8b
+             Reply-To: for all three settings of reply_goes_to_list which
49da8b
+             respectively will result in just the poster's address, the
49da8b
+             poster's address and the list posting address or the poster's
49da8b
+             address and the explicit reply_to_address in the outgoing
49da8b
+             Reply-To: header. If first_strip_reply_to = No the poster's
49da8b
+             address in the original From: header, if not already included in
49da8b
+             the Reply-To:, will be added to any existing Reply-To:
49da8b
+             address(es).
49da8b
+             

These actions, whether selected here or via

49da8b
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
49da8b
+             dmarc_moderation_action, do not apply to messages in digests
49da8b
+             or archives or sent to usenet via the Mail<->News gateways.
49da8b
+             

If

49da8b
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
49da8b
+             dmarc_moderation_action applies to this message with an
49da8b
+             action other than Accept, that action rather than this is
49da8b
+             applied""")),
49da8b
+
49da8b
             ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0,
49da8b
              _("""Hide the sender of a message, replacing it with the list
49da8b
              address (Removes From, Sender and Reply-To fields)""")),
49da8b
diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py
49da8b
old mode 100644
49da8b
new mode 100755
49da8b
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
49da8b
index 75eff2b..5d717bb 100644
49da8b
--- a/Mailman/Gui/Privacy.py
49da8b
+++ b/Mailman/Gui/Privacy.py
49da8b
@@ -158,6 +158,11 @@ class Privacy(GUIBase):
49da8b
             ]
49da8b
 
49da8b
         adminurl = mlist.GetScriptURL('admin', absolute=1)
49da8b
+    
49da8b
+        if mlist.dmarc_quarantine_moderation_action:
49da8b
+            quarantine = _('/Quarantine')
49da8b
+        else:
49da8b
+            quarantine = ''
49da8b
         sender_rtn = [
49da8b
             _("""When a message is posted to the list, a series of
49da8b
             moderation steps are taken to decide whether a moderator must
49da8b
@@ -235,6 +240,59 @@ class Privacy(GUIBase):
49da8b
              >rejection notice to
49da8b
              be sent to moderated members who post to this list.""")),
49da8b
 
49da8b
+            ('dmarc_moderation_action', mm_cfg.Radio,
49da8b
+             (_('Accept'), _('Munge From'), _('Wrap Message'), _('Reject'),
49da8b
+                 _('Discard')), 0,
49da8b
+             _("""Action to take when anyone posts to the
49da8b
+             list from a domain with a DMARC Reject%(quarantine)s Policy."""),
49da8b
+
49da8b
+             _("""
  • Munge From -- applies the
49da8b
+             href="?VARHELP=general/from_is_list">from_is_list Munge From
49da8b
+             transformation to these messages.
49da8b
+
49da8b
+             

  • Wrap Message -- applies the
  • 49da8b
    +             href="?VARHELP=general/from_is_list">from_is_list Wrap
    49da8b
    +             Message transformation to these messages.
    49da8b
    +
    49da8b
    +             

  • Reject -- this automatically rejects the message by
  • 49da8b
    +             sending a bounce notice to the post's author.  The text of the
    49da8b
    +             bounce notice can be 
    49da8b
    +             href="?VARHELP=privacy/sender/dmarc_moderation_notice"
    49da8b
    +             >configured by you.
    49da8b
    +
    49da8b
    +             

  • Discard -- this simply discards the message, with
  • 49da8b
    +             no notice sent to the post's author.
    49da8b
    +             
    49da8b
    +
    49da8b
    +             

    This setting takes precedence over the

    49da8b
    +             href="?VARHELP=general/from_is_list"> from_is_list setting
    49da8b
    +             if the message is From: an affected domain and the setting is
    49da8b
    +             other than Accept.""")),
    49da8b
    +
    49da8b
    +            ('dmarc_quarantine_moderation_action', mm_cfg.Radio,
    49da8b
    +             (_('No'), _('Yes')), 0,
    49da8b
    +             _("""Shall the above dmarc_moderation_action apply to messages
    49da8b
    +               From: domains with DMARC p=quarantine as well as p=reject"""),
    49da8b
    +
    49da8b
    +             _("""
    • No -- this applies dmarc_moderation_action to
    49da8b
    +               only those posts From: a domain with DMARC p=reject.  This is
    49da8b
    +               appropriate if you are concerned about bounced messages, but
    49da8b
    +               want to apply dmarc_moderation_action to as few messages as
    49da8b
    +               possible.
    49da8b
    +               

  • Yes -- this applies dmarc_moderation_action to
  • 49da8b
    +               posts From: a domain with DMARC p=reject or p=quarantine.
    49da8b
    +               

    If a message is From: a domain with DMARC p=quarantine

    49da8b
    +               and dmarc_moderation_action is not applied (this set to No)
    49da8b
    +               the message will likely not bounce, but will be delivered to
    49da8b
    +               recipients' spam folders or other hard to find places.""")),
    49da8b
    +
    49da8b
    +            ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1,
    49da8b
    +             _("""Text to include in any
    49da8b
    +             
    49da8b
    +             >rejection notice to
    49da8b
    +             be sent to anyone who posts to this list from a domain
    49da8b
    +             with a DMARC Reject%(quarantine)s Policy.""")),
    49da8b
    +
    49da8b
                 _('Non-member filters'),
    49da8b
     
    49da8b
                 ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1,
    49da8b
    @@ -399,7 +457,7 @@ class Privacy(GUIBase):
    49da8b
                  case, each rule is matched in turn, with processing stopped after
    49da8b
                  the first match.
    49da8b
     
    49da8b
    -             Note that headers are collected from all the attachments 
    49da8b
    +             Note that headers are collected from all the attachments
    49da8b
                  (except for the mailman administrivia message) and
    49da8b
                  matched against the regular expressions. With this feature,
    49da8b
                  you can effectively sort out messages with dangerous file
    49da8b
    @@ -442,6 +500,11 @@ class Privacy(GUIBase):
    49da8b
             # an option.
    49da8b
             if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE:
    49da8b
                 val += 1
    49da8b
    +        if (property == 'dmarc_moderation_action' and
    49da8b
    +                val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION):
    49da8b
    +            doc.addError(_("""dmarc_moderation_action must be >= the configured
    49da8b
    +                           default value."""))
    49da8b
    +            val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    49da8b
             setattr(mlist, property, val)
    49da8b
     
    49da8b
         # We need to handle the header_filter_rules widgets specially, but
    49da8b
    diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py
    49da8b
    index 038034c..549d8e7 100644
    49da8b
    --- a/Mailman/Handlers/AvoidDuplicates.py
    49da8b
    +++ b/Mailman/Handlers/AvoidDuplicates.py
    49da8b
    @@ -24,6 +24,7 @@ warning header, or pass it through, depending on the user's preferences.
    49da8b
     
    49da8b
     from email.Utils import getaddresses, formataddr
    49da8b
     from Mailman import mm_cfg
    49da8b
    +from Mailman.Handlers.CookHeaders import change_header
    49da8b
     
    49da8b
     COMMASPACE = ', '
    49da8b
     
    49da8b
    @@ -95,6 +96,10 @@ def process(mlist, msg, msgdata):
    49da8b
         # Set the new list of recipients
    49da8b
         msgdata['recips'] = newrecips
    49da8b
         # RFC 2822 specifies zero or one CC header
    49da8b
    -    del msg['cc']
    49da8b
         if ccaddrs:
    49da8b
    -        msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()])
    49da8b
    +        change_header('Cc',
    49da8b
    +        COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]),
    49da8b
    +        mlist, msg, msgdata)
    49da8b
    +    else:
    49da8b
    +        del msg['cc']
    49da8b
    +
    49da8b
    diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
    49da8b
    old mode 100644
    49da8b
    new mode 100755
    49da8b
    index 8e7e668..c556967
    49da8b
    --- a/Mailman/Handlers/CookHeaders.py
    49da8b
    +++ b/Mailman/Handlers/CookHeaders.py
    49da8b
    @@ -64,13 +64,23 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None):
    49da8b
             charset = 'us-ascii'
    49da8b
         return Header(s, charset, maxlinelen, header_name, continuation_ws)
    49da8b
     
    49da8b
    +def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True):
    49da8b
    +    if ((msgdata.get('from_is_list') == 2 or
    49da8b
    +        (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and 
    49da8b
    +        not msgdata.get('_fasttrack')
    49da8b
    +       ) or name.lower() in ('from', 'reply-to'):
    49da8b
    +        msgdata.setdefault('add_header', {})[name] = value
    49da8b
    +    elif repl or not msg.has_key(name):
    49da8b
    +        if delete:
    49da8b
    +            del msg[name]
    49da8b
    +        msg[name] = value
    49da8b
    +
    49da8b
     
    49da8b
     
    49da8b
     def process(mlist, msg, msgdata):
    49da8b
         # Set the "X-Ack: no" header if noack flag is set.
    49da8b
         if msgdata.get('noack'):
    49da8b
    -        del msg['x-ack']
    49da8b
    -        msg['X-Ack'] = 'no'
    49da8b
    +        change_header('X-Ack', 'no', mlist, msg, msgdata)
    49da8b
         # Because we're going to modify various important headers in the email
    49da8b
         # message, we want to save some of the information in the msgdata
    49da8b
         # dictionary for later.  Specifically, the sender header will get waxed,
    49da8b
    @@ -87,7 +97,8 @@ def process(mlist, msg, msgdata):
    49da8b
                 pass
    49da8b
         # Mark message so we know we've been here, but leave any existing
    49da8b
         # X-BeenThere's intact.
    49da8b
    -    msg['X-BeenThere'] = mlist.GetListEmail()
    49da8b
    +    change_header('X-BeenThere', mlist.GetListEmail(),
    49da8b
    +                  mlist, msg, msgdata, delete=False)
    49da8b
         # Add Precedence: and other useful headers.  None of these are standard
    49da8b
         # and finding information on some of them are fairly difficult.  Some are
    49da8b
         # just common practice, and we'll add more here as they become necessary.
    49da8b
    @@ -101,12 +112,31 @@ def process(mlist, msg, msgdata):
    49da8b
         # known exploits in a particular version of Mailman and we know a site is
    49da8b
         # using such an old version, they may be vulnerable.  It's too easy to
    49da8b
         # edit the code to add a configuration variable to handle this.
    49da8b
    -    if not msg.has_key('x-mailman-version'):
    49da8b
    -        msg['X-Mailman-Version'] = mm_cfg.VERSION
    49da8b
    +    change_header('X-Mailman-Version', mm_cfg.VERSION,
    49da8b
    +                  mlist, msg, msgdata, repl=False)
    49da8b
         # We set "Precedence: list" because this is the recommendation from the
    49da8b
         # sendmail docs, the most authoritative source of this header's semantics.
    49da8b
    -    if not msg.has_key('precedence'):
    49da8b
    -        msg['Precedence'] = 'list'
    49da8b
    +    change_header('Precedence', 'list',
    49da8b
    +                  mlist, msg, msgdata, repl=False)
    49da8b
    +    # Do we change the from so the list takes ownership of the email
    49da8b
    +    if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack:
    49da8b
    +        realname, email = parseaddr(msg['from'])
    49da8b
    +        if not realname:
    49da8b
    +            if mlist.isMember(email):
    49da8b
    +                realname = mlist.getMemberName(email) or email
    49da8b
    +            else:
    49da8b
    +                realname = email
    49da8b
    +        # Remove domain from realname if it looks like an email address
    49da8b
    +        realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
    49da8b
    +        # Remember the original From: here for adding to Reply-To: below.
    49da8b
    +        o_from = parseaddr(msg['from'])
    49da8b
    +        change_header('From',
    49da8b
    +                      formataddr(('%s via %s' % (realname, mlist.real_name),
    49da8b
    +                                 mlist.GetListEmail())),
    49da8b
    +                      mlist, msg, msgdata)
    49da8b
    +    else:
    49da8b
    +        # Use this as a flag
    49da8b
    +        o_from = None
    49da8b
         # Reply-To: munging.  Do not do this if the message is "fast tracked",
    49da8b
         # meaning it is internally crafted and delivered to a specific user.  BAW:
    49da8b
         # Yuck, I really hate this feature but I've caved under the sheer pressure
    49da8b
    @@ -136,18 +166,23 @@ def process(mlist, msg, msgdata):
    49da8b
                 orig = msg.get_all('reply-to', [])
    49da8b
                 for pair in getaddresses(orig):
    49da8b
                     add(pair)
    49da8b
    +        # We also need to put the old From: in Reply-To: in all cases.
    49da8b
    +        if o_from:
    49da8b
    +            add(o_from)
    49da8b
             # Set Reply-To: header to point back to this list.  Add this last
    49da8b
             # because some folks think that some MUAs make it easier to delete
    49da8b
             # addresses from the right than from the left.
    49da8b
             if mlist.reply_goes_to_list == 1:
    49da8b
                 i18ndesc = uheader(mlist, mlist.description, 'Reply-To')
    49da8b
                 add((str(i18ndesc), mlist.GetListEmail()))
    49da8b
    -        del msg['reply-to']
    49da8b
             # Don't put Reply-To: back if there's nothing to add!
    49da8b
             if new:
    49da8b
                 # Preserve order
    49da8b
    -            msg['Reply-To'] = COMMASPACE.join(
    49da8b
    -                [formataddr(pair) for pair in new])
    49da8b
    +            change_header('Reply-To', 
    49da8b
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    49da8b
    +                          mlist, msg, msgdata)
    49da8b
    +        else:
    49da8b
    +            del msg['reply-to']
    49da8b
             # The To field normally contains the list posting address.  However
    49da8b
             # when messages are fully personalized, that header will get
    49da8b
             # overwritten with the address of the recipient.  We need to get the
    49da8b
    @@ -158,18 +193,31 @@ def process(mlist, msg, msgdata):
    49da8b
             # above code?
    49da8b
             # Also skip Cc if this is an anonymous list as list posting address
    49da8b
             # is already in From and Reply-To in this case.
    49da8b
    -        if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \
    49da8b
    -           and not mlist.anonymous_list:
    49da8b
    +        # We do add the Cc in cases where From: header munging is being done
    49da8b
    +        # because even though the list address is in From:, the Reply-To:
    49da8b
    +        # poster will override it. Brain dead MUAs may then address the list
    49da8b
    +        # twice on a 'reply all', but reasonable MUAs should do the right
    49da8b
    +        # thing.
    49da8b
    +        if (mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 and
    49da8b
    +            not mlist.anonymous_list):
    49da8b
                 # Watch out for existing Cc headers, merge, and remove dups.  Note
    49da8b
                 # that RFC 2822 says only zero or one Cc header is allowed.
    49da8b
                 new = []
    49da8b
                 d = {}
    49da8b
    -            for pair in getaddresses(msg.get_all('cc', [])):
    49da8b
    -                add(pair)
    49da8b
    +            # AvoidDuplicates may have set a new Cc: in msgdata.add_header,
    49da8b
    +            # so check that.
    49da8b
    +            if (msgdata.has_key('add_header') and
    49da8b
    +                    msgdata['add_header'].has_key('Cc')):
    49da8b
    +                for pair in getaddresses([msgdata['add_header']['Cc']]):
    49da8b
    +                    add(pair)
    49da8b
    +            else:
    49da8b
    +                for pair in getaddresses(msg.get_all('cc', [])):
    49da8b
    +                    add(pair)
    49da8b
                 i18ndesc = uheader(mlist, mlist.description, 'Cc')
    49da8b
                 add((str(i18ndesc), mlist.GetListEmail()))
    49da8b
    -            del msg['Cc']
    49da8b
    -            msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new])
    49da8b
    +            change_header('Cc',
    49da8b
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    49da8b
    +                          mlist, msg, msgdata)
    49da8b
         # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only
    49da8b
         # if the message is being crafted for a specific list (e.g. not for the
    49da8b
         # password reminders).
    49da8b
    @@ -191,8 +239,7 @@ def process(mlist, msg, msgdata):
    49da8b
             # without desc we need to ensure the MUST brackets
    49da8b
             listid_h = '<%s>' % listid
    49da8b
         # We always add a List-ID: header.
    49da8b
    -    del msg['list-id']
    49da8b
    -    msg['List-Id'] = listid_h
    49da8b
    +    change_header('List-Id', listid_h, mlist, msg, msgdata)
    49da8b
         # For internally crafted messages, we also add a (nonstandard),
    49da8b
         # "X-List-Administrivia: yes" header.  For all others (i.e. those coming
    49da8b
         # from list posts), we add a bunch of other RFC 2369 headers.
    49da8b
    @@ -219,13 +266,12 @@ def process(mlist, msg, msgdata):
    49da8b
         # First we delete any pre-existing headers because the RFC permits only
    49da8b
         # one copy of each, and we want to be sure it's ours.
    49da8b
         for h, v in headers.items():
    49da8b
    -        del msg[h]
    49da8b
             # Wrap these lines if they are too long.  78 character width probably
    49da8b
             # shouldn't be hardcoded, but is at least text-MUA friendly.  The
    49da8b
             # adding of 2 is for the colon-space separator.
    49da8b
             if len(h) + 2 + len(v) > 78:
    49da8b
                 v = CONTINUATION.join(v.split(', '))
    49da8b
    -        msg[h] = v
    49da8b
    +        change_header(h, v, mlist, msg, msgdata)
    49da8b
     
    49da8b
     
    49da8b
     
    49da8b
    @@ -302,8 +348,7 @@ def prefix_subject(mlist, msg, msgdata):
    49da8b
                         h = u' '.join([prefix, subject])
    49da8b
                 h = h.encode('us-ascii')
    49da8b
                 h = uheader(mlist, h, 'Subject', continuation_ws=ws)
    49da8b
    -            del msg['subject']
    49da8b
    -            msg['Subject'] = h
    49da8b
    +            change_header('Subject', h, mlist, msg, msgdata)
    49da8b
                 ss = u' '.join([recolon, subject])
    49da8b
                 ss = ss.encode('us-ascii')
    49da8b
                 ss = uheader(mlist, ss, 'Subject', continuation_ws=ws)
    49da8b
    @@ -321,8 +366,7 @@ def prefix_subject(mlist, msg, msgdata):
    49da8b
         # TK: Subject is concatenated and unicode string.
    49da8b
         subject = subject.encode(cset, 'replace')
    49da8b
         h.append(subject, cset)
    49da8b
    -    del msg['subject']
    49da8b
    -    msg['Subject'] = h
    49da8b
    +    change_header('Subject', h, mlist, msg, msgdata)
    49da8b
         ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws)
    49da8b
         ss.append(subject, cset)
    49da8b
         msgdata['stripped_subject'] = ss
    49da8b
    diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py
    49da8b
    index a362d96..2f1f38f 100644
    49da8b
    --- a/Mailman/Handlers/Moderate.py
    49da8b
    +++ b/Mailman/Handlers/Moderate.py
    49da8b
    @@ -21,6 +21,7 @@
    49da8b
     import re
    49da8b
     from email.MIMEMessage import MIMEMessage
    49da8b
     from email.MIMEText import MIMEText
    49da8b
    +from email.Utils import parseaddr
    49da8b
     
    49da8b
     from Mailman import mm_cfg
    49da8b
     from Mailman import Utils
    49da8b
    @@ -47,9 +48,34 @@ class ModeratedMemberPost(Hold.ModeratedPost):
    49da8b
     
    49da8b
     
    49da8b
     def process(mlist, msg, msgdata):
    49da8b
    -    if msgdata.get('approved') or msgdata.get('fromusenet'):
    49da8b
    +    if msgdata.get('approved'):
    49da8b
             return
    49da8b
    -    # First of all, is the poster a member or not?
    49da8b
    +    # Before anything else, check DMARC if necessary.
    49da8b
    +    msgdata['from_is_list'] = 0
    49da8b
    +    dn, addr = parseaddr(msg.get('from'))
    49da8b
    +    if addr and mlist.dmarc_moderation_action > 0:
    49da8b
    +        if Utils.IsDMARCProhibited(mlist, addr):
    49da8b
    +            # Note that for dmarc_moderation_action, 0 = Accept, 
    49da8b
    +            #    1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard
    49da8b
    +            if mlist.dmarc_moderation_action == 1:
    49da8b
    +                msgdata['from_is_list'] = 1
    49da8b
    +            elif mlist.dmarc_moderation_action == 2:
    49da8b
    +                msgdata['from_is_list'] = 2
    49da8b
    +            elif mlist.dmarc_moderation_action == 3:
    49da8b
    +                # Reject
    49da8b
    +                text = mlist.dmarc_moderation_notice
    49da8b
    +                if text:
    49da8b
    +                    text = Utils.wrap(text)
    49da8b
    +                else:
    49da8b
    +                    text = Utils.wrap(_(
    49da8b
    +"""You are not allowed to post to this mailing list From: a domain which
    49da8b
    +publishes a DMARC policy of reject or quarantine, and your message has been
    49da8b
    +automatically rejected.  If you think that your messages are being rejected in
    49da8b
    +error, contact the mailing list owner at %(listowner)s."""))
    49da8b
    +                raise Errors.RejectMessage, text
    49da8b
    +            elif mlist.dmarc_moderation_action == 4:
    49da8b
    +                raise Errors.DiscardMessage
    49da8b
    +    # Then, is the poster a member or not?
    49da8b
         for sender in msg.get_senders():
    49da8b
             if mlist.isMember(sender):
    49da8b
                 break
    49da8b
    @@ -105,7 +131,7 @@ def process(mlist, msg, msgdata):
    49da8b
         # moderation configuration variables.  Handle by way of generic non-member
    49da8b
         # action.
    49da8b
         assert 0 <= mlist.generic_nonmember_action <= 4
    49da8b
    -    if mlist.generic_nonmember_action == 0:
    49da8b
    +    if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'):
    49da8b
             # Accept
    49da8b
             return
    49da8b
         elif mlist.generic_nonmember_action == 1:
    49da8b
    diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py
    49da8b
    index 0d3ce49..2117290 100644
    49da8b
    --- a/Mailman/Handlers/Tagger.py
    49da8b
    +++ b/Mailman/Handlers/Tagger.py
    49da8b
    @@ -24,6 +24,7 @@ import email.Iterators
    49da8b
     
    49da8b
     from Mailman import Utils
    49da8b
     from Mailman.Logging.Syslog import syslog
    49da8b
    +from Mailman.Handlers.CookHeaders import change_header
    49da8b
     
    49da8b
     CRNL = '\r\n'
    49da8b
     EMPTYSTRING = ''
    49da8b
    @@ -60,8 +61,9 @@ def process(mlist, msg, msgdata):
    49da8b
                     break
    49da8b
         if hits:
    49da8b
             msgdata['topichits'] = hits.keys()
    49da8b
    -        msg['X-Topics'] = NLTAB.join(hits.keys())
    49da8b
    -    
    49da8b
    +        change_header('X-Topics', NLTAB.join(hits.keys()),
    49da8b
    +                      mlist, msg, msgdata, delete=False)
    49da8b
    +
    49da8b
     
    49da8b
     
    49da8b
     def scanbody(msg, numlines=None):
    49da8b
    diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py
    49da8b
    new file mode 100644
    49da8b
    index 0000000..9678f6f
    49da8b
    --- /dev/null
    49da8b
    +++ b/Mailman/Handlers/WrapMessage.py
    49da8b
    @@ -0,0 +1,72 @@
    49da8b
    +# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
    49da8b
    +#
    49da8b
    +# This program is free software; you can redistribute it and/or
    49da8b
    +# modify it under the terms of the GNU General Public License
    49da8b
    +# as published by the Free Software Foundation; either version 2
    49da8b
    +# of the License, or (at your option) any later version.
    49da8b
    +#
    49da8b
    +# This program is distributed in the hope that it will be useful,
    49da8b
    +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    49da8b
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    49da8b
    +# GNU General Public License for more details.
    49da8b
    +#
    49da8b
    +# You should have received a copy of the GNU General Public License
    49da8b
    +# along with this program; if not, write to the Free Software
    49da8b
    +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
    49da8b
    +# USA.
    49da8b
    +
    49da8b
    +"""Wrap the message in an outer message/rfc822 part and transfer/add
    49da8b
    +some headers from the original.
    49da8b
    +
    49da8b
    +Also, in the case of Munge From, replace the From: and Reply-To: in the
    49da8b
    +original message.
    49da8b
    +"""
    49da8b
    +
    49da8b
    +import copy
    49da8b
    +
    49da8b
    +from Mailman import mm_cfg
    49da8b
    +from Mailman.Utils import unique_message_id
    49da8b
    +from Mailman.Message import Message
    49da8b
    +
    49da8b
    +# Headers from the original that we want to keep in the wrapper.
    49da8b
    +KEEPERS = ('to',
    49da8b
    +           'in-reply-to',
    49da8b
    +           'references',
    49da8b
    +           'x-mailman-approved-at',
    49da8b
    +          )
    49da8b
    +
    49da8b
    +
    49da8b
    +
    49da8b
    +def process(mlist, msg, msgdata):
    49da8b
    +    # This is the negation of we're wrapping because dmarc_moderation_action
    49da8b
    +    # is wrap this message or from_is_list applies and is wrap.
    49da8b
    +    if not (msgdata.get('from_is_list') == 2 or
    49da8b
    +            (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
    49da8b
    +        # Now see if we need to add a From: and/or Reply-To: without wrapping.
    49da8b
    +        a_h = msgdata.get('add_header')
    49da8b
    +        if a_h:
    49da8b
    +            if a_h.get('From'):
    49da8b
    +                del msg['from']
    49da8b
    +                msg['From'] = a_h.get('From')
    49da8b
    +            if a_h.get('Reply-To'):
    49da8b
    +                del msg['reply-to']
    49da8b
    +                msg['Reply-To'] = a_h.get('Reply-To')
    49da8b
    +        return
    49da8b
    +
    49da8b
    +    # There are various headers in msg that we don't want, so we basically
    49da8b
    +    # make a copy of the msg, then delete almost everything and set/copy
    49da8b
    +    # what we want.
    49da8b
    +    omsg = copy.deepcopy(msg)
    49da8b
    +    for key in msg.keys():
    49da8b
    +        if key.lower() not in KEEPERS:
    49da8b
    +            del msg[key]
    49da8b
    +    msg['MIME-Version'] = '1.0'
    49da8b
    +    msg['Content-Type'] = 'message/rfc822'
    49da8b
    +    msg['Content-Disposition'] = 'inline'
    49da8b
    +    msg['Message-ID'] = unique_message_id(mlist)
    49da8b
    +    # Add the headers from CookHeaders.
    49da8b
    +    for k, v in msgdata['add_header'].items():
    49da8b
    +        msg[k] = v
    49da8b
    +    # And set the payload.
    49da8b
    +    msg.set_payload(omsg.as_string())
    49da8b
    +
    49da8b
    diff --git a/Mailman/MailList.py b/Mailman/MailList.py
    49da8b
    old mode 100644
    49da8b
    new mode 100755
    49da8b
    index 6083fb1..f948b69
    49da8b
    --- a/Mailman/MailList.py
    49da8b
    +++ b/Mailman/MailList.py
    49da8b
    @@ -346,6 +346,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    49da8b
             self.bounce_matching_headers = \
    49da8b
                     mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS
    49da8b
             self.header_filter_rules = []
    49da8b
    +        self.from_is_list = mm_cfg.DEFAULT_FROM_IS_LIST
    49da8b
             self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST
    49da8b
             internalname = self.internal_name()
    49da8b
             self.real_name = internalname[0].upper() + internalname[1:]
    49da8b
    @@ -386,6 +387,10 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    49da8b
             # 2==Discard
    49da8b
             self.member_moderation_action = 0
    49da8b
             self.member_moderation_notice = ''
    49da8b
    +        self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    49da8b
    +        self.dmarc_quarantine_moderation_action = (
    49da8b
    +            mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    49da8b
    +        self.dmarc_moderation_notice = ''
    49da8b
             self.accept_these_nonmembers = []
    49da8b
             self.hold_these_nonmembers = []
    49da8b
             self.reject_these_nonmembers = []
    49da8b
    @@ -712,7 +717,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    49da8b
         def CheckVersion(self, stored_state):
    49da8b
             """Auto-update schema if necessary."""
    49da8b
             if self.data_version >= mm_cfg.DATA_FILE_VERSION:
    49da8b
    -            return
    49da8b
    +            # Some lists could have been created by newer Mailman version than
    49da8b
    +            # this one. We are adding just few variables, so check for these
    49da8b
    +            # variables explicitely.
    49da8b
    +            if (hasattr(self, "from_is_list")
    49da8b
    +                and hasattr(self, "dmarc_moderation_action")
    49da8b
    +                and hasattr(self, "dmarc_moderation_notice")
    49da8b
    +                and hasattr(self, "dmarc_quarantine_moderation_action")):
    49da8b
    +                return
    49da8b
             # Initialize any new variables
    49da8b
             self.InitVars()
    49da8b
             # Then reload the database (but don't recurse).  Force a reload even
    49da8b
    @@ -1025,7 +1030,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    49da8b
             # And send an acknowledgement to the user...
    49da8b
             if userack:
    49da8b
                 self.SendUnsubscribeAck(emailaddr, userlang)
    49da8b
    -        # ...and to the administrator
    49da8b
    +        # ...and to the administrator in the correct language.  (LP: #1308655)
    49da8b
    +        i18n.set_language(self.preferred_language)
    49da8b
             if admin_notif:
    49da8b
                 realname = self.real_name
    49da8b
                 subject = _('%(realname)s unsubscribe notification')
    49da8b
    diff --git a/Mailman/Message.py b/Mailman/Message.py
    49da8b
    index 84e4aa2..13e7ff2 100644
    49da8b
    --- a/Mailman/Message.py
    49da8b
    +++ b/Mailman/Message.py
    49da8b
    @@ -61,6 +61,43 @@ class Generator(email.Generator.Generator):
    49da8b
     
    49da8b
     
    49da8b
     
    49da8b
    +class Generator(email.Generator.Generator):
    49da8b
    +    """Generates output from a Message object tree, keeping signatures.
    49da8b
    +
    49da8b
    +       Headers will by default _not_ be folded in attachments.
    49da8b
    +    """
    49da8b
    +    def __init__(self, outfp, mangle_from_=True,
    49da8b
    +                 maxheaderlen=78, children_maxheaderlen=0):
    49da8b
    +        email.Generator.Generator.__init__(self, outfp,
    49da8b
    +                mangle_from_=mangle_from_, maxheaderlen=maxheaderlen)
    49da8b
    +        self.__children_maxheaderlen = children_maxheaderlen
    49da8b
    +
    49da8b
    +    def clone(self, fp):
    49da8b
    +        """Clone this generator with maxheaderlen set for children"""
    49da8b
    +        return self.__class__(fp, self._mangle_from_,
    49da8b
    +                self.__children_maxheaderlen, self.__children_maxheaderlen)
    49da8b
    +
    49da8b
    +    # This is the _handle_message method with the fix for bug 7970.
    49da8b
    +    def _handle_message(self, msg):
    49da8b
    +        s = StringIO()
    49da8b
    +        g = self.clone(s)
    49da8b
    +        # The payload of a message/rfc822 part should be a multipart sequence
    49da8b
    +        # of length 1.  The zeroth element of the list should be the Message
    49da8b
    +        # object for the subpart.  Extract that object, stringify it, and
    49da8b
    +        # write it out.
    49da8b
    +        # Except, it turns out, when it's a string instead, which happens when
    49da8b
    +        # and only when HeaderParser is used on a message of mime type
    49da8b
    +        # message/rfc822.  Such messages are generated by, for example,
    49da8b
    +        # Groupwise when forwarding unadorned messages.  (Issue 7970.)  So
    49da8b
    +        # in that case we just emit the string body.
    49da8b
    +        payload = msg.get_payload()
    49da8b
    +        if isinstance(payload, list):
    49da8b
    +            g.flatten(msg.get_payload(0), unixfrom=False)
    49da8b
    +            payload = s.getvalue()
    49da8b
    +        self._fp.write(payload)
    49da8b
    +
    49da8b
    +
    49da8b
    +
    49da8b
     class Message(email.Message.Message):
    49da8b
         def __init__(self):
    49da8b
             # We need a version number so that we can optimize __setstate__()
    49da8b
    @@ -243,6 +280,20 @@ class Message(email.Message.Message):
    49da8b
             return fp.getvalue()
    49da8b
     
    49da8b
     
    49da8b
    +    def as_string(self, unixfrom=False, mangle_from_=True):
    49da8b
    +        """Return entire formatted message as a string using
    49da8b
    +        Mailman.Message.Generator.
    49da8b
    +
    49da8b
    +        Operates like email.Message.Message.as_string, only
    49da8b
    +	using Mailman's Message.Generator class. Only the top headers will
    49da8b
    +        get folded.
    49da8b
    +        """
    49da8b
    +        fp = StringIO()
    49da8b
    +        g = Generator(fp, mangle_from_=mangle_from_)
    49da8b
    +        g.flatten(self, unixfrom=unixfrom)
    49da8b
    +        return fp.getvalue()
    49da8b
    +
    49da8b
    +
    49da8b
     
    49da8b
     class UserNotification(Message):
    49da8b
         """Class for internally crafted messages."""
    49da8b
    diff --git a/Mailman/Utils.py b/Mailman/Utils.py
    49da8b
    index c8275df..8021942 100644
    49da8b
    --- a/Mailman/Utils.py
    49da8b
    +++ b/Mailman/Utils.py
    49da8b
    @@ -71,6 +71,14 @@ except NameError:
    49da8b
         True = 1
    49da8b
         False = 0
    49da8b
     
    49da8b
    +try:
    49da8b
    +    import dns.resolver
    49da8b
    +    import dns.rdatatype
    49da8b
    +    from dns.exception import DNSException
    49da8b
    +    dns_resolver = True
    49da8b
    +except ImportError:
    49da8b
    +    dns_resolver = False
    49da8b
    +
    49da8b
     EMPTYSTRING = ''
    49da8b
     UEMPTYSTRING = u''
    49da8b
     NL = '\n'
    49da8b
    @@ -1047,3 +1055,91 @@ def suspiciousHTML(html):
    49da8b
         else:
    49da8b
             return False
    49da8b
     
    49da8b
    +
    49da8b
    +
    49da8b
    +
    49da8b
    +# This takes an email address, and returns True if DMARC policy is p=reject
    49da8b
    +# or possibly quarantine.
    49da8b
    +def IsDMARCProhibited(mlist, email):
    49da8b
    +    if not dns_resolver:
    49da8b
    +         return False
    49da8b
    +
    49da8b
    +    email = email.lower()
    49da8b
    +    at_sign = email.find('@')
    49da8b
    +    if at_sign < 1:
    49da8b
    +        return False
    49da8b
    +    dmarc_domain = '_dmarc.' + email[at_sign+1:]
    49da8b
    +
    49da8b
    +    try:
    49da8b
    +        resolver = dns.resolver.Resolver()
    49da8b
    +        resolver.timeout = float(mm_cfg.DMARC_RESOLVER_TIMEOUT)
    49da8b
    +        resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME)
    49da8b
    +        txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
    49da8b
    +    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
    49da8b
    +        return False
    49da8b
    +    except DNSException, e:
    49da8b
    +        syslog('error',
    49da8b
    +               'DNSException: Unable to query DMARC policy for %s (%s). %s',
    49da8b
    +              email, dmarc_domain, e.__class__)
    49da8b
    +        return False
    49da8b
    +    else:
    49da8b
    +# people are already being dumb, don't trust them to provide honest DNS
    49da8b
    +# where the answer section only contains what was asked for, nor to include
    49da8b
    +# CNAMEs before the values they point to.
    49da8b
    +        full_record = ""
    49da8b
    +        results_by_name = {}
    49da8b
    +        cnames = {}
    49da8b
    +        want_names = set([dmarc_domain + '.'])
    49da8b
    +        for txt_rec in txt_recs.response.answer:
    49da8b
    +            if txt_rec.rdtype == dns.rdatatype.CNAME:
    49da8b
    +                cnames[txt_rec.name.to_text()] = (
    49da8b
    +                    txt_rec.items[0].target.to_text())
    49da8b
    +            if txt_rec.rdtype != dns.rdatatype.TXT:
    49da8b
    +                continue
    49da8b
    +            results_by_name.setdefault(txt_rec.name.to_text(), []).append(
    49da8b
    +                "".join(txt_rec.items[0].strings))
    49da8b
    +        expands = list(want_names)
    49da8b
    +        seen = set(expands)
    49da8b
    +        while expands:
    49da8b
    +            item = expands.pop(0)
    49da8b
    +            if item in cnames:
    49da8b
    +                if cnames[item] in seen:
    49da8b
    +                    continue # cname loop
    49da8b
    +                expands.append(cnames[item])
    49da8b
    +                seen.add(cnames[item])
    49da8b
    +                want_names.add(cnames[item])
    49da8b
    +                want_names.discard(item)
    49da8b
    +
    49da8b
    +        if len(want_names) != 1:
    49da8b
    +            syslog('error',
    49da8b
    +                   """multiple DMARC entries in results for %s,
    49da8b
    +                   processing each to be strict""",
    49da8b
    +                   dmarc_domain)
    49da8b
    +        for name in want_names:
    49da8b
    +            if name not in results_by_name:
    49da8b
    +                continue
    49da8b
    +            dmarcs = filter(lambda n: n.startswith('v=DMARC1;'),
    49da8b
    +                            results_by_name[name])
    49da8b
    +            if len(dmarcs) == 0:
    49da8b
    +                return False
    49da8b
    +            if len(dmarcs) > 1:
    49da8b
    +                syslog('error',
    49da8b
    +                       """RRset of TXT records for %s has %d v=DMARC1 entries;
    49da8b
    +                       testing them all""",
    49da8b
    +                        dmarc_domain, len(dmarc))
    49da8b
    +            for entry in dmarcs:
    49da8b
    +                if re.search(r'\bp=reject\b', entry, re.IGNORECASE):
    49da8b
    +                    syslog('vette',
    49da8b
    +                        'DMARC lookup for %s (%s) found p=reject in %s = %s',
    49da8b
    +                        email, dmarc_domain, name, entry)
    49da8b
    +                    return True
    49da8b
    +
    49da8b
    +                if (mlist.dmarc_quarantine_moderation_action and
    49da8b
    +                    re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)):
    49da8b
    +                    syslog('vette',
    49da8b
    +                      'DMARC lookup for %s (%s) found p=quarantine in %s = %s',
    49da8b
    +                            email, dmarc_domain, name, entry)
    49da8b
    +                    return True
    49da8b
    +
    49da8b
    +    return False
    49da8b
    +
    49da8b
    diff --git a/Mailman/Version.py b/Mailman/Version.py
    49da8b
    index 05e6500..af4a2df 100644
    49da8b
    --- a/Mailman/Version.py
    49da8b
    +++ b/Mailman/Version.py
    49da8b
    @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
    49da8b
                    (REL_LEVEL << 4)  | (REL_SERIAL << 0))
    49da8b
     
    49da8b
     # config.pck schema version number
    49da8b
    -DATA_FILE_VERSION = 100
    49da8b
    +DATA_FILE_VERSION = 101
    49da8b
     
    49da8b
     # qfile/*.db schema version number
    49da8b
     QFILE_SCHEMA_VERSION = 3
    49da8b
    diff --git a/Mailman/versions.py b/Mailman/versions.py
    49da8b
    old mode 100644
    49da8b
    new mode 100755
    49da8b
    index 81fafd5..138e770
    49da8b
    --- a/Mailman/versions.py
    49da8b
    +++ b/Mailman/versions.py
    49da8b
    @@ -313,6 +313,9 @@ def UpdateOldVars(l, stored_state):
    49da8b
                 pass
    49da8b
             else:
    49da8b
                 l.digest_members[k] = 0
    49da8b
    +    # from_is_list was called author_is_list in 2.1.16rc2 (only).
    49da8b
    +    PreferStored('author_is_list', 'from_is_list',
    49da8b
    +                 mm_cfg.DEFAULT_FROM_IS_LIST)
    49da8b
     
    49da8b
     
    49da8b
     
    49da8b
    @@ -383,6 +386,11 @@ def NewVars(l):
    49da8b
         # the current GUI description model.  So, 0==Hold, 1==Reject, 2==Discard
    49da8b
         add_only_if_missing('member_moderation_action', 0)
    49da8b
         add_only_if_missing('member_moderation_notice', '')
    49da8b
    +    add_only_if_missing('dmarc_moderation_action', 
    49da8b
    +                       mm_cfg.DEFAULT_DMARC_MODERATION_ACTION)
    49da8b
    +    add_only_if_missing('dmarc_quarantine_moderation_action',
    49da8b
    +                       mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    49da8b
    +    add_only_if_missing('dmarc_moderation_notice', '')
    49da8b
         add_only_if_missing('new_member_options',
    49da8b
                             mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS)
    49da8b
         # Emergency moderation flag
    49da8b
    diff --git a/contrib/majordomo2mailman.pl b/contrib/majordomo2mailman.pl
    49da8b
    index c874862..770dc57 100644
    49da8b
    --- a/contrib/majordomo2mailman.pl
    49da8b
    +++ b/contrib/majordomo2mailman.pl
    49da8b
    @@ -480,6 +480,7 @@ sub init_defaultmmconf {
    49da8b
     		  'max_num_recipients', "10",
    49da8b
     		  'forbidden_posters', "[]",
    49da8b
     		  'bounce_matching_headers',  "\"\"\"\n\"\"\"\n",
    49da8b
    +		  'from_is_list', "0",
    49da8b
     		  'anonymous_list', "0",
    49da8b
     		  'nondigestable', "1",
    49da8b
     		  'digestable', "1",