areguera / rpms / mailman

Forked from rpms/mailman 4 years ago
Clone
7812c9
diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py
7812c9
index a6a952a..0901b50 100644
7812c9
--- a/Mailman/Bouncers/SimpleMatch.py
7812c9
+++ b/Mailman/Bouncers/SimpleMatch.py
7812c9
@@ -42,7 +42,7 @@ PATTERNS = [
7812c9
     # sz-sb.de, corridor.com, nfg.nl
7812c9
     (_c('the following addresses had'),
7812c9
      _c('transcript of session follows'),
7812c9
-     _c(r'<(?P<fulladdr>[^>]*)>|\(expanded from: [^>)]*)>?\)')),
7812c9
+     _c(r'^ *(\(expanded from: )?[^\s@]+@[^\s@>]+?)>?\)?\s*$')),
7812c9
     # robanal.demon.co.uk
7812c9
     (_c('this message was created automatically by mail delivery software'),
7812c9
      _c('original message follows'),
7812c9
diff --git a/Mailman/Bouncers/Yahoo.py b/Mailman/Bouncers/Yahoo.py
7812c9
index b3edf4f..08ede54 100644
7812c9
--- a/Mailman/Bouncers/Yahoo.py
7812c9
+++ b/Mailman/Bouncers/Yahoo.py
7812c9
@@ -20,9 +20,15 @@ import re
7812c9
 import email
7812c9
 from email.Utils import parseaddr
7812c9
 
7812c9
-tcre = re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE)
7812c9
+tcre = (re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE),
7812c9
+        re.compile(r'Sorry, we were unable to deliver your message to '
7812c9
+                   r'the following address(\(es\))?\.',
7812c9
+                   re.IGNORECASE),
7812c9
+        )
7812c9
 acre = re.compile(r'<(?P<addr>[^>]*)>:')
7812c9
-ecre = re.compile(r'--- Original message follows')
7812c9
+ecre = (re.compile(r'--- Original message follows'),
7812c9
+        re.compile(r'--- Below this line is a copy of the message'),
7812c9
+        )
7812c9
 
7812c9
 
7812c9
 
7812c9
@@ -36,18 +42,26 @@ def process(msg):
7812c9
     # simple state machine
7812c9
     #     0 == nothing seen
7812c9
     #     1 == tag line seen
7812c9
+    #     2 == end line seen
7812c9
     state = 0
7812c9
     for line in email.Iterators.body_line_iterator(msg):
7812c9
         line = line.strip()
7812c9
-        if state == 0 and tcre.match(line):
7812c9
-            state = 1
7812c9
+        if state == 0:
7812c9
+            for cre in tcre:
7812c9
+                if cre.match(line):
7812c9
+                    state = 1
7812c9
+                    break
7812c9
         elif state == 1:
7812c9
             mo = acre.match(line)
7812c9
             if mo:
7812c9
                 addrs.append(mo.group('addr'))
7812c9
                 continue
7812c9
-            mo = ecre.match(line)
7812c9
-            if mo:
7812c9
-                # we're at the end of the error response
7812c9
-                break
7812c9
+            for cre in ecre:
7812c9
+                mo = cre.match(line)
7812c9
+                if mo:
7812c9
+                    # we're at the end of the error response
7812c9
+                    state = 2
7812c9
+                    break
7812c9
+        elif state == 2:
7812c9
+            break
7812c9
     return addrs
7812c9
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
7812c9
old mode 100644
7812c9
new mode 100755
7812c9
index 4fe63db..8e42f54
7812c9
--- a/Mailman/Defaults.py.in
7812c9
+++ b/Mailman/Defaults.py.in
7812c9
@@ -505,6 +505,7 @@ GLOBAL_PIPELINE = [
7812c9
     # (outgoing) path, finally leaving the message in the outgoing queue.
7812c9
     'AfterDelivery',
7812c9
     'Acknowledge',
7812c9
+    'WrapMessage',
7812c9
     'ToOutgoing',
7812c9
     ]
7812c9
 
7812c9
@@ -914,6 +915,29 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No
7812c9
 # moderators?
7812c9
 DEFAULT_FORWARD_AUTO_DISCARDS = Yes
7812c9
 
7812c9
+# Shall dmarc_moderation_action be applied to messages From: domains with
7812c9
+# a DMARC policy of quarantine as well as reject?  This sets the default for
7812c9
+# the list setting that controls it.
7812c9
+DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION = Yes
7812c9
+
7812c9
+# Default action for posts whose From: address domain has a DMARC policy of
7812c9
+# reject or quarantine.  See DEFAULT_FROM_IS_LIST below.  Whatever is set as
7812c9
+# the default here precludes the list owner from setting a lower value.
7812c9
+# 0 = Accept
7812c9
+# 1 = Munge From
7812c9
+# 2 = Wrap Message
7812c9
+# 3 = Reject
7812c9
+# 4 = Discard
7812c9
+DEFAULT_DMARC_MODERATION_ACTION = 0
7812c9
+
7812c9
+# Parameters for DMARC DNS lookups. If you are seeing 'DNSException:
7812c9
+# Unable to query DMARC policy ...' entries in your error log, you may need
7812c9
+# to adjust these.
7812c9
+# The time to wait for a response from a name server before timeout.
7812c9
+DMARC_RESOLVER_TIMEOUT = seconds(3)
7812c9
+# The total time to spend trying to get an answer to the question.
7812c9
+DMARC_RESOLVER_LIFETIME = seconds(5)
7812c9
+
7812c9
 # What shold happen to non-member posts which are do not match explicit
7812c9
 # non-member actions?
7812c9
 # 0 = Accept
7812c9
@@ -950,6 +974,25 @@ DEFAULT_SEND_WELCOME_MSG = Yes
7812c9
 # Send goodbye messages to unsubscribed members?
7812c9
 DEFAULT_SEND_GOODBYE_MSG = Yes
7812c9
 
7812c9
+# Some list posts and mail to the -owner address may contain DomainKey or
7812c9
+# DomainKeys Identified Mail (DKIM) signature headers <http://www.dkim.org/>.
7812c9
+# Various list transformations to the message such as adding a list header or
7812c9
+# footer or scrubbing attachments or even reply-to munging can break these
7812c9
+# signatures.  It is generally felt that these signatures have value, even if
7812c9
+# broken and even if the outgoing message is resigned.  However, some sites
7812c9
+# may wish to remove these headers by setting this to Yes.
7812c9
+REMOVE_DKIM_HEADERS = No
7812c9
+
7812c9
+# The following is a three way setting.  It sets the default for the list's
7812c9
+# from_is_list policy which is applied to all posts except those for which a
7812c9
+# dmarc_moderation_action other than accept applies.
7812c9
+# 0 -> Do not rewrite the From: or wrap the message.
7812c9
+# 1 -> Rewrite the From: header of posts replacing the posters address with
7812c9
+#      that of the list.  Also see REMOVE_DKIM_HEADERS above.
7812c9
+# 2 -> Do not modify the From: of the message, but wrap the message in an outer
7812c9
+#      message From the list address.
7812c9
+DEFAULT_FROM_IS_LIST = 0
7812c9
+
7812c9
 # Wipe sender information, and make it look like the list-admin
7812c9
 # address sends all messages
7812c9
 DEFAULT_ANONYMOUS_LIST = No
7812c9
diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py
7812c9
index 8271a30..05dc9ba 100644
7812c9
--- a/Mailman/Gui/General.py
7812c9
+++ b/Mailman/Gui/General.py
7812c9
@@ -153,6 +153,72 @@ class General(GUIBase):
7812c9
                             (listname %%05d) -> (listname 00123)
7812c9
              """)),
7812c9
 
7812c9
+            ('from_is_list', mm_cfg.Radio,
7812c9
+             (_('No'), _('Munge From'), _('Wrap Message')), 0,
7812c9
+             _("""Replace the From: header address with the list's posting
7812c9
+             address to mitigate issues stemming from the original From:
7812c9
+             domain's DMARC or similar policies."""),
7812c9
+             _("""Several protocols now in wide use attempt to ensure that use
7812c9
+             of the domain in the author's address (ie, in the From: header
7812c9
+             field) is authorized by that domain.  These protocols may be
7812c9
+             incompatible with common list features such as footers, causing
7812c9
+             participating email services to bounce list traffic merely
7812c9
+             because of the address in the From: field.  This has resulted
7812c9
+             in members being unsubscribed despite being perfectly able to
7812c9
+             receive mail.
7812c9
+             

7812c9
+             The following actions are applied to all list messages when
7812c9
+             selected here.  To apply these actions only to messages where the
7812c9
+             domain in the From: header is determined to use such a protocol,
7812c9
+             see the 
7812c9
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
7812c9
+             dmarc_moderation_action settings under Privacy options...
7812c9
+             -> Sender filters.
7812c9
+             

Settings:

7812c9
+             
7812c9
+             
No
7812c9
+             
Do nothing special. This is appropriate for anonymous lists.
7812c9
+             It is appropriate for dedicated announcement lists, unless the
7812c9
+             From: address of authorized posters might be in a domain with a
7812c9
+             DMARC or similar policy. It is also appropriate if you choose to
7812c9
+             use dmarc_moderation_action other than Accept for this list.
7812c9
+             
Munge From
7812c9
+             
This action replaces the poster's address in the From: header
7812c9
+             with the list's posting address and adds the poster's address to
7812c9
+             the addresses in the original Reply-To: header.
7812c9
+             
Wrap Message
7812c9
+             
Just wrap the message in an outer message with the From:
7812c9
+             header containing the list's posting address and with the original
7812c9
+             From: address added to the addresses in the original Reply-To:
7812c9
+             header and with Content-Type: message/rfc822.  This is effectively
7812c9
+             a one message MIME format digest.
7812c9
+             
7812c9
+             

The transformations for anonymous_list are applied before

7812c9
+             any of these actions. It is not useful to apply actions other
7812c9
+             than No to an anonymous list, and if you do so, the result may
7812c9
+             be surprising.
7812c9
+             

The Reply-To: header munging actions below interact with these

7812c9
+             actions as follows:
7812c9
+             

first_strip_reply_to = Yes will remove all the incoming

7812c9
+             Reply-To: addresses but will still add the poster's address to
7812c9
+             Reply-To: for all three settings of reply_goes_to_list which
7812c9
+             respectively will result in just the poster's address, the
7812c9
+             poster's address and the list posting address or the poster's
7812c9
+             address and the explicit reply_to_address in the outgoing
7812c9
+             Reply-To: header. If first_strip_reply_to = No the poster's
7812c9
+             address in the original From: header, if not already included in
7812c9
+             the Reply-To:, will be added to any existing Reply-To:
7812c9
+             address(es).
7812c9
+             

These actions, whether selected here or via

7812c9
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
7812c9
+             dmarc_moderation_action, do not apply to messages in digests
7812c9
+             or archives or sent to usenet via the Mail<->News gateways.
7812c9
+             

If

7812c9
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
7812c9
+             dmarc_moderation_action applies to this message with an
7812c9
+             action other than Accept, that action rather than this is
7812c9
+             applied""")),
7812c9
+
7812c9
             ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0,
7812c9
              _("""Hide the sender of a message, replacing it with the list
7812c9
              address (Removes From, Sender and Reply-To fields)""")),
7812c9
diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py
7812c9
old mode 100644
7812c9
new mode 100755
7812c9
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
7812c9
index 75eff2b..5d717bb 100644
7812c9
--- a/Mailman/Gui/Privacy.py
7812c9
+++ b/Mailman/Gui/Privacy.py
7812c9
@@ -158,6 +158,11 @@ class Privacy(GUIBase):
7812c9
             ]
7812c9
 
7812c9
         adminurl = mlist.GetScriptURL('admin', absolute=1)
7812c9
+    
7812c9
+        if mlist.dmarc_quarantine_moderation_action:
7812c9
+            quarantine = _('/Quarantine')
7812c9
+        else:
7812c9
+            quarantine = ''
7812c9
         sender_rtn = [
7812c9
             _("""When a message is posted to the list, a series of
7812c9
             moderation steps are taken to decide whether a moderator must
7812c9
@@ -235,6 +240,59 @@ class Privacy(GUIBase):
7812c9
              >rejection notice to
7812c9
              be sent to moderated members who post to this list.""")),
7812c9
 
7812c9
+            ('dmarc_moderation_action', mm_cfg.Radio,
7812c9
+             (_('Accept'), _('Munge From'), _('Wrap Message'), _('Reject'),
7812c9
+                 _('Discard')), 0,
7812c9
+             _("""Action to take when anyone posts to the
7812c9
+             list from a domain with a DMARC Reject%(quarantine)s Policy."""),
7812c9
+
7812c9
+             _("""
  • Munge From -- applies the
7812c9
+             href="?VARHELP=general/from_is_list">from_is_list Munge From
7812c9
+             transformation to these messages.
7812c9
+
7812c9
+             

  • Wrap Message -- applies the
  • 7812c9
    +             href="?VARHELP=general/from_is_list">from_is_list Wrap
    7812c9
    +             Message transformation to these messages.
    7812c9
    +
    7812c9
    +             

  • Reject -- this automatically rejects the message by
  • 7812c9
    +             sending a bounce notice to the post's author.  The text of the
    7812c9
    +             bounce notice can be 
    7812c9
    +             href="?VARHELP=privacy/sender/dmarc_moderation_notice"
    7812c9
    +             >configured by you.
    7812c9
    +
    7812c9
    +             

  • Discard -- this simply discards the message, with
  • 7812c9
    +             no notice sent to the post's author.
    7812c9
    +             
    7812c9
    +
    7812c9
    +             

    This setting takes precedence over the

    7812c9
    +             href="?VARHELP=general/from_is_list"> from_is_list setting
    7812c9
    +             if the message is From: an affected domain and the setting is
    7812c9
    +             other than Accept.""")),
    7812c9
    +
    7812c9
    +            ('dmarc_quarantine_moderation_action', mm_cfg.Radio,
    7812c9
    +             (_('No'), _('Yes')), 0,
    7812c9
    +             _("""Shall the above dmarc_moderation_action apply to messages
    7812c9
    +               From: domains with DMARC p=quarantine as well as p=reject"""),
    7812c9
    +
    7812c9
    +             _("""
    • No -- this applies dmarc_moderation_action to
    7812c9
    +               only those posts From: a domain with DMARC p=reject.  This is
    7812c9
    +               appropriate if you are concerned about bounced messages, but
    7812c9
    +               want to apply dmarc_moderation_action to as few messages as
    7812c9
    +               possible.
    7812c9
    +               

  • Yes -- this applies dmarc_moderation_action to
  • 7812c9
    +               posts From: a domain with DMARC p=reject or p=quarantine.
    7812c9
    +               

    If a message is From: a domain with DMARC p=quarantine

    7812c9
    +               and dmarc_moderation_action is not applied (this set to No)
    7812c9
    +               the message will likely not bounce, but will be delivered to
    7812c9
    +               recipients' spam folders or other hard to find places.""")),
    7812c9
    +
    7812c9
    +            ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1,
    7812c9
    +             _("""Text to include in any
    7812c9
    +             
    7812c9
    +             >rejection notice to
    7812c9
    +             be sent to anyone who posts to this list from a domain
    7812c9
    +             with a DMARC Reject%(quarantine)s Policy.""")),
    7812c9
    +
    7812c9
                 _('Non-member filters'),
    7812c9
     
    7812c9
                 ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1,
    7812c9
    @@ -399,7 +457,7 @@ class Privacy(GUIBase):
    7812c9
                  case, each rule is matched in turn, with processing stopped after
    7812c9
                  the first match.
    7812c9
     
    7812c9
    -             Note that headers are collected from all the attachments 
    7812c9
    +             Note that headers are collected from all the attachments
    7812c9
                  (except for the mailman administrivia message) and
    7812c9
                  matched against the regular expressions. With this feature,
    7812c9
                  you can effectively sort out messages with dangerous file
    7812c9
    @@ -442,6 +500,11 @@ class Privacy(GUIBase):
    7812c9
             # an option.
    7812c9
             if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE:
    7812c9
                 val += 1
    7812c9
    +        if (property == 'dmarc_moderation_action' and
    7812c9
    +                val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION):
    7812c9
    +            doc.addError(_("""dmarc_moderation_action must be >= the configured
    7812c9
    +                           default value."""))
    7812c9
    +            val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    7812c9
             setattr(mlist, property, val)
    7812c9
     
    7812c9
         # We need to handle the header_filter_rules widgets specially, but
    7812c9
    diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py
    7812c9
    index 038034c..549d8e7 100644
    7812c9
    --- a/Mailman/Handlers/AvoidDuplicates.py
    7812c9
    +++ b/Mailman/Handlers/AvoidDuplicates.py
    7812c9
    @@ -24,6 +24,7 @@ warning header, or pass it through, depending on the user's preferences.
    7812c9
     
    7812c9
     from email.Utils import getaddresses, formataddr
    7812c9
     from Mailman import mm_cfg
    7812c9
    +from Mailman.Handlers.CookHeaders import change_header
    7812c9
     
    7812c9
     COMMASPACE = ', '
    7812c9
     
    7812c9
    @@ -95,6 +96,10 @@ def process(mlist, msg, msgdata):
    7812c9
         # Set the new list of recipients
    7812c9
         msgdata['recips'] = newrecips
    7812c9
         # RFC 2822 specifies zero or one CC header
    7812c9
    -    del msg['cc']
    7812c9
         if ccaddrs:
    7812c9
    -        msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()])
    7812c9
    +        change_header('Cc',
    7812c9
    +        COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]),
    7812c9
    +        mlist, msg, msgdata)
    7812c9
    +    else:
    7812c9
    +        del msg['cc']
    7812c9
    +
    7812c9
    diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
    7812c9
    old mode 100644
    7812c9
    new mode 100755
    7812c9
    index 8e7e668..c556967
    7812c9
    --- a/Mailman/Handlers/CookHeaders.py
    7812c9
    +++ b/Mailman/Handlers/CookHeaders.py
    7812c9
    @@ -64,13 +64,23 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None):
    7812c9
             charset = 'us-ascii'
    7812c9
         return Header(s, charset, maxlinelen, header_name, continuation_ws)
    7812c9
     
    7812c9
    +def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True):
    7812c9
    +    if ((msgdata.get('from_is_list') == 2 or
    7812c9
    +        (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and 
    7812c9
    +        not msgdata.get('_fasttrack')
    7812c9
    +       ) or name.lower() in ('from', 'reply-to'):
    7812c9
    +        msgdata.setdefault('add_header', {})[name] = value
    7812c9
    +    elif repl or not msg.has_key(name):
    7812c9
    +        if delete:
    7812c9
    +            del msg[name]
    7812c9
    +        msg[name] = value
    7812c9
    +
    7812c9
     
    7812c9
     
    7812c9
     def process(mlist, msg, msgdata):
    7812c9
         # Set the "X-Ack: no" header if noack flag is set.
    7812c9
         if msgdata.get('noack'):
    7812c9
    -        del msg['x-ack']
    7812c9
    -        msg['X-Ack'] = 'no'
    7812c9
    +        change_header('X-Ack', 'no', mlist, msg, msgdata)
    7812c9
         # Because we're going to modify various important headers in the email
    7812c9
         # message, we want to save some of the information in the msgdata
    7812c9
         # dictionary for later.  Specifically, the sender header will get waxed,
    7812c9
    @@ -87,7 +97,8 @@ def process(mlist, msg, msgdata):
    7812c9
                 pass
    7812c9
         # Mark message so we know we've been here, but leave any existing
    7812c9
         # X-BeenThere's intact.
    7812c9
    -    msg['X-BeenThere'] = mlist.GetListEmail()
    7812c9
    +    change_header('X-BeenThere', mlist.GetListEmail(),
    7812c9
    +                  mlist, msg, msgdata, delete=False)
    7812c9
         # Add Precedence: and other useful headers.  None of these are standard
    7812c9
         # and finding information on some of them are fairly difficult.  Some are
    7812c9
         # just common practice, and we'll add more here as they become necessary.
    7812c9
    @@ -101,12 +112,31 @@ def process(mlist, msg, msgdata):
    7812c9
         # known exploits in a particular version of Mailman and we know a site is
    7812c9
         # using such an old version, they may be vulnerable.  It's too easy to
    7812c9
         # edit the code to add a configuration variable to handle this.
    7812c9
    -    if not msg.has_key('x-mailman-version'):
    7812c9
    -        msg['X-Mailman-Version'] = mm_cfg.VERSION
    7812c9
    +    change_header('X-Mailman-Version', mm_cfg.VERSION,
    7812c9
    +                  mlist, msg, msgdata, repl=False)
    7812c9
         # We set "Precedence: list" because this is the recommendation from the
    7812c9
         # sendmail docs, the most authoritative source of this header's semantics.
    7812c9
    -    if not msg.has_key('precedence'):
    7812c9
    -        msg['Precedence'] = 'list'
    7812c9
    +    change_header('Precedence', 'list',
    7812c9
    +                  mlist, msg, msgdata, repl=False)
    7812c9
    +    # Do we change the from so the list takes ownership of the email
    7812c9
    +    if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack:
    7812c9
    +        realname, email = parseaddr(msg['from'])
    7812c9
    +        if not realname:
    7812c9
    +            if mlist.isMember(email):
    7812c9
    +                realname = mlist.getMemberName(email) or email
    7812c9
    +            else:
    7812c9
    +                realname = email
    7812c9
    +        # Remove domain from realname if it looks like an email address
    7812c9
    +        realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
    7812c9
    +        # Remember the original From: here for adding to Reply-To: below.
    7812c9
    +        o_from = parseaddr(msg['from'])
    7812c9
    +        change_header('From',
    7812c9
    +                      formataddr(('%s via %s' % (realname, mlist.real_name),
    7812c9
    +                                 mlist.GetListEmail())),
    7812c9
    +                      mlist, msg, msgdata)
    7812c9
    +    else:
    7812c9
    +        # Use this as a flag
    7812c9
    +        o_from = None
    7812c9
         # Reply-To: munging.  Do not do this if the message is "fast tracked",
    7812c9
         # meaning it is internally crafted and delivered to a specific user.  BAW:
    7812c9
         # Yuck, I really hate this feature but I've caved under the sheer pressure
    7812c9
    @@ -136,18 +166,23 @@ def process(mlist, msg, msgdata):
    7812c9
                 orig = msg.get_all('reply-to', [])
    7812c9
                 for pair in getaddresses(orig):
    7812c9
                     add(pair)
    7812c9
    +        # We also need to put the old From: in Reply-To: in all cases.
    7812c9
    +        if o_from:
    7812c9
    +            add(o_from)
    7812c9
             # Set Reply-To: header to point back to this list.  Add this last
    7812c9
             # because some folks think that some MUAs make it easier to delete
    7812c9
             # addresses from the right than from the left.
    7812c9
             if mlist.reply_goes_to_list == 1:
    7812c9
                 i18ndesc = uheader(mlist, mlist.description, 'Reply-To')
    7812c9
                 add((str(i18ndesc), mlist.GetListEmail()))
    7812c9
    -        del msg['reply-to']
    7812c9
             # Don't put Reply-To: back if there's nothing to add!
    7812c9
             if new:
    7812c9
                 # Preserve order
    7812c9
    -            msg['Reply-To'] = COMMASPACE.join(
    7812c9
    -                [formataddr(pair) for pair in new])
    7812c9
    +            change_header('Reply-To', 
    7812c9
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    7812c9
    +                          mlist, msg, msgdata)
    7812c9
    +        else:
    7812c9
    +            del msg['reply-to']
    7812c9
             # The To field normally contains the list posting address.  However
    7812c9
             # when messages are fully personalized, that header will get
    7812c9
             # overwritten with the address of the recipient.  We need to get the
    7812c9
    @@ -158,18 +193,31 @@ def process(mlist, msg, msgdata):
    7812c9
             # above code?
    7812c9
             # Also skip Cc if this is an anonymous list as list posting address
    7812c9
             # is already in From and Reply-To in this case.
    7812c9
    -        if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \
    7812c9
    -           and not mlist.anonymous_list:
    7812c9
    +        # We do add the Cc in cases where From: header munging is being done
    7812c9
    +        # because even though the list address is in From:, the Reply-To:
    7812c9
    +        # poster will override it. Brain dead MUAs may then address the list
    7812c9
    +        # twice on a 'reply all', but reasonable MUAs should do the right
    7812c9
    +        # thing.
    7812c9
    +        if (mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 and
    7812c9
    +            not mlist.anonymous_list):
    7812c9
                 # Watch out for existing Cc headers, merge, and remove dups.  Note
    7812c9
                 # that RFC 2822 says only zero or one Cc header is allowed.
    7812c9
                 new = []
    7812c9
                 d = {}
    7812c9
    -            for pair in getaddresses(msg.get_all('cc', [])):
    7812c9
    -                add(pair)
    7812c9
    +            # AvoidDuplicates may have set a new Cc: in msgdata.add_header,
    7812c9
    +            # so check that.
    7812c9
    +            if (msgdata.has_key('add_header') and
    7812c9
    +                    msgdata['add_header'].has_key('Cc')):
    7812c9
    +                for pair in getaddresses([msgdata['add_header']['Cc']]):
    7812c9
    +                    add(pair)
    7812c9
    +            else:
    7812c9
    +                for pair in getaddresses(msg.get_all('cc', [])):
    7812c9
    +                    add(pair)
    7812c9
                 i18ndesc = uheader(mlist, mlist.description, 'Cc')
    7812c9
                 add((str(i18ndesc), mlist.GetListEmail()))
    7812c9
    -            del msg['Cc']
    7812c9
    -            msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new])
    7812c9
    +            change_header('Cc',
    7812c9
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    7812c9
    +                          mlist, msg, msgdata)
    7812c9
         # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only
    7812c9
         # if the message is being crafted for a specific list (e.g. not for the
    7812c9
         # password reminders).
    7812c9
    @@ -191,8 +239,7 @@ def process(mlist, msg, msgdata):
    7812c9
             # without desc we need to ensure the MUST brackets
    7812c9
             listid_h = '<%s>' % listid
    7812c9
         # We always add a List-ID: header.
    7812c9
    -    del msg['list-id']
    7812c9
    -    msg['List-Id'] = listid_h
    7812c9
    +    change_header('List-Id', listid_h, mlist, msg, msgdata)
    7812c9
         # For internally crafted messages, we also add a (nonstandard),
    7812c9
         # "X-List-Administrivia: yes" header.  For all others (i.e. those coming
    7812c9
         # from list posts), we add a bunch of other RFC 2369 headers.
    7812c9
    @@ -219,13 +266,12 @@ def process(mlist, msg, msgdata):
    7812c9
         # First we delete any pre-existing headers because the RFC permits only
    7812c9
         # one copy of each, and we want to be sure it's ours.
    7812c9
         for h, v in headers.items():
    7812c9
    -        del msg[h]
    7812c9
             # Wrap these lines if they are too long.  78 character width probably
    7812c9
             # shouldn't be hardcoded, but is at least text-MUA friendly.  The
    7812c9
             # adding of 2 is for the colon-space separator.
    7812c9
             if len(h) + 2 + len(v) > 78:
    7812c9
                 v = CONTINUATION.join(v.split(', '))
    7812c9
    -        msg[h] = v
    7812c9
    +        change_header(h, v, mlist, msg, msgdata)
    7812c9
     
    7812c9
     
    7812c9
     
    7812c9
    @@ -302,8 +348,7 @@ def prefix_subject(mlist, msg, msgdata):
    7812c9
                         h = u' '.join([prefix, subject])
    7812c9
                 h = h.encode('us-ascii')
    7812c9
                 h = uheader(mlist, h, 'Subject', continuation_ws=ws)
    7812c9
    -            del msg['subject']
    7812c9
    -            msg['Subject'] = h
    7812c9
    +            change_header('Subject', h, mlist, msg, msgdata)
    7812c9
                 ss = u' '.join([recolon, subject])
    7812c9
                 ss = ss.encode('us-ascii')
    7812c9
                 ss = uheader(mlist, ss, 'Subject', continuation_ws=ws)
    7812c9
    @@ -321,8 +366,7 @@ def prefix_subject(mlist, msg, msgdata):
    7812c9
         # TK: Subject is concatenated and unicode string.
    7812c9
         subject = subject.encode(cset, 'replace')
    7812c9
         h.append(subject, cset)
    7812c9
    -    del msg['subject']
    7812c9
    -    msg['Subject'] = h
    7812c9
    +    change_header('Subject', h, mlist, msg, msgdata)
    7812c9
         ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws)
    7812c9
         ss.append(subject, cset)
    7812c9
         msgdata['stripped_subject'] = ss
    7812c9
    diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py
    7812c9
    index a362d96..2f1f38f 100644
    7812c9
    --- a/Mailman/Handlers/Moderate.py
    7812c9
    +++ b/Mailman/Handlers/Moderate.py
    7812c9
    @@ -21,6 +21,7 @@
    7812c9
     import re
    7812c9
     from email.MIMEMessage import MIMEMessage
    7812c9
     from email.MIMEText import MIMEText
    7812c9
    +from email.Utils import parseaddr
    7812c9
     
    7812c9
     from Mailman import mm_cfg
    7812c9
     from Mailman import Utils
    7812c9
    @@ -47,9 +48,34 @@ class ModeratedMemberPost(Hold.ModeratedPost):
    7812c9
     
    7812c9
     
    7812c9
     def process(mlist, msg, msgdata):
    7812c9
    -    if msgdata.get('approved') or msgdata.get('fromusenet'):
    7812c9
    +    if msgdata.get('approved'):
    7812c9
             return
    7812c9
    -    # First of all, is the poster a member or not?
    7812c9
    +    # Before anything else, check DMARC if necessary.
    7812c9
    +    msgdata['from_is_list'] = 0
    7812c9
    +    dn, addr = parseaddr(msg.get('from'))
    7812c9
    +    if addr and mlist.dmarc_moderation_action > 0:
    7812c9
    +        if Utils.IsDMARCProhibited(mlist, addr):
    7812c9
    +            # Note that for dmarc_moderation_action, 0 = Accept, 
    7812c9
    +            #    1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard
    7812c9
    +            if mlist.dmarc_moderation_action == 1:
    7812c9
    +                msgdata['from_is_list'] = 1
    7812c9
    +            elif mlist.dmarc_moderation_action == 2:
    7812c9
    +                msgdata['from_is_list'] = 2
    7812c9
    +            elif mlist.dmarc_moderation_action == 3:
    7812c9
    +                # Reject
    7812c9
    +                text = mlist.dmarc_moderation_notice
    7812c9
    +                if text:
    7812c9
    +                    text = Utils.wrap(text)
    7812c9
    +                else:
    7812c9
    +                    text = Utils.wrap(_(
    7812c9
    +"""You are not allowed to post to this mailing list From: a domain which
    7812c9
    +publishes a DMARC policy of reject or quarantine, and your message has been
    7812c9
    +automatically rejected.  If you think that your messages are being rejected in
    7812c9
    +error, contact the mailing list owner at %(listowner)s."""))
    7812c9
    +                raise Errors.RejectMessage, text
    7812c9
    +            elif mlist.dmarc_moderation_action == 4:
    7812c9
    +                raise Errors.DiscardMessage
    7812c9
    +    # Then, is the poster a member or not?
    7812c9
         for sender in msg.get_senders():
    7812c9
             if mlist.isMember(sender):
    7812c9
                 break
    7812c9
    @@ -105,7 +131,7 @@ def process(mlist, msg, msgdata):
    7812c9
         # moderation configuration variables.  Handle by way of generic non-member
    7812c9
         # action.
    7812c9
         assert 0 <= mlist.generic_nonmember_action <= 4
    7812c9
    -    if mlist.generic_nonmember_action == 0:
    7812c9
    +    if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'):
    7812c9
             # Accept
    7812c9
             return
    7812c9
         elif mlist.generic_nonmember_action == 1:
    7812c9
    diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py
    7812c9
    index 0d3ce49..2117290 100644
    7812c9
    --- a/Mailman/Handlers/Tagger.py
    7812c9
    +++ b/Mailman/Handlers/Tagger.py
    7812c9
    @@ -24,6 +24,7 @@ import email.Iterators
    7812c9
     
    7812c9
     from Mailman import Utils
    7812c9
     from Mailman.Logging.Syslog import syslog
    7812c9
    +from Mailman.Handlers.CookHeaders import change_header
    7812c9
     
    7812c9
     CRNL = '\r\n'
    7812c9
     EMPTYSTRING = ''
    7812c9
    @@ -60,8 +61,9 @@ def process(mlist, msg, msgdata):
    7812c9
                     break
    7812c9
         if hits:
    7812c9
             msgdata['topichits'] = hits.keys()
    7812c9
    -        msg['X-Topics'] = NLTAB.join(hits.keys())
    7812c9
    -    
    7812c9
    +        change_header('X-Topics', NLTAB.join(hits.keys()),
    7812c9
    +                      mlist, msg, msgdata, delete=False)
    7812c9
    +
    7812c9
     
    7812c9
     
    7812c9
     def scanbody(msg, numlines=None):
    7812c9
    diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py
    7812c9
    new file mode 100644
    7812c9
    index 0000000..9678f6f
    7812c9
    --- /dev/null
    7812c9
    +++ b/Mailman/Handlers/WrapMessage.py
    7812c9
    @@ -0,0 +1,72 @@
    7812c9
    +# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
    7812c9
    +#
    7812c9
    +# This program is free software; you can redistribute it and/or
    7812c9
    +# modify it under the terms of the GNU General Public License
    7812c9
    +# as published by the Free Software Foundation; either version 2
    7812c9
    +# of the License, or (at your option) any later version.
    7812c9
    +#
    7812c9
    +# This program is distributed in the hope that it will be useful,
    7812c9
    +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    7812c9
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    7812c9
    +# GNU General Public License for more details.
    7812c9
    +#
    7812c9
    +# You should have received a copy of the GNU General Public License
    7812c9
    +# along with this program; if not, write to the Free Software
    7812c9
    +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
    7812c9
    +# USA.
    7812c9
    +
    7812c9
    +"""Wrap the message in an outer message/rfc822 part and transfer/add
    7812c9
    +some headers from the original.
    7812c9
    +
    7812c9
    +Also, in the case of Munge From, replace the From: and Reply-To: in the
    7812c9
    +original message.
    7812c9
    +"""
    7812c9
    +
    7812c9
    +import copy
    7812c9
    +
    7812c9
    +from Mailman import mm_cfg
    7812c9
    +from Mailman.Utils import unique_message_id
    7812c9
    +from Mailman.Message import Message
    7812c9
    +
    7812c9
    +# Headers from the original that we want to keep in the wrapper.
    7812c9
    +KEEPERS = ('to',
    7812c9
    +           'in-reply-to',
    7812c9
    +           'references',
    7812c9
    +           'x-mailman-approved-at',
    7812c9
    +          )
    7812c9
    +
    7812c9
    +
    7812c9
    +
    7812c9
    +def process(mlist, msg, msgdata):
    7812c9
    +    # This is the negation of we're wrapping because dmarc_moderation_action
    7812c9
    +    # is wrap this message or from_is_list applies and is wrap.
    7812c9
    +    if not (msgdata.get('from_is_list') == 2 or
    7812c9
    +            (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
    7812c9
    +        # Now see if we need to add a From: and/or Reply-To: without wrapping.
    7812c9
    +        a_h = msgdata.get('add_header')
    7812c9
    +        if a_h:
    7812c9
    +            if a_h.get('From'):
    7812c9
    +                del msg['from']
    7812c9
    +                msg['From'] = a_h.get('From')
    7812c9
    +            if a_h.get('Reply-To'):
    7812c9
    +                del msg['reply-to']
    7812c9
    +                msg['Reply-To'] = a_h.get('Reply-To')
    7812c9
    +        return
    7812c9
    +
    7812c9
    +    # There are various headers in msg that we don't want, so we basically
    7812c9
    +    # make a copy of the msg, then delete almost everything and set/copy
    7812c9
    +    # what we want.
    7812c9
    +    omsg = copy.deepcopy(msg)
    7812c9
    +    for key in msg.keys():
    7812c9
    +        if key.lower() not in KEEPERS:
    7812c9
    +            del msg[key]
    7812c9
    +    msg['MIME-Version'] = '1.0'
    7812c9
    +    msg['Content-Type'] = 'message/rfc822'
    7812c9
    +    msg['Content-Disposition'] = 'inline'
    7812c9
    +    msg['Message-ID'] = unique_message_id(mlist)
    7812c9
    +    # Add the headers from CookHeaders.
    7812c9
    +    for k, v in msgdata['add_header'].items():
    7812c9
    +        msg[k] = v
    7812c9
    +    # And set the payload.
    7812c9
    +    msg.set_payload(omsg.as_string())
    7812c9
    +
    7812c9
    diff --git a/Mailman/MailList.py b/Mailman/MailList.py
    7812c9
    old mode 100644
    7812c9
    new mode 100755
    7812c9
    index 6083fb1..f948b69
    7812c9
    --- a/Mailman/MailList.py
    7812c9
    +++ b/Mailman/MailList.py
    7812c9
    @@ -346,6 +346,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    7812c9
             self.bounce_matching_headers = \
    7812c9
                     mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS
    7812c9
             self.header_filter_rules = []
    7812c9
    +        self.from_is_list = mm_cfg.DEFAULT_FROM_IS_LIST
    7812c9
             self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST
    7812c9
             internalname = self.internal_name()
    7812c9
             self.real_name = internalname[0].upper() + internalname[1:]
    7812c9
    @@ -386,6 +387,10 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    7812c9
             # 2==Discard
    7812c9
             self.member_moderation_action = 0
    7812c9
             self.member_moderation_notice = ''
    7812c9
    +        self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    7812c9
    +        self.dmarc_quarantine_moderation_action = (
    7812c9
    +            mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    7812c9
    +        self.dmarc_moderation_notice = ''
    7812c9
             self.accept_these_nonmembers = []
    7812c9
             self.hold_these_nonmembers = []
    7812c9
             self.reject_these_nonmembers = []
    7812c9
    @@ -712,7 +717,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    7812c9
         def CheckVersion(self, stored_state):
    7812c9
             """Auto-update schema if necessary."""
    7812c9
             if self.data_version >= mm_cfg.DATA_FILE_VERSION:
    7812c9
    -            return
    7812c9
    +            # Some lists could have been created by newer Mailman version than
    7812c9
    +            # this one. We are adding just few variables, so check for these
    7812c9
    +            # variables explicitely.
    7812c9
    +            if (hasattr(self, "from_is_list")
    7812c9
    +                and hasattr(self, "dmarc_moderation_action")
    7812c9
    +                and hasattr(self, "dmarc_moderation_notice")
    7812c9
    +                and hasattr(self, "dmarc_quarantine_moderation_action")):
    7812c9
    +                return
    7812c9
             # Initialize any new variables
    7812c9
             self.InitVars()
    7812c9
             # Then reload the database (but don't recurse).  Force a reload even
    7812c9
    @@ -1025,7 +1030,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    7812c9
             # And send an acknowledgement to the user...
    7812c9
             if userack:
    7812c9
                 self.SendUnsubscribeAck(emailaddr, userlang)
    7812c9
    -        # ...and to the administrator
    7812c9
    +        # ...and to the administrator in the correct language.  (LP: #1308655)
    7812c9
    +        i18n.set_language(self.preferred_language)
    7812c9
             if admin_notif:
    7812c9
                 realname = self.real_name
    7812c9
                 subject = _('%(realname)s unsubscribe notification')
    7812c9
    diff --git a/Mailman/Message.py b/Mailman/Message.py
    7812c9
    index 84e4aa2..13e7ff2 100644
    7812c9
    --- a/Mailman/Message.py
    7812c9
    +++ b/Mailman/Message.py
    7812c9
    @@ -61,6 +61,43 @@ class Generator(email.Generator.Generator):
    7812c9
     
    7812c9
     
    7812c9
     
    7812c9
    +class Generator(email.Generator.Generator):
    7812c9
    +    """Generates output from a Message object tree, keeping signatures.
    7812c9
    +
    7812c9
    +       Headers will by default _not_ be folded in attachments.
    7812c9
    +    """
    7812c9
    +    def __init__(self, outfp, mangle_from_=True,
    7812c9
    +                 maxheaderlen=78, children_maxheaderlen=0):
    7812c9
    +        email.Generator.Generator.__init__(self, outfp,
    7812c9
    +                mangle_from_=mangle_from_, maxheaderlen=maxheaderlen)
    7812c9
    +        self.__children_maxheaderlen = children_maxheaderlen
    7812c9
    +
    7812c9
    +    def clone(self, fp):
    7812c9
    +        """Clone this generator with maxheaderlen set for children"""
    7812c9
    +        return self.__class__(fp, self._mangle_from_,
    7812c9
    +                self.__children_maxheaderlen, self.__children_maxheaderlen)
    7812c9
    +
    7812c9
    +    # This is the _handle_message method with the fix for bug 7970.
    7812c9
    +    def _handle_message(self, msg):
    7812c9
    +        s = StringIO()
    7812c9
    +        g = self.clone(s)
    7812c9
    +        # The payload of a message/rfc822 part should be a multipart sequence
    7812c9
    +        # of length 1.  The zeroth element of the list should be the Message
    7812c9
    +        # object for the subpart.  Extract that object, stringify it, and
    7812c9
    +        # write it out.
    7812c9
    +        # Except, it turns out, when it's a string instead, which happens when
    7812c9
    +        # and only when HeaderParser is used on a message of mime type
    7812c9
    +        # message/rfc822.  Such messages are generated by, for example,
    7812c9
    +        # Groupwise when forwarding unadorned messages.  (Issue 7970.)  So
    7812c9
    +        # in that case we just emit the string body.
    7812c9
    +        payload = msg.get_payload()
    7812c9
    +        if isinstance(payload, list):
    7812c9
    +            g.flatten(msg.get_payload(0), unixfrom=False)
    7812c9
    +            payload = s.getvalue()
    7812c9
    +        self._fp.write(payload)
    7812c9
    +
    7812c9
    +
    7812c9
    +
    7812c9
     class Message(email.Message.Message):
    7812c9
         def __init__(self):
    7812c9
             # We need a version number so that we can optimize __setstate__()
    7812c9
    @@ -243,6 +280,20 @@ class Message(email.Message.Message):
    7812c9
             return fp.getvalue()
    7812c9
     
    7812c9
     
    7812c9
    +    def as_string(self, unixfrom=False, mangle_from_=True):
    7812c9
    +        """Return entire formatted message as a string using
    7812c9
    +        Mailman.Message.Generator.
    7812c9
    +
    7812c9
    +        Operates like email.Message.Message.as_string, only
    7812c9
    +	using Mailman's Message.Generator class. Only the top headers will
    7812c9
    +        get folded.
    7812c9
    +        """
    7812c9
    +        fp = StringIO()
    7812c9
    +        g = Generator(fp, mangle_from_=mangle_from_)
    7812c9
    +        g.flatten(self, unixfrom=unixfrom)
    7812c9
    +        return fp.getvalue()
    7812c9
    +
    7812c9
    +
    7812c9
     
    7812c9
     class UserNotification(Message):
    7812c9
         """Class for internally crafted messages."""
    7812c9
    diff --git a/Mailman/Utils.py b/Mailman/Utils.py
    7812c9
    index c8275df..8021942 100644
    7812c9
    --- a/Mailman/Utils.py
    7812c9
    +++ b/Mailman/Utils.py
    7812c9
    @@ -71,6 +71,14 @@ except NameError:
    7812c9
         True = 1
    7812c9
         False = 0
    7812c9
     
    7812c9
    +try:
    7812c9
    +    import dns.resolver
    7812c9
    +    import dns.rdatatype
    7812c9
    +    from dns.exception import DNSException
    7812c9
    +    dns_resolver = True
    7812c9
    +except ImportError:
    7812c9
    +    dns_resolver = False
    7812c9
    +
    7812c9
     EMPTYSTRING = ''
    7812c9
     UEMPTYSTRING = u''
    7812c9
     NL = '\n'
    7812c9
    @@ -1047,3 +1055,91 @@ def suspiciousHTML(html):
    7812c9
         else:
    7812c9
             return False
    7812c9
     
    7812c9
    +
    7812c9
    +
    7812c9
    +
    7812c9
    +# This takes an email address, and returns True if DMARC policy is p=reject
    7812c9
    +# or possibly quarantine.
    7812c9
    +def IsDMARCProhibited(mlist, email):
    7812c9
    +    if not dns_resolver:
    7812c9
    +         return False
    7812c9
    +
    7812c9
    +    email = email.lower()
    7812c9
    +    at_sign = email.find('@')
    7812c9
    +    if at_sign < 1:
    7812c9
    +        return False
    7812c9
    +    dmarc_domain = '_dmarc.' + email[at_sign+1:]
    7812c9
    +
    7812c9
    +    try:
    7812c9
    +        resolver = dns.resolver.Resolver()
    7812c9
    +        resolver.timeout = float(mm_cfg.DMARC_RESOLVER_TIMEOUT)
    7812c9
    +        resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME)
    7812c9
    +        txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
    7812c9
    +    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
    7812c9
    +        return False
    7812c9
    +    except DNSException, e:
    7812c9
    +        syslog('error',
    7812c9
    +               'DNSException: Unable to query DMARC policy for %s (%s). %s',
    7812c9
    +              email, dmarc_domain, e.__class__)
    7812c9
    +        return False
    7812c9
    +    else:
    7812c9
    +# people are already being dumb, don't trust them to provide honest DNS
    7812c9
    +# where the answer section only contains what was asked for, nor to include
    7812c9
    +# CNAMEs before the values they point to.
    7812c9
    +        full_record = ""
    7812c9
    +        results_by_name = {}
    7812c9
    +        cnames = {}
    7812c9
    +        want_names = set([dmarc_domain + '.'])
    7812c9
    +        for txt_rec in txt_recs.response.answer:
    7812c9
    +            if txt_rec.rdtype == dns.rdatatype.CNAME:
    7812c9
    +                cnames[txt_rec.name.to_text()] = (
    7812c9
    +                    txt_rec.items[0].target.to_text())
    7812c9
    +            if txt_rec.rdtype != dns.rdatatype.TXT:
    7812c9
    +                continue
    7812c9
    +            results_by_name.setdefault(txt_rec.name.to_text(), []).append(
    7812c9
    +                "".join(txt_rec.items[0].strings))
    7812c9
    +        expands = list(want_names)
    7812c9
    +        seen = set(expands)
    7812c9
    +        while expands:
    7812c9
    +            item = expands.pop(0)
    7812c9
    +            if item in cnames:
    7812c9
    +                if cnames[item] in seen:
    7812c9
    +                    continue # cname loop
    7812c9
    +                expands.append(cnames[item])
    7812c9
    +                seen.add(cnames[item])
    7812c9
    +                want_names.add(cnames[item])
    7812c9
    +                want_names.discard(item)
    7812c9
    +
    7812c9
    +        if len(want_names) != 1:
    7812c9
    +            syslog('error',
    7812c9
    +                   """multiple DMARC entries in results for %s,
    7812c9
    +                   processing each to be strict""",
    7812c9
    +                   dmarc_domain)
    7812c9
    +        for name in want_names:
    7812c9
    +            if name not in results_by_name:
    7812c9
    +                continue
    7812c9
    +            dmarcs = filter(lambda n: n.startswith('v=DMARC1;'),
    7812c9
    +                            results_by_name[name])
    7812c9
    +            if len(dmarcs) == 0:
    7812c9
    +                return False
    7812c9
    +            if len(dmarcs) > 1:
    7812c9
    +                syslog('error',
    7812c9
    +                       """RRset of TXT records for %s has %d v=DMARC1 entries;
    7812c9
    +                       testing them all""",
    7812c9
    +                        dmarc_domain, len(dmarc))
    7812c9
    +            for entry in dmarcs:
    7812c9
    +                if re.search(r'\bp=reject\b', entry, re.IGNORECASE):
    7812c9
    +                    syslog('vette',
    7812c9
    +                        'DMARC lookup for %s (%s) found p=reject in %s = %s',
    7812c9
    +                        email, dmarc_domain, name, entry)
    7812c9
    +                    return True
    7812c9
    +
    7812c9
    +                if (mlist.dmarc_quarantine_moderation_action and
    7812c9
    +                    re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)):
    7812c9
    +                    syslog('vette',
    7812c9
    +                      'DMARC lookup for %s (%s) found p=quarantine in %s = %s',
    7812c9
    +                            email, dmarc_domain, name, entry)
    7812c9
    +                    return True
    7812c9
    +
    7812c9
    +    return False
    7812c9
    +
    7812c9
    diff --git a/Mailman/Version.py b/Mailman/Version.py
    7812c9
    index 05e6500..af4a2df 100644
    7812c9
    --- a/Mailman/Version.py
    7812c9
    +++ b/Mailman/Version.py
    7812c9
    @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
    7812c9
                    (REL_LEVEL << 4)  | (REL_SERIAL << 0))
    7812c9
     
    7812c9
     # config.pck schema version number
    7812c9
    -DATA_FILE_VERSION = 100
    7812c9
    +DATA_FILE_VERSION = 101
    7812c9
     
    7812c9
     # qfile/*.db schema version number
    7812c9
     QFILE_SCHEMA_VERSION = 3
    7812c9
    diff --git a/Mailman/versions.py b/Mailman/versions.py
    7812c9
    old mode 100644
    7812c9
    new mode 100755
    7812c9
    index 81fafd5..138e770
    7812c9
    --- a/Mailman/versions.py
    7812c9
    +++ b/Mailman/versions.py
    7812c9
    @@ -313,6 +313,9 @@ def UpdateOldVars(l, stored_state):
    7812c9
                 pass
    7812c9
             else:
    7812c9
                 l.digest_members[k] = 0
    7812c9
    +    # from_is_list was called author_is_list in 2.1.16rc2 (only).
    7812c9
    +    PreferStored('author_is_list', 'from_is_list',
    7812c9
    +                 mm_cfg.DEFAULT_FROM_IS_LIST)
    7812c9
     
    7812c9
     
    7812c9
     
    7812c9
    @@ -383,6 +386,11 @@ def NewVars(l):
    7812c9
         # the current GUI description model.  So, 0==Hold, 1==Reject, 2==Discard
    7812c9
         add_only_if_missing('member_moderation_action', 0)
    7812c9
         add_only_if_missing('member_moderation_notice', '')
    7812c9
    +    add_only_if_missing('dmarc_moderation_action', 
    7812c9
    +                       mm_cfg.DEFAULT_DMARC_MODERATION_ACTION)
    7812c9
    +    add_only_if_missing('dmarc_quarantine_moderation_action',
    7812c9
    +                       mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    7812c9
    +    add_only_if_missing('dmarc_moderation_notice', '')
    7812c9
         add_only_if_missing('new_member_options',
    7812c9
                             mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS)
    7812c9
         # Emergency moderation flag
    7812c9
    diff --git a/contrib/majordomo2mailman.pl b/contrib/majordomo2mailman.pl
    7812c9
    index c874862..770dc57 100644
    7812c9
    --- a/contrib/majordomo2mailman.pl
    7812c9
    +++ b/contrib/majordomo2mailman.pl
    7812c9
    @@ -480,6 +480,7 @@ sub init_defaultmmconf {
    7812c9
     		  'max_num_recipients', "10",
    7812c9
     		  'forbidden_posters', "[]",
    7812c9
     		  'bounce_matching_headers',  "\"\"\"\n\"\"\"\n",
    7812c9
    +		  'from_is_list', "0",
    7812c9
     		  'anonymous_list', "0",
    7812c9
     		  'nondigestable', "1",
    7812c9
     		  'digestable', "1",