You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
905 lines
35 KiB
905 lines
35 KiB
# -*- coding: utf-8 -*-
|
|
"""Mailer for emencia.django.newsletter"""
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import threading
|
|
import mimetypes
|
|
import base64
|
|
import quopri
|
|
from random import sample
|
|
from StringIO import StringIO
|
|
from datetime import datetime
|
|
from datetime import timedelta
|
|
from smtplib import SMTPRecipientsRefused
|
|
from pymorphy2 import MorphAnalyzer
|
|
morph = MorphAnalyzer()
|
|
from logging import getLogger
|
|
log = getLogger('mail')
|
|
from itertools import chain
|
|
|
|
try:
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
from email.mime.Encoders import encode_base64
|
|
from email.mime.MIMEAudio import MIMEAudio
|
|
from email.mime.MIMEBase import MIMEBase
|
|
from email.mime.MIMEImage import MIMEImage
|
|
except ImportError: # Python 2.4 compatibility
|
|
from email.MIMEMultipart import MIMEMultipart
|
|
from email.MIMEText import MIMEText
|
|
from email.Encoders import encode_base64
|
|
from email.MIMEAudio import MIMEAudio
|
|
from email.MIMEBase import MIMEBase
|
|
from email.MIMEImage import MIMEImage
|
|
from email import message_from_file
|
|
from html2text import html2text as html2text_orig
|
|
from django.contrib.sites.models import Site
|
|
from django.template import Context, Template
|
|
from django.template.loader import render_to_string, get_template
|
|
from django.utils.encoding import smart_str
|
|
from django.utils.encoding import smart_unicode
|
|
from django.utils.translation import ugettext as _
|
|
from django.core.files.storage import default_storage
|
|
from django.core.urlresolvers import reverse
|
|
from django.core.exceptions import SuspiciousOperation
|
|
from django.db.models import Q
|
|
from django.contrib.staticfiles import finders
|
|
from django.conf import settings
|
|
|
|
from sorl.thumbnail import get_thumbnail
|
|
from sorl.thumbnail.images import ImageFile
|
|
from haystack.models import SearchResult
|
|
|
|
from emencia.django.newsletter.models import Newsletter
|
|
from emencia.django.newsletter.models import ContactMailingStatus
|
|
from emencia.django.newsletter.models import Contact
|
|
from emencia.django.newsletter.utils.tokens import tokenize
|
|
from emencia.django.newsletter.utils.newsletter import track_links
|
|
from emencia.django.newsletter.utils.newsletter import body_insertion
|
|
from emencia.django.newsletter.settings import TRACKING_LINKS
|
|
from emencia.django.newsletter.settings import TRACKING_IMAGE
|
|
from emencia.django.newsletter.settings import TRACKING_IMAGE_FORMAT
|
|
from emencia.django.newsletter.settings import UNIQUE_KEY_LENGTH
|
|
from emencia.django.newsletter.settings import UNIQUE_KEY_CHAR_SET
|
|
from emencia.django.newsletter.settings import INCLUDE_UNSUBSCRIPTION
|
|
from emencia.django.newsletter.settings import SLEEP_BETWEEN_SENDING
|
|
from emencia.django.newsletter.settings import \
|
|
RESTART_CONNECTION_BETWEEN_SENDING
|
|
|
|
import HTMLParser
|
|
import chardet
|
|
|
|
from theme.models import Theme
|
|
|
|
|
|
if not hasattr(timedelta, 'total_seconds'):
|
|
def total_seconds(td):
|
|
return ((td.microseconds +
|
|
(td.seconds + td.days * 24 * 3600) * 1000000) /
|
|
1000000.0)
|
|
else:
|
|
total_seconds = lambda td: td.total_seconds()
|
|
|
|
|
|
LINK_RE = re.compile(r"https?://([^ \n]+\n)+[^ \n]+", re.MULTILINE)
|
|
cid_rx = re.compile(r'^<(?P<id>.*)>$')
|
|
|
|
|
|
def html2text(html):
|
|
"""Use html2text but repair newlines cutting urls.
|
|
Need to use this hack until
|
|
https://github.com/aaronsw/html2text/issues/#issue/7 is not fixed"""
|
|
txt = html2text_orig(html)
|
|
return txt
|
|
|
|
|
|
def encodestring(instring, tabs=0):
|
|
outfile = StringIO()
|
|
quopri.encode(StringIO(instring), outfile, tabs)
|
|
return outfile.getvalue()
|
|
|
|
def decodestring(instring):
|
|
outfile = StringIO()
|
|
quopri.decode(StringIO(instring), outfile)
|
|
return outfile.getvalue()
|
|
|
|
|
|
dailymail_attahcments = {
|
|
'logo1': 'newsletter/images/logo1.png',
|
|
'logo2': 'newsletter/images/logo2.png',
|
|
'marker': 'newsletter/images/marker.png',
|
|
'calendar': 'newsletter/images/calendar.png',
|
|
'm1': 'newsletter/images/m1.png',
|
|
'm2': 'newsletter/images/m2.png',
|
|
'm3': 'newsletter/images/m3.png',
|
|
'm4': 'newsletter/images/m4.png',
|
|
'expo': 'newsletter/images/expo.png',
|
|
# 'news2': 'newsletter/images/news2.jpg',
|
|
# 'b': 'newsletter/images/b.png',
|
|
'arrow': 'newsletter/images/arrow.png',
|
|
'white_arrow': 'newsletter/images/white_arrow.png',
|
|
'site_logo': 'newsletter/images/site_logo.png',
|
|
'instagram': 'newsletter/images/instagram.png',
|
|
'youtube': 'newsletter/images/youtube.png',
|
|
'facebook': 'newsletter/images/facebook.png',
|
|
'linkedin': 'newsletter/images/linkedin.png',
|
|
'vk': 'newsletter/images/vk.png',
|
|
'twitter': 'newsletter/images/twitter.png',
|
|
}
|
|
context_attachments_size = {
|
|
'recommended': '170x170',
|
|
'news': '170x170',
|
|
'blog': '200x170',
|
|
'moscow': '109x114',
|
|
'russia': '109x114',
|
|
'foreign': '109x114',
|
|
}
|
|
|
|
|
|
class NewsLetterSender(object):
|
|
|
|
def __init__(self, newsletter, test=False, verbose=0):
|
|
self.test = test
|
|
self.verbose = verbose
|
|
self.newsletter = newsletter
|
|
self.settings_links = True
|
|
self.newsletter_template = Template(self.newsletter.content)
|
|
self.themes = dict(Theme.objects.language('ru').all().values_list('pk', 'name'))
|
|
|
|
|
|
self.newsletter_template2 = None
|
|
self.ab_state = Newsletter.A
|
|
if newsletter.ab_testing == True:
|
|
self.newsletter_template2 = Template(self.newsletter.content2)
|
|
if self.newsletter.ab_final_stage:
|
|
self.ab_state = self.newsletter.ab_final_choice
|
|
|
|
#
|
|
self.announce = self.newsletter.dailymail
|
|
self.local_dev = getattr(settings, 'LOCAL_DEV', False)
|
|
|
|
def build_message(self, contact, announce_context=None, name=None):
|
|
"""
|
|
Build the email as a multipart message containing
|
|
a multipart alternative for text (plain, HTML) plus
|
|
all the attached files.
|
|
"""
|
|
|
|
content_html = self.build_email_content(contact, announce_context, name)
|
|
|
|
h = HTMLParser.HTMLParser()
|
|
content_html = h.unescape(content_html)
|
|
content_text = html2text(content_html)
|
|
|
|
content_html = encodestring(smart_str(content_html))
|
|
content_text = encodestring(smart_str(content_text))
|
|
|
|
|
|
message = MIMEMultipart()
|
|
|
|
message['Subject'] = smart_str(self.build_title_content(contact))
|
|
message['From'] = smart_str(self.newsletter.header_sender)
|
|
message['Reply-to'] = smart_str(self.newsletter.header_reply)
|
|
message['To'] = smart_str(contact.mail_format())
|
|
|
|
message_alt = MIMEMultipart('alternative')
|
|
text = MIMEText(content_text, 'plain', 'UTF-8')
|
|
text.replace_header('Content-Transfer-Encoding', 'quoted-printable')
|
|
|
|
message_alt.attach(text)
|
|
html = MIMEText(content_html, 'html', 'UTF-8')
|
|
html.replace_header('Content-Transfer-Encoding', 'quoted-printable')
|
|
#encode_base64(text)
|
|
message_alt.attach(html)
|
|
message.attach(message_alt)
|
|
|
|
for attachment in self.attachments:
|
|
if self.newsletter.dailymail and hasattr(attachment, 'cid') and not attachment.cid in content_html:
|
|
continue
|
|
message.attach(attachment)
|
|
|
|
if self.announce and announce_context:
|
|
# add announce attachments
|
|
announce_attachments = self.build_daily_ctx_attachments(announce_context)
|
|
for attachment in announce_attachments:
|
|
message.attach(attachment)
|
|
|
|
for header, value in self.newsletter.server.custom_headers.items():
|
|
message[header] = value
|
|
|
|
uidb36, token = tokenize(contact)
|
|
unsubscribe_link = 'http://' + Site.objects.get_current().domain + reverse('newsletter_mailinglist_unsubscribe_hard', args=[self.newsletter.slug, uidb36, token])
|
|
|
|
message['List-Unsubscribe'] = '<' + unsubscribe_link + '>'
|
|
message['List-Id'] = str(self.newsletter.id)
|
|
return message
|
|
|
|
def build_attachments(self):
|
|
"""Build email's attachment messages"""
|
|
attachments = []
|
|
|
|
for attachment in self.newsletter.attachment_set.all():
|
|
ctype, encoding = mimetypes.guess_type(attachment.file_attachment.path)
|
|
|
|
if ctype is None or encoding is not None:
|
|
ctype = 'application/octet-stream'
|
|
|
|
maintype, subtype = ctype.split('/', 1)
|
|
|
|
fd = open(attachment.file_attachment.path, 'rb')
|
|
if maintype == 'text':
|
|
message_attachment = MIMEText(fd.read(), _subtype=subtype)
|
|
|
|
elif maintype == 'message':
|
|
message_attachment = message_from_file(fd)
|
|
elif maintype == 'image':
|
|
message_attachment = MIMEImage(fd.read(), _subtype=subtype)
|
|
elif maintype == 'audio':
|
|
message_attachment = MIMEAudio(fd.read(), _subtype=subtype)
|
|
else:
|
|
message_attachment = MIMEBase(maintype, subtype)
|
|
message_attachment.set_payload(fd.read())
|
|
encode_base64(message_attachment)
|
|
fd.close()
|
|
message_attachment.add_header('Content-ID', '<'+attachment.title+'>')
|
|
#message_attachment.add_header('Content-Disposition', 'attachment',
|
|
# filename=attachment.title)
|
|
attachments.append(message_attachment)
|
|
|
|
return attachments
|
|
|
|
def build_daily_ctx_attachments(self, context):
|
|
attachments = []
|
|
for section in ['recommended', 'news', 'blog']:
|
|
resize = context_attachments_size.get(section)
|
|
_obj = context.get(section)
|
|
if _obj:
|
|
if isinstance(_obj, SearchResult):
|
|
_obj = _obj.object
|
|
msg_attachment = self.gen_attachment_logo(_obj, prefix=section, resize=resize)
|
|
if msg_attachment:
|
|
attachments.append(msg_attachment)
|
|
for section in ['moscow', 'russia', 'foreign']:
|
|
objects = context.get(section)
|
|
if objects:
|
|
resize = context_attachments_size.get(section)
|
|
for event in objects:
|
|
prefix = '{}_{}_'.format(section, event.object.event_type)
|
|
msg_attachment = self.gen_attachment_logo(event.object, prefix=prefix, resize=resize, dailymail=section)
|
|
if msg_attachment:
|
|
attachments.append(msg_attachment)
|
|
return attachments
|
|
|
|
def build_daily_attachments(self):
|
|
attachments = []
|
|
for cid, path in dailymail_attahcments.iteritems():
|
|
atchm = self.gen_attachment_by_path(path, cid)
|
|
atchm.cid = 'cid:' + cid
|
|
attachments.append(atchm)
|
|
if self.newsletter.banner:
|
|
attachments.append(self.gen_attachment_logo(logo=self.newsletter.banner, prefix='inside_image', resize='600x129'))
|
|
return attachments
|
|
|
|
def build_announce_attachments(self, context):
|
|
# todo: move hardcoded prefixes to setting (uses in templates)
|
|
conf = context.get('conf', [])
|
|
expo = context.get('expo', [])
|
|
news = context.get('news')
|
|
blogs = context.get('blogs')
|
|
attachments = []
|
|
for event in chain(conf, expo):
|
|
message_attachment = self.gen_attachment_logo(event, prefix='mail_expo_logo_')
|
|
if message_attachment:
|
|
attachments.append(message_attachment)
|
|
|
|
if news:
|
|
for item in news:
|
|
message_attachment = self.gen_attachment_logo(item, prefix='mail_news_logo_')
|
|
if message_attachment:
|
|
attachments.append(message_attachment)
|
|
|
|
if blogs:
|
|
for item in blogs:
|
|
message_attachment = self.gen_attachment_logo(item, prefix='mail_blogs_logo_')
|
|
if message_attachment:
|
|
attachments.append(message_attachment)
|
|
|
|
return attachments
|
|
|
|
def gen_attachment_by_path(self, path, cid):
|
|
try:
|
|
ctype, encoding = mimetypes.guess_type(path)
|
|
except SuspiciousOperation as e:
|
|
return None
|
|
if ctype is None or encoding is not None:
|
|
ctype = 'application/octet-stream'
|
|
maintype, subtype = ctype.split('/', 1)
|
|
try:
|
|
fd = open(os.path.join(settings.MEDIA_ROOT, path), 'rb')
|
|
if maintype == 'image':
|
|
message_attachment = MIMEImage(fd.read(), _subtype=subtype)
|
|
else:
|
|
message_attachment = MIMEBase(maintype, subtype)
|
|
message_attachment.set_payload(fd.read())
|
|
encode_base64(message_attachment)
|
|
except IOError as e:
|
|
return None
|
|
else:
|
|
fd.close()
|
|
message_attachment.add_header('Content-ID', '<{}>'.format(cid))
|
|
return message_attachment
|
|
|
|
def gen_attachment_logo(self, obj=None, prefix='logo_', resize=None, logo=None, **kwargs):
|
|
logo_path = None
|
|
if obj is not None:
|
|
if prefix == 'blog':
|
|
preview = obj.get_blog_preview2()
|
|
if preview and preview.file_path:
|
|
logo_path = preview.file_path.path
|
|
else:
|
|
logo = getattr(obj, 'logo')
|
|
if not logo and not logo_path:
|
|
logo_path = default_storage.path('newsletter/images/no-logo.png')
|
|
elif logo and not logo_path:
|
|
logo_path = logo.path
|
|
print(logo, logo_path, obj)
|
|
|
|
try:
|
|
ctype, encoding = mimetypes.guess_type(logo_path)
|
|
except SuspiciousOperation:
|
|
return None
|
|
if ctype is None or encoding is not None:
|
|
ctype = 'application/octet-stream'
|
|
|
|
maintype, subtype = ctype.split('/', 1)
|
|
try:
|
|
if resize is not None:
|
|
fd = get_thumbnail(logo_path, resize, **kwargs)
|
|
# import pdb; pdb.set_trace()
|
|
# fd = open(thumb.path, 'rb')
|
|
else:
|
|
fd = open(logo_path, 'rb')
|
|
except IOError:
|
|
return None
|
|
if maintype == 'image':
|
|
message_attachment = MIMEImage(fd.read(), _subtype=subtype)
|
|
else:
|
|
message_attachment = MIMEBase(maintype, subtype)
|
|
message_attachment.set_payload(fd.read())
|
|
encode_base64(message_attachment)
|
|
|
|
if not isinstance(fd, ImageFile):
|
|
fd.close()
|
|
|
|
cid = prefix
|
|
if obj is not None:
|
|
cid += '%d'%obj.id
|
|
|
|
message_attachment.add_header('Content-ID', '<%s>'%cid)
|
|
|
|
return message_attachment
|
|
|
|
def build_title_content(self, contact):
|
|
"""Generate the email title for a contact"""
|
|
# context = Context({'contact': contact,
|
|
# 'UNIQUE_KEY': ''.join(sample(UNIQUE_KEY_CHAR_SET,
|
|
# UNIQUE_KEY_LENGTH))})
|
|
if not self.newsletter.ab_testing or (self.newsletter.ab_testing and self.ab_state == Newsletter.A):
|
|
title = self.newsletter.title.format(**self.preheader_ctx)
|
|
elif self.newsletter.ab_testing and self.ab_state == Newsletter.B:
|
|
title = self.newsletter.title2.format(**self.preheader_ctx)
|
|
return title
|
|
|
|
def build_email_content(self, contact, announce_context=None, name=None):
|
|
"""Generate the mail for a contact"""
|
|
uidb36, token = tokenize(contact)
|
|
context = Context({'contact': contact,
|
|
'domain': Site.objects.get_current().domain,
|
|
'newsletter': self.newsletter,
|
|
'tracking_image_format': TRACKING_IMAGE_FORMAT,
|
|
'uidb36': uidb36, 'token': token,
|
|
'name': name or contact.first_name or contact.last_name or _(u'Подписчик'),
|
|
'settings_links': self.settings_links,
|
|
})
|
|
if self.announce:
|
|
# render template by default announce template
|
|
# template = get_template('newsletter/announce_template.html')
|
|
template = get_template('newsletter/AutomaticEmail_v2.html')
|
|
context.update(announce_context)
|
|
content = template.render(context)
|
|
elif self.newsletter.ab_testing == True:
|
|
if self.ab_state == Newsletter.A:
|
|
content = self.newsletter_template.render(context)
|
|
else:
|
|
content = self.newsletter_template2.render(context)
|
|
else:
|
|
content = self.newsletter_template.render(context)
|
|
|
|
if TRACKING_LINKS:
|
|
content = track_links(content, context)
|
|
# uncomment if wanna include this link
|
|
#link_site = render_to_string('newsletter/newsletter_link_site.html', context)
|
|
#content = body_insertion(content, link_site)
|
|
|
|
|
|
if INCLUDE_UNSUBSCRIPTION and not self.announce:
|
|
unsubscription = render_to_string('newsletter/newsletter_link_unsubscribe.html', context)
|
|
content = body_insertion(content, unsubscription, end=True)
|
|
|
|
if TRACKING_IMAGE:
|
|
image_tracking = render_to_string('newsletter/newsletter_image_tracking.html', context)
|
|
content = body_insertion(content, image_tracking, end=True)
|
|
|
|
# include preheader
|
|
preheader = None
|
|
if self.newsletter.dailymail:
|
|
if not self.newsletter.preheader:
|
|
preheader = contact.build_dailymail_preheader(self.preheader_ctx)
|
|
else:
|
|
preheader = self.newsletter.preheader.format(**self.preheader_ctx)
|
|
elif self.newsletter.preheader:
|
|
if not self.newsletter.ab_testing or self.ab_state == Newsletter.A:
|
|
preheader = self.newsletter.preheader.format(**self.preheader_ctx)
|
|
else:
|
|
preheader = self.newsletter.preheader2.format(**self.preheader_ctx)
|
|
if preheader is not None:
|
|
preheader_html = render_to_string('newsletter/newsletter_preheader.html', {'preheader': preheader})
|
|
content = body_insertion(content, preheader_html)
|
|
|
|
return smart_unicode(content)
|
|
|
|
def build_preheader_ctx(self, contact, name=None):
|
|
t_add = u''
|
|
count = contact.themes.count()
|
|
if count > 3:
|
|
count -= 3
|
|
theme_word = morph.parse(u'тема')[0]
|
|
t_add = _(u' и еще {count} {theme_word}').format(
|
|
count=count,
|
|
theme_word=theme_word.make_agree_with_number(count).word,
|
|
)
|
|
self.preheader_ctx = {
|
|
'name': name or contact.first_name or contact.last_name or _(u'Подписчик'),
|
|
'themes': u', '.join([self.themes.get(x) for x in contact.themes.all().values_list('pk', flat=True)[:3]]) + t_add,
|
|
}
|
|
|
|
def update_newsletter_status(self):
|
|
"""Update the status of the newsletter"""
|
|
if self.test:
|
|
return
|
|
|
|
if self.newsletter.status == Newsletter.WAITING:
|
|
self.newsletter.status = Newsletter.SENDING
|
|
if self.newsletter.status == Newsletter.SENDING and not self.expedition_list:
|
|
self.newsletter.status = Newsletter.SENT
|
|
if self.newsletter.ab_testing:
|
|
self.newsletter.ab_final_stage = True
|
|
|
|
# if self.announce and not self.expedition_list:
|
|
# self.newsletter.status = Newsletter.SENT
|
|
|
|
self.newsletter.save()
|
|
|
|
@property
|
|
def can_send(self):
|
|
"""Check if the newsletter can be sent"""
|
|
if self.test:
|
|
return True
|
|
if self.newsletter.sending_date <= datetime.now() and \
|
|
(self.newsletter.status == Newsletter.WAITING or \
|
|
self.newsletter.status == Newsletter.SENDING):
|
|
return True
|
|
|
|
return False
|
|
|
|
@property
|
|
def expedition_list(self):
|
|
"""Build the expedition list"""
|
|
if self.test:
|
|
return self.newsletter.test_contacts.all()
|
|
|
|
if self.announce:
|
|
already_sent = ContactMailingStatus.objects\
|
|
.filter(status__in=(ContactMailingStatus.SENT, ContactMailingStatus.ANNOUNCE_NO_DATA),
|
|
newsletter=self.newsletter)\
|
|
.values_list('contact__id', flat=True)
|
|
expedition_list = self.newsletter.mailing_list.expedition_set().exclude(id__in=already_sent)
|
|
return expedition_list
|
|
else:
|
|
qs = self.newsletter.mailing_list.expedition_set()
|
|
|
|
status_params = {'status': ContactMailingStatus.SENT}
|
|
if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
|
|
status_params = {'status__in': [ContactMailingStatus.SENT, ContactMailingStatus.AB_WAITING]}
|
|
|
|
if self.newsletter.theme_for_filter:
|
|
already_sent = \
|
|
ContactMailingStatus.objects.filter(
|
|
Q(newsletter=self.newsletter) | \
|
|
Q(newsletter__mailing_list=self.newsletter.mailing_list,
|
|
creation_date__gte=datetime.now() - timedelta(days=5)),
|
|
**status_params
|
|
).values_list('contact__id', flat=True)
|
|
|
|
qs = qs.filter(contactsettings__theme=self.newsletter.theme_for_filter)
|
|
|
|
else:
|
|
already_sent = \
|
|
ContactMailingStatus.objects.filter(
|
|
newsletter=self.newsletter,
|
|
**status_params
|
|
).values_list('contact__id', flat=True)
|
|
|
|
qs = qs.exclude(id__in=already_sent)
|
|
|
|
return qs
|
|
|
|
def update_contact_status(self, contact, exception, send):
|
|
if not send:
|
|
status = ContactMailingStatus.ANNOUNCE_NO_DATA
|
|
elif exception is None:
|
|
status = (self.test
|
|
and ContactMailingStatus.SENT_TEST
|
|
or ContactMailingStatus.SENT)
|
|
elif isinstance(exception, (UnicodeError, SMTPRecipientsRefused)):
|
|
status = ContactMailingStatus.INVALID
|
|
contact.valid = False
|
|
contact.save()
|
|
else:
|
|
# signal error
|
|
print >>sys.stderr, 'smtp connection raises %s' % exception
|
|
status = ContactMailingStatus.ERROR
|
|
|
|
if self.newsletter.dailymail and status in [ContactMailingStatus.SENT_TEST, ContactMailingStatus.SENT]:
|
|
contact.last_mailing_date = datetime.now().date()
|
|
contact.save()
|
|
params = {
|
|
'newsletter': self.newsletter,
|
|
'contact': contact,
|
|
'status': status,
|
|
}
|
|
if self.newsletter.ab_testing:
|
|
if self.newsletter.ab_final_stage:
|
|
try:
|
|
contact = ContactMailingStatus.objects.get(
|
|
newsletter=self.newsletter, contact=contact,
|
|
status=ContactMailingStatus.AB_WAITING)
|
|
contact.status, contact.ab = status, self.ab_state
|
|
contact.save()
|
|
return
|
|
except ContactMailingStatus.DoesNotExist:
|
|
print('contactmailing status not found - creating...')
|
|
params.update({'ab': self.ab_state})
|
|
ContactMailingStatus.objects.create(**params)
|
|
|
|
def test():
|
|
from django.utils.translation import activate
|
|
activate('ru')
|
|
n = Newsletter.objects.get(pk=214)
|
|
m = Mailer(n, test=True)
|
|
m.smtp_connect()
|
|
m.attachments = m.build_daily_attachments()
|
|
for ct in m.expedition_list:
|
|
m.build_preheader_ctx(ct)
|
|
announce_context = ct.get_announce_context_v2() if m.announce else None
|
|
message = m.build_message(ct, announce_context)
|
|
m.smtp.sendmail(m.newsletter.header_sender,
|
|
ct.email,
|
|
message.as_string())
|
|
|
|
class Mailer(NewsLetterSender):
|
|
"""Mailer for generating and sending newsletters
|
|
In test mode the mailer always send mails but do not log it"""
|
|
smtp = None
|
|
|
|
def send_to_friends(self, contact, contacts):
|
|
if not self.can_send:
|
|
return
|
|
|
|
if not self.smtp and not self.local_dev:
|
|
self.smtp_connect()
|
|
|
|
self.attachments = self.build_daily_attachments()
|
|
|
|
for _contact in contacts:
|
|
send = True
|
|
self.build_preheader_ctx(contact, name=_contact.get('name'))
|
|
announce_context = contact.get_announce_context_v2(self.newsletter.sending_date)
|
|
if not announce_context:
|
|
send = False
|
|
try:
|
|
log.info(u'Trying send to {email}'.format(email=contact.email))
|
|
# pass
|
|
if send:
|
|
message = self.build_message(contact, announce_context, _contact.get('name'))
|
|
if not self.local_dev:
|
|
self.smtp.sendmail(self.newsletter.header_sender,
|
|
_contact.get('email'),
|
|
message.as_string())
|
|
except (Exception,) as e:
|
|
exception = e
|
|
log.info(u'Exception was raised while sending to {email}: {exception}'.format(
|
|
email=contact.email, exception=exception))
|
|
else:
|
|
exception = None
|
|
if not self.local_dev:
|
|
self.smtp.quit()
|
|
|
|
def run(self):
|
|
"""Send the mails"""
|
|
if not self.can_send:
|
|
return
|
|
|
|
if not self.smtp and not self.local_dev:
|
|
self.smtp_connect()
|
|
|
|
if self.newsletter.dailymail:
|
|
self.attachments = self.build_daily_attachments()
|
|
else:
|
|
self.attachments = self.build_attachments()
|
|
|
|
expedition_list = self.expedition_list
|
|
|
|
number_of_recipients = len(expedition_list)
|
|
if self.verbose:
|
|
print '%i emails will be sent' % number_of_recipients
|
|
|
|
i = 1
|
|
for contact in expedition_list:
|
|
send = True
|
|
if self.verbose:
|
|
print '- Processing %s/%s (%s)' % (
|
|
i, number_of_recipients, contact.pk)
|
|
|
|
#
|
|
self.build_preheader_ctx(contact)
|
|
# check if events for this newsletter exists
|
|
announce_context = contact.get_announce_context_v2() if self.announce else None
|
|
if self.announce and not announce_context:
|
|
send = False
|
|
try:
|
|
log.info(u'Trying send to {email}'.format(email=contact.email))
|
|
# pass
|
|
if send:
|
|
message = self.build_message(contact, announce_context)
|
|
if not self.local_dev:
|
|
self.smtp.sendmail(self.newsletter.header_sender,
|
|
contact.email,
|
|
message.as_string())
|
|
# print(message)
|
|
except (Exception, ) as e:
|
|
exception = e
|
|
print(exception)
|
|
log.info(u'Exception was raised while sending to {email}: {exception}'.format(
|
|
email=contact.email, exception=exception))
|
|
else:
|
|
exception = None
|
|
|
|
# обновляем статус
|
|
self.update_contact_status(contact, exception, send)
|
|
self.update_ab_state()
|
|
# удаляем контакт из маркировки на второй этап
|
|
if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
|
|
try:
|
|
self.second_stage_expedition_list_ids.remove(contact.pk)
|
|
except ValueError:
|
|
pass
|
|
|
|
if SLEEP_BETWEEN_SENDING:
|
|
time.sleep(SLEEP_BETWEEN_SENDING)
|
|
if RESTART_CONNECTION_BETWEEN_SENDING:
|
|
self.smtp.quit()
|
|
self.smtp_connect()
|
|
|
|
i += 1
|
|
|
|
# маркируем оставшиеся контакты на второй этап
|
|
self.mark_contacts_for_second_stage()
|
|
|
|
if not self.local_dev:
|
|
self.smtp.quit()
|
|
self.update_newsletter_status()
|
|
|
|
def smtp_connect(self):
|
|
"""Make a connection to the SMTP"""
|
|
self.smtp = self.newsletter.server.connect()
|
|
|
|
@property
|
|
def expedition_list(self):
|
|
"""Build the expedition list"""
|
|
self.second_stage_expedition_list_ids = []
|
|
|
|
self.credits = self.newsletter.server.credits()
|
|
if self.credits <= 0:
|
|
return []
|
|
|
|
if self.newsletter.dailymail and not self.test:
|
|
return self.newsletter.get_dailymail_subscribers()
|
|
|
|
qs = super(Mailer, self).expedition_list
|
|
|
|
qs = qs[:self.credits]
|
|
|
|
if self.test:
|
|
return qs
|
|
|
|
if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
|
|
# сохраняем себе все айд контактов с текущей итерации,
|
|
# после отправки писем исключим все отправленные и поставим
|
|
# в ожидание второго этапа
|
|
self.second_stage_expedition_list_ids = list(qs.values_list('pk', flat=True))
|
|
print('всего контактов %s' % len(self.second_stage_expedition_list_ids))
|
|
# берем заданый процент от всех контактов в текущей итерации
|
|
_slice = int(round(float(len(self.second_stage_expedition_list_ids))/100 * (self.newsletter.ab_first_stage or 5)))
|
|
qs = qs[:_slice]
|
|
return qs
|
|
|
|
def mark_contacts_for_second_stage(self):
|
|
if self.second_stage_expedition_list_ids:
|
|
func = lambda pk: ContactMailingStatus(
|
|
newsletter=self.newsletter, contact_id=pk,
|
|
status=ContactMailingStatus.AB_WAITING)
|
|
contact_ms = list(map(func, self.second_stage_expedition_list_ids))
|
|
ContactMailingStatus.objects.bulk_create(contact_ms)
|
|
print('Marked for second stage %s.' % len(self.second_stage_expedition_list_ids))
|
|
|
|
def update_ab_state(self):
|
|
if not self.newsletter.ab_testing or self.newsletter.ab_final_stage: return
|
|
self.ab_state = Newsletter.A if self.ab_state == Newsletter.B else Newsletter.B
|
|
|
|
@property
|
|
def can_send(self):
|
|
"""Check if the newsletter can be sent"""
|
|
if self.newsletter.server.credits() <= 0:
|
|
return False
|
|
return super(Mailer, self).can_send
|
|
|
|
|
|
class SMTPMailer(object):
|
|
"""for generating and sending newsletters
|
|
|
|
SMTPMailer takes the problem on a different basis than Mailer, it use
|
|
a SMTP server and make a roundrobin over all newsletters to be sent
|
|
dispatching it's send command to smtp server regularly over time to
|
|
reach the limit.
|
|
|
|
It is more robust in term of predictability.
|
|
|
|
In test mode the mailer always send mails but do not log it"""
|
|
|
|
smtp = None
|
|
|
|
def __init__(self, server, test=False, verbose=0):
|
|
self.start = datetime.now()
|
|
self.server = server
|
|
self.test = test
|
|
self.verbose = verbose
|
|
self.stop_event = threading.Event()
|
|
|
|
def run(self):
|
|
"""send mails
|
|
"""
|
|
sending = dict()
|
|
candidates = self.get_candidates()
|
|
roundrobin = []
|
|
|
|
if not self.smtp:
|
|
self.smtp_connect()
|
|
|
|
delay = self.server.delay()
|
|
|
|
i = 1
|
|
sleep_time = 0
|
|
while (not self.stop_event.wait(sleep_time) and
|
|
not self.stop_event.is_set()):
|
|
if not roundrobin:
|
|
# refresh the list
|
|
for expedition in candidates:
|
|
if expedition.id not in sending and expedition.can_send:
|
|
sending[expedition.id] = expedition()
|
|
|
|
roundrobin = list(sending.keys())
|
|
|
|
if roundrobin:
|
|
nl_id = roundrobin.pop()
|
|
nl = sending[nl_id]
|
|
|
|
try:
|
|
self.smtp.sendmail(*nl.next())
|
|
except StopIteration:
|
|
del sending[nl_id]
|
|
except Exception, e:
|
|
nl.throw(e)
|
|
else:
|
|
nl.next()
|
|
|
|
sleep_time = (delay * i -
|
|
total_seconds(datetime.now() - self.start))
|
|
if SLEEP_BETWEEN_SENDING:
|
|
sleep_time = max(time.sleep(SLEEP_BETWEEN_SENDING), sleep_time)
|
|
if RESTART_CONNECTION_BETWEEN_SENDING:
|
|
self.smtp.quit()
|
|
self.smtp_connect()
|
|
i += 1
|
|
else:
|
|
# no work, sleep a bit and some reset
|
|
sleep_time = 600
|
|
i = 1
|
|
self.start = datetime.now()
|
|
|
|
if sleep_time < 0:
|
|
sleep_time = 0
|
|
|
|
self.smtp.quit()
|
|
|
|
def get_candidates(self):
|
|
"""get candidates NL"""
|
|
return [NewsLetterExpedition(nl, self)
|
|
for nl in Newsletter.objects.filter(server=self.server)]
|
|
|
|
def smtp_connect(self):
|
|
"""Make a connection to the SMTP"""
|
|
self.smtp = self.server.connect()
|
|
|
|
|
|
class NewsLetterExpedition(NewsLetterSender):
|
|
"""coroutine that will give messages to be sent with mailer
|
|
|
|
between to message it alternate with None so that
|
|
the mailer give it a chance to save status to db
|
|
"""
|
|
|
|
def __init__(self, newsletter, mailer):
|
|
super(NewsLetterExpedition, self).__init__(
|
|
newsletter, test=mailer.test, verbose=mailer.verbose)
|
|
self.mailer = mailer
|
|
self.id = newsletter.id
|
|
|
|
def __call__(self):
|
|
"""iterator on messages to be sent
|
|
"""
|
|
newsletter = self.newsletter
|
|
|
|
title = 'smtp-%s (%s), nl-%s (%s)' % (
|
|
self.mailer.server.id, self.mailer.server.name[:10],
|
|
newsletter.id, newsletter.title[:10])
|
|
# ajust len
|
|
title = '%-30s' % title
|
|
|
|
self.attachments = self.build_attachments()
|
|
|
|
expedition_list = self.expedition_list
|
|
|
|
number_of_recipients = len(expedition_list)
|
|
if self.verbose:
|
|
print '%s %s: %i emails will be sent' % (
|
|
datetime.now().strftime('%Y-%m-%d'),
|
|
title, number_of_recipients)
|
|
|
|
try:
|
|
i = 1
|
|
for contact in expedition_list:
|
|
if self.verbose:
|
|
print '%s %s: processing %s/%s (%s)' % (
|
|
datetime.now().strftime('%H:%M:%S'),
|
|
title, i, number_of_recipients, contact.pk)
|
|
|
|
try:
|
|
message = self.build_message(contact)
|
|
yield (smart_str(self.newsletter.header_sender),
|
|
contact.email,
|
|
message.as_string())
|
|
except Exception, e:
|
|
exception = e
|
|
else:
|
|
exception = None
|
|
|
|
self.update_contact_status(contact, exception)
|
|
i += 1
|
|
# this one permits us to save to database imediately
|
|
# and acknoledge eventual exceptions
|
|
yield None
|
|
finally:
|
|
self.update_newsletter_status()
|
|
|