# -*- coding: utf-8 -*- """Mailer for emencia.django.newsletter""" import re import sys import time import threading import mimetypes import base64 import quopri from random import sample from StringIO import StringIO from datetime import datetime from datetime import timedelta from smtplib import SMTPRecipientsRefused from pymorphy2 import MorphAnalyzer morph = MorphAnalyzer() from logging import getLogger log = getLogger('mail') try: from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.Encoders import encode_base64 from email.mime.MIMEAudio import MIMEAudio from email.mime.MIMEBase import MIMEBase from email.mime.MIMEImage import MIMEImage except ImportError: # Python 2.4 compatibility from email.MIMEMultipart import MIMEMultipart from email.MIMEText import MIMEText from email.Encoders import encode_base64 from email.MIMEAudio import MIMEAudio from email.MIMEBase import MIMEBase from email.MIMEImage import MIMEImage from email import message_from_file from html2text import html2text as html2text_orig from django.contrib.sites.models import Site from django.template import Context, Template from django.template.loader import render_to_string, get_template from django.utils.encoding import smart_str from django.utils.encoding import smart_unicode from django.utils.translation import ugettext as _ from django.core.urlresolvers import reverse from django.core.exceptions import SuspiciousOperation from django.db.models import Q from emencia.django.newsletter.models import Newsletter from emencia.django.newsletter.models import ContactMailingStatus from emencia.django.newsletter.utils.tokens import tokenize from emencia.django.newsletter.utils.newsletter import track_links from emencia.django.newsletter.utils.newsletter import body_insertion from emencia.django.newsletter.settings import TRACKING_LINKS from emencia.django.newsletter.settings import TRACKING_IMAGE from emencia.django.newsletter.settings import TRACKING_IMAGE_FORMAT from emencia.django.newsletter.settings import UNIQUE_KEY_LENGTH from emencia.django.newsletter.settings import UNIQUE_KEY_CHAR_SET from emencia.django.newsletter.settings import INCLUDE_UNSUBSCRIPTION from emencia.django.newsletter.settings import SLEEP_BETWEEN_SENDING from emencia.django.newsletter.settings import \ RESTART_CONNECTION_BETWEEN_SENDING import HTMLParser import chardet from theme.models import Theme if not hasattr(timedelta, 'total_seconds'): def total_seconds(td): return ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 1000000) / 1000000.0) else: total_seconds = lambda td: td.total_seconds() LINK_RE = re.compile(r"https?://([^ \n]+\n)+[^ \n]+", re.MULTILINE) def html2text(html): """Use html2text but repair newlines cutting urls. Need to use this hack until https://github.com/aaronsw/html2text/issues/#issue/7 is not fixed""" txt = html2text_orig(html) return txt def encodestring(instring, tabs=0): outfile = StringIO() quopri.encode(StringIO(instring), outfile, tabs) return outfile.getvalue() def decodestring(instring): outfile = StringIO() quopri.decode(StringIO(instring), outfile) return outfile.getvalue() class NewsLetterSender(object): def __init__(self, newsletter, test=False, verbose=0): self.test = test self.verbose = verbose self.newsletter = newsletter self.newsletter_template = Template(self.newsletter.content) self.themes = dict(Theme.objects.language('ru').all().values_list('pk', 'name')) self.newsletter_template2 = None self.title_template2 = None self.ab_state = Newsletter.A if newsletter.ab_testing == True: self.newsletter_template2 = Template(self.newsletter.content2) self.title_template2 = Template(self.newsletter.title2) if self.newsletter.ab_final_stage: self.ab_state = self.newsletter.ab_final_choice self.title_template = Template(self.newsletter.title) # self.announce = self.newsletter.mailing_list.announce def build_message(self, contact, announce_context=None): """ Build the email as a multipart message containing a multipart alternative for text (plain, HTML) plus all the attached files. """ content_html = self.build_email_content(contact, announce_context) h = HTMLParser.HTMLParser() content_html = h.unescape(content_html) content_text = html2text(content_html) content_html = encodestring(smart_str(content_html)) content_text = encodestring(smart_str(content_text)) message = MIMEMultipart() message['Subject'] = smart_str(self.build_title_content(contact)) message['From'] = smart_str(self.newsletter.header_sender) message['Reply-to'] = smart_str(self.newsletter.header_reply) message['To'] = smart_str(contact.mail_format()) message_alt = MIMEMultipart('alternative') text = MIMEText(content_text, 'plain', 'UTF-8') text.replace_header('Content-Transfer-Encoding', 'quoted-printable') message_alt.attach(text) html = MIMEText(content_html, 'html', 'UTF-8') html.replace_header('Content-Transfer-Encoding', 'quoted-printable') #encode_base64(text) message_alt.attach(html) message.attach(message_alt) for attachment in self.attachments: message.attach(attachment) if announce_context: # add announce attachments announce_attachments = self.build_announce_attachments(announce_context) for attachment in announce_attachments: message.attach(attachment) for header, value in self.newsletter.server.custom_headers.items(): message[header] = value uidb36, token = tokenize(contact) unsubscribe_link = 'http://' + Site.objects.get_current().domain + reverse('newsletter_mailinglist_unsubscribe_hard', args=[self.newsletter.slug, uidb36, token]) message['List-Unsubscribe'] = '<' + unsubscribe_link + '>' message['List-Id'] = str(self.newsletter.id) return message def build_attachments(self): """Build email's attachment messages""" attachments = [] for attachment in self.newsletter.attachment_set.all(): ctype, encoding = mimetypes.guess_type(attachment.file_attachment.path) if ctype is None or encoding is not None: ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) fd = open(attachment.file_attachment.path, 'rb') if maintype == 'text': message_attachment = MIMEText(fd.read(), _subtype=subtype) elif maintype == 'message': message_attachment = message_from_file(fd) elif maintype == 'image': message_attachment = MIMEImage(fd.read(), _subtype=subtype) elif maintype == 'audio': message_attachment = MIMEAudio(fd.read(), _subtype=subtype) else: message_attachment = MIMEBase(maintype, subtype) message_attachment.set_payload(fd.read()) encode_base64(message_attachment) fd.close() message_attachment.add_header('Content-ID', '<'+attachment.title+'>') #message_attachment.add_header('Content-Disposition', 'attachment', # filename=attachment.title) attachments.append(message_attachment) return attachments def build_announce_attachments(self, context): # todo: move hardcoded prefixes to setting (uses in templates) events = context['events'] news = context.get('news') blogs = context.get('blogs') attachments = [] for event in events: message_attachment = self.gen_attachment_logo(event, prefix='mail_expo_logo_') if message_attachment: attachments.append(message_attachment) if news: for item in news: message_attachment = self.gen_attachment_logo(item, prefix='mail_news_logo_') if message_attachment: attachments.append(message_attachment) if blogs: for item in blogs: message_attachment = self.gen_attachment_logo(item, prefix='mail_blogs_logo_') if message_attachment: attachments.append(message_attachment) return attachments def gen_attachment_logo(self, obj, prefix='logo_'): logo = getattr(obj, 'logo') if not logo: return None try: ctype, encoding = mimetypes.guess_type(logo.path) except SuspiciousOperation: return None if ctype is None or encoding is not None: ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) try: fd = open(logo.path, 'rb') except IOError: return None if maintype == 'image': message_attachment = MIMEImage(fd.read(), _subtype=subtype) else: message_attachment = MIMEBase(maintype, subtype) message_attachment.set_payload(fd.read()) encode_base64(message_attachment) fd.close() cid = prefix + '%d'%obj.id message_attachment.add_header('Content-ID', '<%s>'%cid) return message_attachment def build_title_content(self, contact): """Generate the email title for a contact""" context = Context({'contact': contact, 'UNIQUE_KEY': ''.join(sample(UNIQUE_KEY_CHAR_SET, UNIQUE_KEY_LENGTH))}) if self.newsletter.ab_testing and self.ab_state == Newsletter.A: title = self.title_template.render(context) elif self.newsletter.ab_testing and self.ab_state == Newsletter.B: title = self.title_template2.render(context) else: title = self.title_template.render(context) return title def build_email_content(self, contact, announce_context=None): """Generate the mail for a contact""" uidb36, token = tokenize(contact) context = Context({'contact': contact, 'domain': Site.objects.get_current().domain, 'newsletter': self.newsletter, 'tracking_image_format': TRACKING_IMAGE_FORMAT, 'uidb36': uidb36, 'token': token}) if self.announce: # render template by default announce template template = get_template('newsletter/announce_template.html') context.update(announce_context) content = template.render(context) elif self.newsletter.ab_testing == True: if self.ab_state == Newsletter.A: content = self.newsletter_template.render(context) else: content = self.newsletter_template2.render(context) else: content = self.newsletter_template.render(context) if TRACKING_LINKS: content = track_links(content, context) # uncomment if wanna include this link #link_site = render_to_string('newsletter/newsletter_link_site.html', context) #content = body_insertion(content, link_site) if INCLUDE_UNSUBSCRIPTION: unsubscription = render_to_string('newsletter/newsletter_link_unsubscribe.html', context) content = body_insertion(content, unsubscription, end=True) if TRACKING_IMAGE: image_tracking = render_to_string('newsletter/newsletter_image_tracking.html', context) content = body_insertion(content, image_tracking, end=True) # include preheader if self.newsletter.preheader: preheader = self.newsletter.preheader.format(**self.build_preheader_ctx(contact)) preheader_html = render_to_string('newsletter/newsletter_preheader.html', {'preheader': preheader}) content = body_insertion(content, preheader_html) return smart_unicode(content) def build_preheader_ctx(self, contact): t_add = '' count = contact.contactsettings.theme.count() if count > 3: count -= 3 theme_word = morph.parse(u'тема')[0] t_add = _(u' и еще {count} {theme_word}').format( count=count, theme_word=theme_word.make_agree_with_number(count).word, ) return { 'name': contact.first_name or contact.last_name, 'themes': ', '.join([self.themes.get(x) for x in contact.contactsettings.theme.all().values_list('pk', flat=True)[:3]]) + t_add, } def update_newsletter_status(self): """Update the status of the newsletter""" if self.test: return if self.newsletter.status == Newsletter.WAITING: self.newsletter.status = Newsletter.SENDING if self.newsletter.status == Newsletter.SENDING and \ self.newsletter.mails_sent() >= \ self.newsletter.mailing_list.expedition_set().count(): self.newsletter.status = Newsletter.SENT if self.newsletter.ab_testing: self.newsletter.ab_final_stage = True if self.announce and not self.expedition_list: self.newsletter.status = Newsletter.SENT self.newsletter.save() @property def can_send(self): """Check if the newsletter can be sent""" if self.test: return True if self.newsletter.sending_date <= datetime.now() and \ (self.newsletter.status == Newsletter.WAITING or \ self.newsletter.status == Newsletter.SENDING): return True return False @property def expedition_list(self): """Build the expedition list""" if self.test: return self.newsletter.test_contacts.all() if self.announce: already_sent = ContactMailingStatus.objects\ .filter(status__in=(ContactMailingStatus.SENT, ContactMailingStatus.ANNOUNCE_NO_DATA), newsletter=self.newsletter)\ .values_list('contact__id', flat=True) expedition_list = self.newsletter.mailing_list.expedition_set().exclude(id__in=already_sent) return expedition_list else: qs = self.newsletter.mailing_list.expedition_set() status_params = {'status': ContactMailingStatus.SENT} if self.newsletter.ab_testing and not self.newsletter.ab_final_stage: status_params = {'status__in': [ContactMailingStatus.SENT, ContactMailingStatus.AB_WAITING]} if self.newsletter.theme_for_filter: already_sent = \ ContactMailingStatus.objects.filter( Q(newsletter=self.newsletter) | \ Q(newsletter__mailing_list=self.newsletter.mailing_list, creation_date__gte=datetime.now() - timedelta(days=2)), **status_params ).values_list('contact__id', flat=True) qs = qs.filter(contactsettings__theme=self.newsletter.theme_for_filter) else: already_sent = \ ContactMailingStatus.objects.filter( newsletter=self.newsletter, **status_params ).values_list('contact__id', flat=True) qs = qs.exclude(id__in=already_sent) return qs def update_contact_status(self, contact, exception, send): if not send: status = ContactMailingStatus.ANNOUNCE_NO_DATA elif exception is None: status = (self.test and ContactMailingStatus.SENT_TEST or ContactMailingStatus.SENT) elif isinstance(exception, (UnicodeError, SMTPRecipientsRefused)): status = ContactMailingStatus.INVALID contact.valid = False contact.save() else: # signal error print >>sys.stderr, 'smtp connection raises %s' % exception status = ContactMailingStatus.ERROR params = { 'newsletter': self.newsletter, 'contact': contact, 'status': status, } if self.newsletter.ab_testing: if self.newsletter.ab_final_stage: try: contact = ContactMailingStatus.objects.get( newsletter=self.newsletter, contact=contact, status=ContactMailingStatus.AB_WAITING) contact.status, contact.ab = status, self.ab_state contact.save() return except ContactMailingStatus.DoesNotExist: print('contactmailing status not found - creating...') params.update({'ab': self.ab_state}) ContactMailingStatus.objects.create(**params) class Mailer(NewsLetterSender): """Mailer for generating and sending newsletters In test mode the mailer always send mails but do not log it""" smtp = None def run(self): """Send the mails""" if not self.can_send: return if not self.smtp: self.smtp_connect() self.attachments = self.build_attachments() expedition_list = self.expedition_list number_of_recipients = len(expedition_list) if self.verbose: print '%i emails will be sent' % number_of_recipients i = 1 for contact in expedition_list: send = True if self.verbose: print '- Processing %s/%s (%s)' % ( i, number_of_recipients, contact.pk) # check if events for this newsletter exists announce_context = contact.get_announce_context() if self.announce else None if self.announce and not announce_context: send = False try: log.info('Trying send to {email}'.format(email=contact.email)) # pass if send: message = self.build_message(contact, announce_context) self.smtp.sendmail(self.newsletter.header_sender, contact.email, message.as_string()) except (Exception, ) as e: exception = e log.info('Exception was raised while sending to {email}: {exception}'.format( email=contact.email, exception=exception)) else: exception = None self.update_ab_state() # обновляем статус self.update_contact_status(contact, exception, send) # удаляем контакт из маркировки на второй этап if self.newsletter.ab_testing and not self.newsletter.ab_final_stage: try: self.second_stage_expedition_list_ids.remove(contact.pk) except ValueError: pass if SLEEP_BETWEEN_SENDING: time.sleep(SLEEP_BETWEEN_SENDING) if RESTART_CONNECTION_BETWEEN_SENDING: self.smtp.quit() self.smtp_connect() i += 1 # маркируем оставшиеся контакты на второй этап self.mark_contacts_for_second_stage() self.smtp.quit() self.update_newsletter_status() def smtp_connect(self): """Make a connection to the SMTP""" self.smtp = self.newsletter.server.connect() @property def expedition_list(self): """Build the expedition list""" self.credits = self.newsletter.server.credits() if self.credits <= 0: return [] qs = super(Mailer, self).expedition_list qs = qs[:self.credits] self.second_stage_expedition_list_ids = [] if self.test: return qs if self.newsletter.ab_testing and not self.newsletter.ab_final_stage: # сохраняем себе все айд контактов с текущей итерации, # после отправки писем исключим все отправленные и поставим # в ожидание второго этапа self.second_stage_expedition_list_ids = list(qs.values_list('pk', flat=True)) print('всего контактов %s' % len(self.second_stage_expedition_list_ids)) # берем заданый процент от всех контактов в текущей итерации _slice = int(float(len(self.second_stage_expedition_list_ids))/100 * (self.newsletter.ab_first_stage or 5)) qs = qs[:_slice] return qs def mark_contacts_for_second_stage(self): if self.second_stage_expedition_list_ids: func = lambda pk: ContactMailingStatus( newsletter=self.newsletter, contact_id=pk, status=ContactMailingStatus.AB_WAITING) contact_ms = list(map(func, self.second_stage_expedition_list_ids)) ContactMailingStatus.objects.bulk_create(contact_ms) print('Marked for second stage %s.' % len(self.second_stage_expedition_list_ids)) def update_ab_state(self): if not self.newsletter.ab_testing or self.newsletter.ab_final_stage: return self.ab_state = Newsletter.A if self.ab_state == Newsletter.B else Newsletter.B @property def can_send(self): """Check if the newsletter can be sent""" if self.newsletter.server.credits() <= 0: return False return super(Mailer, self).can_send class SMTPMailer(object): """for generating and sending newsletters SMTPMailer takes the problem on a different basis than Mailer, it use a SMTP server and make a roundrobin over all newsletters to be sent dispatching it's send command to smtp server regularly over time to reach the limit. It is more robust in term of predictability. In test mode the mailer always send mails but do not log it""" smtp = None def __init__(self, server, test=False, verbose=0): self.start = datetime.now() self.server = server self.test = test self.verbose = verbose self.stop_event = threading.Event() def run(self): """send mails """ sending = dict() candidates = self.get_candidates() roundrobin = [] if not self.smtp: self.smtp_connect() delay = self.server.delay() i = 1 sleep_time = 0 while (not self.stop_event.wait(sleep_time) and not self.stop_event.is_set()): if not roundrobin: # refresh the list for expedition in candidates: if expedition.id not in sending and expedition.can_send: sending[expedition.id] = expedition() roundrobin = list(sending.keys()) if roundrobin: nl_id = roundrobin.pop() nl = sending[nl_id] try: self.smtp.sendmail(*nl.next()) except StopIteration: del sending[nl_id] except Exception, e: nl.throw(e) else: nl.next() sleep_time = (delay * i - total_seconds(datetime.now() - self.start)) if SLEEP_BETWEEN_SENDING: sleep_time = max(time.sleep(SLEEP_BETWEEN_SENDING), sleep_time) if RESTART_CONNECTION_BETWEEN_SENDING: self.smtp.quit() self.smtp_connect() i += 1 else: # no work, sleep a bit and some reset sleep_time = 600 i = 1 self.start = datetime.now() if sleep_time < 0: sleep_time = 0 self.smtp.quit() def get_candidates(self): """get candidates NL""" return [NewsLetterExpedition(nl, self) for nl in Newsletter.objects.filter(server=self.server)] def smtp_connect(self): """Make a connection to the SMTP""" self.smtp = self.server.connect() class NewsLetterExpedition(NewsLetterSender): """coroutine that will give messages to be sent with mailer between to message it alternate with None so that the mailer give it a chance to save status to db """ def __init__(self, newsletter, mailer): super(NewsLetterExpedition, self).__init__( newsletter, test=mailer.test, verbose=mailer.verbose) self.mailer = mailer self.id = newsletter.id def __call__(self): """iterator on messages to be sent """ newsletter = self.newsletter title = 'smtp-%s (%s), nl-%s (%s)' % ( self.mailer.server.id, self.mailer.server.name[:10], newsletter.id, newsletter.title[:10]) # ajust len title = '%-30s' % title self.attachments = self.build_attachments() expedition_list = self.expedition_list number_of_recipients = len(expedition_list) if self.verbose: print '%s %s: %i emails will be sent' % ( datetime.now().strftime('%Y-%m-%d'), title, number_of_recipients) try: i = 1 for contact in expedition_list: if self.verbose: print '%s %s: processing %s/%s (%s)' % ( datetime.now().strftime('%H:%M:%S'), title, i, number_of_recipients, contact.pk) try: message = self.build_message(contact) yield (smart_str(self.newsletter.header_sender), contact.email, message.as_string()) except Exception, e: exception = e else: exception = None self.update_contact_status(contact, exception) i += 1 # this one permits us to save to database imediately # and acknoledge eventual exceptions yield None finally: self.update_newsletter_status()