# -*- coding: utf-8 -*-
"""Mailer for emencia.django.newsletter"""
import os
import re
import sys
import time
import threading
import mimetypes
import base64
import quopri
from random import sample
from StringIO import StringIO
from datetime import datetime
from datetime import timedelta
from smtplib import SMTPRecipientsRefused
from pymorphy2 import MorphAnalyzer
morph = MorphAnalyzer()
from logging import getLogger
log = getLogger('mail')
from itertools import chain

try:
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    from email.mime.Encoders import encode_base64
    from email.mime.MIMEAudio import MIMEAudio
    from email.mime.MIMEBase import MIMEBase
    from email.mime.MIMEImage import MIMEImage
except ImportError:  # Python 2.4 compatibility
    from email.MIMEMultipart import MIMEMultipart
    from email.MIMEText import MIMEText
    from email.Encoders import encode_base64
    from email.MIMEAudio import MIMEAudio
    from email.MIMEBase import MIMEBase
    from email.MIMEImage import MIMEImage
from email import message_from_file
from html2text import html2text as html2text_orig
from django.contrib.sites.models import Site
from django.template import Context, Template
from django.template.loader import render_to_string, get_template
from django.utils.encoding import smart_str
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext as _
from django.core.files.storage import default_storage
from django.core.urlresolvers import reverse
from django.core.exceptions import SuspiciousOperation
from django.db.models import Q
from django.contrib.staticfiles import finders
from django.conf import settings

from sorl.thumbnail import get_thumbnail
from sorl.thumbnail.images import ImageFile
from haystack.models import SearchResult

from emencia.django.newsletter.models import Newsletter
from emencia.django.newsletter.models import ContactMailingStatus
from emencia.django.newsletter.models import Contact
from emencia.django.newsletter.utils.tokens import tokenize
from emencia.django.newsletter.utils.newsletter import track_links
from emencia.django.newsletter.utils.newsletter import body_insertion
from emencia.django.newsletter.settings import TRACKING_LINKS
from emencia.django.newsletter.settings import TRACKING_IMAGE
from emencia.django.newsletter.settings import TRACKING_IMAGE_FORMAT
from emencia.django.newsletter.settings import UNIQUE_KEY_LENGTH
from emencia.django.newsletter.settings import UNIQUE_KEY_CHAR_SET
from emencia.django.newsletter.settings import INCLUDE_UNSUBSCRIPTION
from emencia.django.newsletter.settings import SLEEP_BETWEEN_SENDING
from emencia.django.newsletter.settings import \
     RESTART_CONNECTION_BETWEEN_SENDING

import HTMLParser
import chardet

from theme.models import Theme


if not hasattr(timedelta, 'total_seconds'):
    def total_seconds(td):
        return ((td.microseconds +
                 (td.seconds + td.days * 24 * 3600) * 1000000) /
                1000000.0)
else:
    total_seconds = lambda td: td.total_seconds()


LINK_RE = re.compile(r"https?://([^ \n]+\n)+[^ \n]+", re.MULTILINE)
cid_rx = re.compile(r'^<(?P<id>.*)>$')


def html2text(html):
    """Use html2text but repair newlines cutting urls.
    Need to use this hack until
    https://github.com/aaronsw/html2text/issues/#issue/7 is not fixed"""
    txt = html2text_orig(html)
    return txt


def encodestring(instring, tabs=0):
    outfile = StringIO()
    quopri.encode(StringIO(instring), outfile, tabs)
    return outfile.getvalue()

def decodestring(instring):
    outfile = StringIO()
    quopri.decode(StringIO(instring), outfile)
    return outfile.getvalue()


dailymail_attahcments = {
    'logo1': 'newsletter/images/logo1.png',
    'logo2': 'newsletter/images/logo2.png',
    'marker': 'newsletter/images/marker.png',
    'calendar': 'newsletter/images/calendar.png',
    'm1': 'newsletter/images/m1.png',
    'm2': 'newsletter/images/m2.png',
    'm3': 'newsletter/images/m3.png',
    'm4': 'newsletter/images/m4.png',
    'expo': 'newsletter/images/expo.png',
    # 'news2': 'newsletter/images/news2.jpg',
    # 'b': 'newsletter/images/b.png',
    'arrow': 'newsletter/images/arrow.png',
    'white_arrow': 'newsletter/images/white_arrow.png',
    'site_logo': 'newsletter/images/site_logo.png',
    'instagram': 'newsletter/images/instagram.png',
    'youtube': 'newsletter/images/youtube.png',
    'facebook': 'newsletter/images/facebook.png',
    'linkedin': 'newsletter/images/linkedin.png',
    'vk': 'newsletter/images/vk.png',
    'twitter': 'newsletter/images/twitter.png',
}
context_attachments_size = {
    'recommended': '170x170',
    'news': '170x170',
    'blog': '200x170',
    'moscow': '109x114',
    'russia': '109x114',
    'foreign': '109x114',
}


class NewsLetterSender(object):

    def __init__(self, newsletter, test=False, verbose=0):
        self.test = test
        self.verbose = verbose
        self.newsletter = newsletter
        self.settings_links = True
        self.newsletter_template = Template(self.newsletter.content)
        self.themes = dict(Theme.objects.language('ru').all().values_list('pk', 'name'))


        self.newsletter_template2 = None
        self.ab_state = Newsletter.A
        if newsletter.ab_testing == True:
            self.newsletter_template2 = Template(self.newsletter.content2)
            if self.newsletter.ab_final_stage:
                self.ab_state = self.newsletter.ab_final_choice

        #
        self.announce = self.newsletter.dailymail
        self.local_dev = getattr(settings, 'LOCAL_DEV', False)

    def build_message(self, contact, announce_context=None, name=None):
        """
        Build the email as a multipart message containing
        a multipart alternative for text (plain, HTML) plus
        all the attached files.
        """

        content_html = self.build_email_content(contact, announce_context, name)

        h = HTMLParser.HTMLParser()
        content_html = h.unescape(content_html)
        content_text = html2text(content_html)

        content_html = encodestring(smart_str(content_html))
        content_text = encodestring(smart_str(content_text))


        message = MIMEMultipart()

        message['Subject'] = smart_str(self.build_title_content(contact))
        message['From'] = smart_str(self.newsletter.header_sender)
        message['Reply-to'] = smart_str(self.newsletter.header_reply)
        message['To'] = smart_str(contact.mail_format())

        message_alt = MIMEMultipart('alternative')
        text = MIMEText(content_text, 'plain', 'UTF-8')
        text.replace_header('Content-Transfer-Encoding', 'quoted-printable')

        message_alt.attach(text)
        html = MIMEText(content_html, 'html', 'UTF-8')
        html.replace_header('Content-Transfer-Encoding', 'quoted-printable')
        #encode_base64(text)
        message_alt.attach(html)
        message.attach(message_alt)

        for attachment in self.attachments:
            if self.newsletter.dailymail and hasattr(attachment, 'cid') and not attachment.cid in content_html:
                continue
            message.attach(attachment)

        if self.announce and announce_context:
            # add announce attachments
            announce_attachments = self.build_daily_ctx_attachments(announce_context)
            for attachment in announce_attachments:
                message.attach(attachment)

        for header, value in self.newsletter.server.custom_headers.items():
            message[header] = value

        uidb36, token = tokenize(contact)
        unsubscribe_link = 'http://' + Site.objects.get_current().domain + reverse('newsletter_mailinglist_unsubscribe_hard', args=[self.newsletter.slug, uidb36, token])

        message['List-Unsubscribe'] = '<' + unsubscribe_link + '>'
        message['List-Id'] = str(self.newsletter.id)
        return message

    def build_attachments(self):
        """Build email's attachment messages"""
        attachments = []

        for attachment in self.newsletter.attachment_set.all():
            ctype, encoding = mimetypes.guess_type(attachment.file_attachment.path)

            if ctype is None or encoding is not None:
                ctype = 'application/octet-stream'

            maintype, subtype = ctype.split('/', 1)

            fd = open(attachment.file_attachment.path, 'rb')
            if maintype == 'text':
                message_attachment = MIMEText(fd.read(), _subtype=subtype)

            elif maintype == 'message':
                message_attachment = message_from_file(fd)
            elif maintype == 'image':
                message_attachment = MIMEImage(fd.read(), _subtype=subtype)
            elif maintype == 'audio':
                message_attachment = MIMEAudio(fd.read(), _subtype=subtype)
            else:
                message_attachment = MIMEBase(maintype, subtype)
                message_attachment.set_payload(fd.read())
                encode_base64(message_attachment)
            fd.close()
            message_attachment.add_header('Content-ID', '<'+attachment.title+'>')
            #message_attachment.add_header('Content-Disposition', 'attachment',
            #                              filename=attachment.title)
            attachments.append(message_attachment)

        return attachments

    def build_daily_ctx_attachments(self, context):
        attachments = []
        for section in ['recommended', 'news', 'blog']:
            resize = context_attachments_size.get(section)
            _obj = context.get(section)
            if _obj:
                if isinstance(_obj, SearchResult):
                    _obj = _obj.object
                msg_attachment = self.gen_attachment_logo(_obj, prefix=section, resize=resize)
                if msg_attachment:
                    attachments.append(msg_attachment)
        for section in ['moscow', 'russia', 'foreign']:
            objects = context.get(section)
            if objects:
                resize = context_attachments_size.get(section)
                for event in objects:
                    prefix = '{}_{}_'.format(section, event.object.event_type)
                    msg_attachment = self.gen_attachment_logo(event.object, prefix=prefix, resize=resize, dailymail=section)
                    if msg_attachment:
                        attachments.append(msg_attachment)
        return attachments

    def build_daily_attachments(self):
        attachments = []
        for cid, path in dailymail_attahcments.iteritems():
            atchm = self.gen_attachment_by_path(path, cid)
            atchm.cid = 'cid:' + cid
            attachments.append(atchm)
        if self.newsletter.banner:
            attachments.append(self.gen_attachment_logo(logo=self.newsletter.banner, prefix='inside_image', resize='600x129'))
        return attachments

    def build_announce_attachments(self, context):
        # todo: move hardcoded prefixes to setting (uses in templates)
        conf = context.get('conf', [])
        expo = context.get('expo', [])
        news = context.get('news')
        blogs = context.get('blogs')
        attachments = []
        for event in chain(conf, expo):
            message_attachment = self.gen_attachment_logo(event, prefix='mail_expo_logo_')
            if message_attachment:
                attachments.append(message_attachment)

        if news:
            for item in news:
                message_attachment = self.gen_attachment_logo(item, prefix='mail_news_logo_')
                if message_attachment:
                    attachments.append(message_attachment)

        if blogs:
            for item in blogs:
                message_attachment = self.gen_attachment_logo(item, prefix='mail_blogs_logo_')
                if message_attachment:
                    attachments.append(message_attachment)

        return attachments

    def gen_attachment_by_path(self, path, cid):
        try:
            ctype, encoding = mimetypes.guess_type(path)
        except SuspiciousOperation as e:
            return None
        if ctype is None or encoding is not None:
            ctype = 'application/octet-stream'
        maintype, subtype = ctype.split('/', 1)
        try:
            fd = open(os.path.join(settings.MEDIA_ROOT, path), 'rb')
            if maintype == 'image':
                message_attachment = MIMEImage(fd.read(), _subtype=subtype)
            else:
                message_attachment = MIMEBase(maintype, subtype)
                message_attachment.set_payload(fd.read())
                encode_base64(message_attachment)
        except IOError as e:
            return None
        else:
            fd.close()
        message_attachment.add_header('Content-ID', '<{}>'.format(cid))
        return message_attachment

    def gen_attachment_logo(self, obj=None, prefix='logo_', resize=None, logo=None, **kwargs):
        logo_path = None
        if obj is not None:
            if prefix == 'blog':
                preview = obj.get_blog_preview2()
                if preview and preview.file_path:
                    logo_path = preview.file_path.path
            else:
                logo = getattr(obj, 'logo')
        if not logo and not logo_path:
            logo_path = default_storage.path('newsletter/images/no-logo.png')
        elif logo and not logo_path:
            logo_path = logo.path
        print(logo, logo_path, obj)

        try:
            ctype, encoding = mimetypes.guess_type(logo_path)
        except SuspiciousOperation:
            return None
        if ctype is None or encoding is not None:
            ctype = 'application/octet-stream'

        maintype, subtype = ctype.split('/', 1)
        try:
            if resize is not None:
                fd = get_thumbnail(logo_path, resize, **kwargs)
                # import pdb; pdb.set_trace()
                # fd = open(thumb.path, 'rb')
            else:
                fd = open(logo_path, 'rb')
        except IOError:
            return None
        if maintype == 'image':
            message_attachment = MIMEImage(fd.read(), _subtype=subtype)
        else:
            message_attachment = MIMEBase(maintype, subtype)
            message_attachment.set_payload(fd.read())
            encode_base64(message_attachment)

        if not isinstance(fd, ImageFile):
            fd.close()

        cid = prefix
        if obj is not None:
            cid += '%d'%obj.id

        message_attachment.add_header('Content-ID', '<%s>'%cid)

        return message_attachment

    def build_title_content(self, contact):
        """Generate the email title for a contact"""
        # context = Context({'contact': contact,
        #                    'UNIQUE_KEY': ''.join(sample(UNIQUE_KEY_CHAR_SET,
        #                                                 UNIQUE_KEY_LENGTH))})
        if not self.newsletter.ab_testing or (self.newsletter.ab_testing and self.ab_state == Newsletter.A):
            title = self.newsletter.title.format(**self.preheader_ctx)
        elif self.newsletter.ab_testing and self.ab_state == Newsletter.B:
            title = self.newsletter.title2.format(**self.preheader_ctx)
        return title

    def build_email_content(self, contact, announce_context=None, name=None):
        """Generate the mail for a contact"""
        uidb36, token = tokenize(contact)
        context = Context({'contact': contact,
                           'domain': Site.objects.get_current().domain,
                           'newsletter': self.newsletter,
                           'tracking_image_format': TRACKING_IMAGE_FORMAT,
                           'uidb36': uidb36, 'token': token,
                           'name': name or contact.first_name or contact.last_name or _(u'Подписчик'),
                           'settings_links': self.settings_links,
                           })
        if self.announce:
            # render template by default announce template
            # template = get_template('newsletter/announce_template.html')
            template = get_template('newsletter/AutomaticEmail_v2.html')
            context.update(announce_context)
            content = template.render(context)
        elif self.newsletter.ab_testing == True:
            if self.ab_state == Newsletter.A:
                content = self.newsletter_template.render(context)
            else:
                content = self.newsletter_template2.render(context)
        else:
            content = self.newsletter_template.render(context)

        if TRACKING_LINKS:
            content = track_links(content, context)
        # uncomment if wanna include this link
        #link_site = render_to_string('newsletter/newsletter_link_site.html', context)
        #content = body_insertion(content, link_site)


        if INCLUDE_UNSUBSCRIPTION and not self.announce:
            unsubscription = render_to_string('newsletter/newsletter_link_unsubscribe.html', context)
            content = body_insertion(content, unsubscription, end=True)

        if TRACKING_IMAGE:
            image_tracking = render_to_string('newsletter/newsletter_image_tracking.html', context)
            content = body_insertion(content, image_tracking, end=True)

        # include preheader
        preheader = None
        if self.newsletter.dailymail:
            if not self.newsletter.preheader:
                preheader = contact.build_dailymail_preheader(self.preheader_ctx)
            else:
                preheader = self.newsletter.preheader.format(**self.preheader_ctx)
        elif self.newsletter.preheader:
            if not self.newsletter.ab_testing or self.ab_state == Newsletter.A:
                preheader = self.newsletter.preheader.format(**self.preheader_ctx)
            else:
                preheader = self.newsletter.preheader2.format(**self.preheader_ctx)
        if preheader is not None:
            preheader_html = render_to_string('newsletter/newsletter_preheader.html', {'preheader': preheader})
            content = body_insertion(content, preheader_html)

        return smart_unicode(content)

    def build_preheader_ctx(self, contact, name=None):
        t_add = u''
        count = contact.themes.count()
        if count > 3:
            count -= 3
            theme_word = morph.parse(u'тема')[0]
            t_add = _(u' и еще {count} {theme_word}').format(
                count=count,
                theme_word=theme_word.make_agree_with_number(count).word,
                )
        self.preheader_ctx = {
            'name': name or contact.first_name or contact.last_name or _(u'Подписчик'),
            'themes': u', '.join([self.themes.get(x) for x in contact.themes.all().values_list('pk', flat=True)[:3]]) + t_add,
        }

    def update_newsletter_status(self):
        """Update the status of the newsletter"""
        if self.test:
            return

        if self.newsletter.status == Newsletter.WAITING:
            self.newsletter.status = Newsletter.SENDING
        if self.newsletter.status == Newsletter.SENDING and not self.expedition_list:
            self.newsletter.status = Newsletter.SENT
            if self.newsletter.ab_testing:
                self.newsletter.ab_final_stage = True

        # if self.announce and not self.expedition_list:
        #     self.newsletter.status = Newsletter.SENT

        self.newsletter.save()

    @property
    def can_send(self):
        """Check if the newsletter can be sent"""
        if self.test:
            return True
        if self.newsletter.sending_date <= datetime.now() and \
               (self.newsletter.status == Newsletter.WAITING or \
                self.newsletter.status == Newsletter.SENDING):
            return True

        return False

    @property
    def expedition_list(self):
        """Build the expedition list"""
        if self.test:
            return self.newsletter.test_contacts.all()

        if self.announce:
            already_sent = ContactMailingStatus.objects\
                .filter(status__in=(ContactMailingStatus.SENT, ContactMailingStatus.ANNOUNCE_NO_DATA),
                        newsletter=self.newsletter)\
                .values_list('contact__id', flat=True)
            expedition_list = self.newsletter.mailing_list.expedition_set().exclude(id__in=already_sent)
            return expedition_list
        else:
            qs = self.newsletter.mailing_list.expedition_set()

            status_params = {'status': ContactMailingStatus.SENT}
            if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
                status_params = {'status__in': [ContactMailingStatus.SENT, ContactMailingStatus.AB_WAITING]}

            if self.newsletter.theme_for_filter:
                already_sent = \
                    ContactMailingStatus.objects.filter(
                        Q(newsletter=self.newsletter) | \
                        Q(newsletter__mailing_list=self.newsletter.mailing_list,
                          creation_date__gte=datetime.now() - timedelta(days=5)),
                        **status_params
                    ).values_list('contact__id', flat=True)

                qs = qs.filter(contactsettings__theme=self.newsletter.theme_for_filter)

            else:
                already_sent = \
                    ContactMailingStatus.objects.filter(
                        newsletter=self.newsletter,
                        **status_params
                    ).values_list('contact__id', flat=True)

            qs = qs.exclude(id__in=already_sent)

            return qs

    def update_contact_status(self, contact, exception, send):
        if not send:
            status = ContactMailingStatus.ANNOUNCE_NO_DATA
        elif exception is None:
            status = (self.test
                      and ContactMailingStatus.SENT_TEST
                      or ContactMailingStatus.SENT)
        elif isinstance(exception, (UnicodeError, SMTPRecipientsRefused)):
            status = ContactMailingStatus.INVALID
            contact.valid = False
            contact.save()
        else:
            # signal error
            print >>sys.stderr, 'smtp connection raises %s' % exception
            status = ContactMailingStatus.ERROR

        if self.newsletter.dailymail and status in [ContactMailingStatus.SENT_TEST, ContactMailingStatus.SENT]:
            contact.last_mailing_date = datetime.now().date()
            contact.save()
        params = {
            'newsletter': self.newsletter,
            'contact': contact,
            'status': status,
            }
        if self.newsletter.ab_testing:
            if self.newsletter.ab_final_stage:
                try:
                    contact = ContactMailingStatus.objects.get(
                        newsletter=self.newsletter, contact=contact,
                        status=ContactMailingStatus.AB_WAITING)
                    contact.status, contact.ab = status, self.ab_state
                    contact.save()
                    return
                except ContactMailingStatus.DoesNotExist:
                    print('contactmailing status not found - creating...')
            params.update({'ab': self.ab_state})
        ContactMailingStatus.objects.create(**params)

def test():
    from django.utils.translation import activate
    activate('ru')
    n = Newsletter.objects.get(pk=214)
    m = Mailer(n, test=True)
    m.smtp_connect()
    m.attachments = m.build_daily_attachments()
    for ct in m.expedition_list:
        m.build_preheader_ctx(ct)
        announce_context = ct.get_announce_context_v2() if m.announce else None
        message = m.build_message(ct, announce_context)
        m.smtp.sendmail(m.newsletter.header_sender,
                           ct.email,
                           message.as_string())

class Mailer(NewsLetterSender):
    """Mailer for generating and sending newsletters
    In test mode the mailer always send mails but do not log it"""
    smtp = None

    def send_to_friends(self, contact, contacts):
        if not self.can_send:
            return

        if not self.smtp and not self.local_dev:
            self.smtp_connect()

        self.attachments = self.build_daily_attachments()

        for _contact in contacts:
            send = True
            self.build_preheader_ctx(contact, name=_contact.get('name'))
            announce_context = contact.get_announce_context_v2(self.newsletter.sending_date)
            if not announce_context:
                send = False
            try:
                log.info(u'Trying send to {email}'.format(email=contact.email))
                # pass
                if send:
                    message = self.build_message(contact, announce_context, _contact.get('name'))
                    if not self.local_dev:
                        self.smtp.sendmail(self.newsletter.header_sender,
                                           _contact.get('email'),
                                           message.as_string())
            except (Exception,) as e:
                exception = e
                log.info(u'Exception was raised while sending to {email}: {exception}'.format(
                    email=contact.email, exception=exception))
            else:
                exception = None
        if not self.local_dev:
            self.smtp.quit()

    def run(self):
        """Send the mails"""
        if not self.can_send:
            return

        if not self.smtp and not self.local_dev:
            self.smtp_connect()

        if self.newsletter.dailymail:
            self.attachments = self.build_daily_attachments()
        else:
            self.attachments = self.build_attachments()

        expedition_list = self.expedition_list

        number_of_recipients = len(expedition_list)
        if self.verbose:
            print '%i emails will be sent' % number_of_recipients

        i = 1
        for contact in expedition_list:
            send = True
            if self.verbose:
                print '- Processing %s/%s (%s)' % (
                    i, number_of_recipients, contact.pk)

            #
            self.build_preheader_ctx(contact)
            # check if events for this newsletter exists
            announce_context = contact.get_announce_context_v2() if self.announce else None
            if self.announce and not announce_context:
                send = False
            try:
                log.info(u'Trying send to {email}'.format(email=contact.email))
                # pass
                if send:
                    message = self.build_message(contact, announce_context)
                    if not self.local_dev:
                        self.smtp.sendmail(self.newsletter.header_sender,
                                           contact.email,
                                           message.as_string())
                    # print(message)
            except (Exception, ) as e:
                exception = e
                print(exception)
                log.info(u'Exception was raised while sending to {email}: {exception}'.format(
                    email=contact.email, exception=exception))
            else:
                exception = None

            # обновляем статус
            self.update_contact_status(contact, exception, send)
            self.update_ab_state()
            # удаляем контакт из маркировки на второй этап
            if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
                try:
                    self.second_stage_expedition_list_ids.remove(contact.pk)
                except ValueError:
                    pass

            if SLEEP_BETWEEN_SENDING:
                time.sleep(SLEEP_BETWEEN_SENDING)
            if RESTART_CONNECTION_BETWEEN_SENDING:
                self.smtp.quit()
                self.smtp_connect()

            i += 1

        # маркируем оставшиеся контакты на второй этап
        self.mark_contacts_for_second_stage()

        if not self.local_dev:
            self.smtp.quit()
        self.update_newsletter_status()

    def smtp_connect(self):
        """Make a connection to the SMTP"""
        self.smtp = self.newsletter.server.connect()

    @property
    def expedition_list(self):
        """Build the expedition list"""
        self.second_stage_expedition_list_ids = []

        self.credits = self.newsletter.server.credits()
        if self.credits <= 0:
            return []

        if self.newsletter.dailymail and not self.test:
            return self.newsletter.get_dailymail_subscribers()

        qs = super(Mailer, self).expedition_list

        qs = qs[:self.credits]

        if self.test:
            return qs

        if self.newsletter.ab_testing and not self.newsletter.ab_final_stage:
            # сохраняем себе все айд контактов с текущей итерации,
            # после отправки писем исключим все отправленные и поставим
            # в ожидание второго этапа
            self.second_stage_expedition_list_ids = list(qs.values_list('pk', flat=True))
            print('всего контактов %s' % len(self.second_stage_expedition_list_ids))
            # берем заданый процент от всех контактов в текущей итерации
            _slice = int(round(float(len(self.second_stage_expedition_list_ids))/100 * (self.newsletter.ab_first_stage or 5)))
            qs = qs[:_slice]
        return qs

    def mark_contacts_for_second_stage(self):
        if self.second_stage_expedition_list_ids:
            func = lambda pk: ContactMailingStatus(
                newsletter=self.newsletter, contact_id=pk,
                status=ContactMailingStatus.AB_WAITING)
            contact_ms = list(map(func, self.second_stage_expedition_list_ids))
            ContactMailingStatus.objects.bulk_create(contact_ms)
            print('Marked for second stage %s.' % len(self.second_stage_expedition_list_ids))

    def update_ab_state(self):
        if not self.newsletter.ab_testing or self.newsletter.ab_final_stage: return
        self.ab_state = Newsletter.A if self.ab_state == Newsletter.B else Newsletter.B

    @property
    def can_send(self):
        """Check if the newsletter can be sent"""
        if self.newsletter.server.credits() <= 0:
            return False
        return super(Mailer, self).can_send


class SMTPMailer(object):
    """for generating and sending newsletters

    SMTPMailer takes the problem on a different basis than Mailer, it use
    a SMTP server and make a roundrobin over all newsletters to be sent
    dispatching it's send command to smtp server regularly over time to
    reach the limit.

    It is more robust in term of predictability.

    In test mode the mailer always send mails but do not log it"""

    smtp = None

    def __init__(self, server, test=False, verbose=0):
        self.start = datetime.now()
        self.server = server
        self.test = test
        self.verbose = verbose
        self.stop_event = threading.Event()

    def run(self):
        """send mails
        """
        sending = dict()
        candidates = self.get_candidates()
        roundrobin = []

        if not self.smtp:
            self.smtp_connect()

        delay = self.server.delay()

        i = 1
        sleep_time = 0
        while (not self.stop_event.wait(sleep_time) and
               not self.stop_event.is_set()):
            if not roundrobin:
                # refresh the list
                for expedition in candidates:
                    if expedition.id not in sending and expedition.can_send:
                        sending[expedition.id] = expedition()

                roundrobin = list(sending.keys())

            if roundrobin:
                nl_id = roundrobin.pop()
                nl = sending[nl_id]

                try:
                    self.smtp.sendmail(*nl.next())
                except StopIteration:
                    del sending[nl_id]
                except Exception, e:
                    nl.throw(e)
                else:
                    nl.next()

                sleep_time = (delay * i -
                              total_seconds(datetime.now() - self.start))
                if SLEEP_BETWEEN_SENDING:
                    sleep_time = max(time.sleep(SLEEP_BETWEEN_SENDING), sleep_time)
                if RESTART_CONNECTION_BETWEEN_SENDING:
                    self.smtp.quit()
                    self.smtp_connect()
                i += 1
            else:
                # no work, sleep a bit and some reset
                sleep_time = 600
                i = 1
                self.start = datetime.now()

            if sleep_time < 0:
                sleep_time = 0

        self.smtp.quit()

    def get_candidates(self):
        """get candidates NL"""
        return [NewsLetterExpedition(nl, self)
                for nl in Newsletter.objects.filter(server=self.server)]

    def smtp_connect(self):
        """Make a connection to the SMTP"""
        self.smtp = self.server.connect()


class NewsLetterExpedition(NewsLetterSender):
    """coroutine that will give messages to be sent with mailer

    between to message it alternate with None so that
    the mailer give it a chance to save status to db
    """

    def __init__(self, newsletter, mailer):
        super(NewsLetterExpedition, self).__init__(
                        newsletter, test=mailer.test, verbose=mailer.verbose)
        self.mailer = mailer
        self.id = newsletter.id

    def __call__(self):
        """iterator on messages to be sent
        """
        newsletter = self.newsletter

        title = 'smtp-%s (%s), nl-%s (%s)' % (
                        self.mailer.server.id, self.mailer.server.name[:10],
                        newsletter.id, newsletter.title[:10])
        # ajust len
        title = '%-30s' % title

        self.attachments = self.build_attachments()

        expedition_list = self.expedition_list

        number_of_recipients = len(expedition_list)
        if self.verbose:
            print '%s %s: %i emails will be sent' % (
                    datetime.now().strftime('%Y-%m-%d'),
                    title, number_of_recipients)

        try:
            i = 1
            for contact in expedition_list:
                if self.verbose:
                    print '%s %s: processing %s/%s (%s)' % (
                        datetime.now().strftime('%H:%M:%S'),
                        title, i, number_of_recipients, contact.pk)

                try:
                    message = self.build_message(contact)
                    yield (smart_str(self.newsletter.header_sender),
                                       contact.email,
                                       message.as_string())
                except Exception, e:
                    exception = e
                else:
                    exception = None

                self.update_contact_status(contact, exception)
                i += 1
                # this one permits us to save to database imediately
                # and acknoledge eventual exceptions
                yield None
        finally:
            self.update_newsletter_status()