# -*- coding: utf-8 -*-
import re
from itertools import chain
from collections import namedtuple
from datetime import datetime
from datetime import timedelta
from datetime import date
from datetime import MAXYEAR, MINYEAR
from dateutil import relativedelta
import calendar
# from calendar import TimeEncoding, month_name


try:
    from collections import ChainMap
except ImportError:
    from chainmap import ChainMap

from django import forms
from django.utils.translation import get_language, ugettext as _
from django.utils.translation import string_concat
from django.utils.encoding import smart_text, force_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.db.models import Count, Sum, Q, ForeignKey, ManyToManyField
from django.db import connection
from django.core.exceptions import ValidationError
from django.core.cache import cache
from django.conf import settings

from haystack.query import SearchQuerySet, RelatedSearchQuerySet, SQ

from functions.model_utils import EnumChoices
# from functions.model_mixin import get_dates
from exposition.models import Exposition
from conference.models import Conference
from theme.models import Theme, Tag
from country.models import Country
from city.models import City
from events.common import MEMBERS, VISITORS, PRICE, TYPES
from events.common import members_mapping, visitors_mapping, price_mapping
from events.common import ExtraWhere, OR, AND


class FilterCheckboxSelectMultiple(forms.CheckboxSelectMultiple):
    def render(self, name, value, attrs=None, choices=()):
        if value is None: value = []
        has_id = attrs and 'id' in attrs
        final_attrs = self.build_attrs(attrs, name=name)
        output = ['<ul>']
        # Normalize to strings
        str_values = set([force_text(v) for v in value])
        for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
            # If an ID attribute was given, add a numeric index as a suffix,
            # so that the checkboxes don't all have the same ID attribute.
            if has_id:
                final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
                label_for = format_html(u' for="{0}"', final_attrs['id'])
            else:
                label_for = ''

            cb = forms.CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
            option_value = force_text(option_value)
            rendered_cb = cb.render(name, option_value)
            option_label = mark_safe(force_text(option_label))
            output.append(format_html(u'<li>{1}<label{0}><span>{2}</span></label></li>',
                                      label_for, rendered_cb, option_label))
        output.append(u'</ul>')
        return mark_safe('\n'.join(output))


class WidgetDefaultMixin(object):
    def widget_attrs(self, widget):
        attrs = super(WidgetDefaultMixin, self).widget_attrs(widget)
        attrs['class'] = 'default'
        return attrs


class CountModelMultipleChoiceField(WidgetDefaultMixin, forms.ModelMultipleChoiceField):
    widget = FilterCheckboxSelectMultiple
    def label_from_instance(self, obj):
        if obj.get('count', None) is None:
            return smart_text(obj.get('name'))
        return u'{label} <i>({count})</i>'.format(label=smart_text(obj.get('name')), count=obj.get('count'))

    def prepare_value(self, value):
        if isinstance(value, dict):
            return value.get('pk')
        return super(CountModelMultipleChoiceField, self).prepare_value(value)

    def clean(self, value):
        # return pks instead of QuerySet
        if self.required and not value:
            raise ValidationError(self.error_messages['required'])
        elif not self.required and not value:
            return self.queryset.none()
        if not isinstance(value, (list, tuple)):
            raise ValidationError(self.error_messages['list'])
        key = self.to_field_name or 'pk'
        for pk in value:
            try:
                self.queryset.filter(**{key: pk})
            except ValueError:
                raise ValidationError(self.error_messages['invalid_pk_value'] % pk)
        pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True)))
        for val in value:
            if int(force_text(val)) not in pks:
                raise ValidationError(self.error_messages['invalid_choice'] % val)
        # Since this overrides the inherited ModelChoiceField.clean
        # we run custom validators here
        self.run_validators(value)
        return pks


class FilterTypedMultipleChoiceField(WidgetDefaultMixin, forms.TypedMultipleChoiceField):
    pass


class MonthChoiceField(FilterTypedMultipleChoiceField):
    year = None
    additional_choices = []
    default_month = None
    def make_year_choice(self, value):
        return (str(value), _(u'{year} год').format(year=value))

    def valid_value(self, value):
        valid = super(MonthChoiceField, self).valid_value(value)
        if not valid:
            if self.year and (value == self.year or check_year(value) == self.year):
                choice = self.make_year_choice(value)
                self.additional_choices.append(choice)
                self.choices.insert(0, choice)
                return True

            value = str(value)
            year = self.year or datetime.today().year
            match, _year, month = get_month_and_year(value)
            if match:
                year = _year
                self.default_month = month
            elif value in settings.MONTHES.keys():
                month = value
                self.default_month = month
            if month in settings.MONTHES.keys():
                choice = (value, string_concat(settings.MONTHES[month]['name'], ' ', year))
                self.additional_choices.append(choice)
                self.choices.insert(0, choice)
                # print(choice, self.choices)

                return super(MonthChoiceField, self).valid_value(value)
        return valid


fields_mapping = {
    'members': 'members_choice',
    'visitors': 'visitors_choice',
    'price': 'price_choice',
}

values_mapping = {
    'members_choice': members_mapping,
    'visitors_choice': visitors_mapping,
    'price_choice': price_mapping,

}

RATING = (
    (1, _(u'Топовые (HIT)')),
    (2, _(u'Члены РСВЯ')),
    (3, _(u'Члены UFI')),
    (4, _(u'ExpoRating')),
)

monthes_abr_to_num = {v.lower(): k for k, v in enumerate(calendar.month_abbr)}
monthes_num_to_abr = {v: k for k, v in monthes_abr_to_num.iteritems()}
year_month_regex = re.compile(r'^((?P<year>\d{4})(?P<month>\w{3}))|((?P<month_>\d{1,2})/(?P<year_>\d{4}))$')


def get_month_and_year(input_string):
    year = None
    month = None
    match = year_month_regex.match(input_string)
    if match:
        month = match.group('month') or monthes_num_to_abr.get(int(match.group('month_')))
        year = int(match.group('year') or match.group('year_'))
    return bool(match), year, month


def check_year(value):
    year = None
    try:
        year = int(value)
    except (ValueError, ):
        return
    else:
        if not MAXYEAR >= year >= MINYEAR:
            return
    return year


class FilterForm(forms.Form):
    _month_choices = None
    _month = None

    event_type = FilterTypedMultipleChoiceField(
        label=_(u'Тип события'), coerce=int,
        choices=TYPES, required=False, widget=FilterCheckboxSelectMultiple())
    theme = CountModelMultipleChoiceField(
        label=_(u'Тематики'), required=False, cache_choices=True,
        queryset=Theme.objects.language().values('pk', 'name'))
    tag = CountModelMultipleChoiceField(
        label=_(u'Теги'), required=False, cache_choices=True,
        queryset=Tag.objects.language().values('pk', 'name'))
    country = CountModelMultipleChoiceField(
        label=_(u'Страны'), required=False, cache_choices=True,
        queryset=Country.objects.language().values('pk', 'name'))
    city = CountModelMultipleChoiceField(
        label=_(u'Города'), required=False, cache_choices=True,
        queryset=City.objects.language().values('pk', 'name'))
    year = forms.RegexField(r'^(?P<year>\d{4})$', max_length=4, min_length=4,
        required=False, widget=forms.HiddenInput())
    default_month = forms.CharField(max_length=3, min_length=3,
        required=False, widget=forms.HiddenInput())
    default_year = forms.RegexField(r'^(?P<year>\d{4})$', max_length=4, min_length=4,
        required=False, widget=forms.HiddenInput())
    month = MonthChoiceField(
        label=_(u'Период'), coerce=str,
        choices=monthes_abr_to_num.iteritems(),
        required=False, widget=FilterCheckboxSelectMultiple())
    date_from = forms.DateField(required=False, input_formats=('%d.%m.%Y',),
                                widget=forms.DateInput(attrs={'class': 'date', 'id': 'dateFrom',
                                                              'placeholder': _(u'дд.мм.гггг')}))
    date_to = forms.DateField(required=False, input_formats=('%d.%m.%Y',),
                              widget=forms.DateInput(attrs={'class': 'date', 'id': 'dateTo',
                                                            'placeholder': _(u'дд.мм.гггг')}))
    price = FilterTypedMultipleChoiceField(
        label=_(u'Стоимость'), coerce=int,
        choices=PRICE,
        required=False, widget=FilterCheckboxSelectMultiple(),
        # help_text=_(u'За 1 м<sub>2</sub> необорудованной площади')
        )
    members = FilterTypedMultipleChoiceField(
        label=_(u'Участники'), coerce=int,
        choices=MEMBERS,
        required=False, widget=FilterCheckboxSelectMultiple())
    visitors = FilterTypedMultipleChoiceField(
        label=_(u'Посетители'), coerce=int,
        choices=VISITORS,
        required=False, widget=FilterCheckboxSelectMultiple())
    rating = FilterTypedMultipleChoiceField(
        label=_(u'Рейтинги'), coerce=int,
        choices=RATING,
        required=False, widget=FilterCheckboxSelectMultiple())


    def __init__(self, *args, **kwargs):
        super(FilterForm, self).__init__(*args, **kwargs)
        self._is_valid = False
        self._models = None
        self._lookup_kwargs = None
        self._lookup_args = None
        self._related_fields = []
        self._local_fields = []
        self.lang = get_language()
        self.data = self.data.copy()
        self.fill_default_choices_from_cache()
        self.fields['month'].choices = self.month_choices()

    def fill_default_choices_from_cache(self):
        timeout = timedelta(hours=1).seconds
        for field in ['theme', 'tag', 'country', 'city']:
            key ='filter_{field}_{lang}'.format(field=field, lang=self.lang)
            choices = cache.get(key)
            if choices is None:
                choices = list(self.fields[field].choices)
                cache.set(key, choices, timeout)
            else:
                self.fields[field].choice_cache = choices

    def get_dates(self):
        if not (self.cleaned_data.get('date_from') or self.cleaned_data.get('date_to')):
            return _(u'Указать диапазон')
        result = ''
        if self.cleaned_data.get('date_from'):
            result += _(u'c {date_from} ').format(date_from=self.cleaned_data.get('date_from').strftime('%d.%m.%Y'))
        if self.cleaned_data.get('date_to'):
            result += _(u'по {date_to}').format(date_to=self.cleaned_data.get('date_to').strftime('%d.%m.%Y'))
        return result

    @classmethod
    def month_choices(cls):
        month = datetime.today().month
        if cls._month != month or cls._month_choices is None:
            year = datetime.today().year
            depth = 7 # на сколько сколько месяцев вперед делать выборку (включая текущий)
            monthes = dict([(v.get('value'), {'abr': k, 'name': v.get('name')}) for k, v in settings.MONTHES.iteritems()])
            choices = []
            for month_num in xrange(month, month + depth):
                _year = year
                if month_num > 12:
                    _year = year + 1
                    month_num -= 12
                month_values = monthes.get(month_num)
                # month_values['name'] = string_concat(month_values['name'], ' ', str(_year))
                choices.append((str(_year) + month_values['abr'],  string_concat(month_values['name'], ' ', str(_year))))
            cls._month_choices = choices
            cls._month = month
        return cls._month_choices

    def clean_year(self, value=None):
        year = value or self.cleaned_data.get('year')
        year = check_year(year)
        self.fields['month'].year = year
        return year

    def clean_default_month(self):
        v =  self.cleaned_data.get('default_month')
        if v and v in settings.MONTHES.keys():
            self.fields['month'].default_month = v
            return v
        return

    def clean_default_year(self):
        y =  self.cleaned_data.get('default_year')
        y = check_year(y)
        return y

    def clean_month(self):
        values = self.cleaned_data.get('month', [])
        field = self.fields['month']
        new_values = []
        for val in values:
            if len(val) == 3 and list(filter(lambda x: x[0] == str(field.year or datetime.today().year) + val, field.choices)):
                new_values.append(str(field.year or datetime.today().year) + val)
                field.choices = filter(lambda x: x[0] != val, field.choices)
            elif '/' in val and list(filter(lambda x: x[0] == val, field.choices)):
                try:
                    label = list(filter(lambda x: x[0] == val, field.choices))[0][1]
                    match, year, month = get_month_and_year(val)
                    if match:
                        new_val = str(year) + month
                        choices = list(filter(lambda x: x[0] != val, field.choices))
                        if not list(filter(lambda x: x[0] == new_val, field.choices)):
                            choices.insert(0, (new_val, label))
                        field.choices = choices
                        new_values.append(new_val)
                except:
                    continue
            else:
                new_values.append(val)
        if new_values != values:
            self.data.setlist('month', new_values)
        # print(new_values, self.data)
        return new_values

    def _post_clean(self):
        # нужно для того, чтобы год зашел в поле month
        # print(self.get_date_begin_periods())
        if not self.cleaned_data.get('default_month') and self.fields['month'].default_month:
            self.data['default_month'] = self.fields['month'].default_month
        year = self.cleaned_data.get('year')
        # if year:
        #     for val, name in self.fields['month'].additional_choices:
        #         self.
        #         additional_choices

    def is_valid(self):
        # if getattr(self, '_is_valid', None) is None:
        self._is_valid = super(FilterForm, self).is_valid()

        # нам нужно сбрасывать сохраненный выбор моделей,
        # т.к. после валидации нужно вернуть только выбранные
        self._models = None
        self.cleaned_data = getattr(self, 'cleaned_data', {})
        return self._is_valid

    @property
    def models(self):
        if self._models is None and self._is_valid:
            val = self.cleaned_data.get('event_type')
            self._models = []
            if TYPES.EXPO in val:
                self._models.append(Exposition)
            if TYPES.CONF in val:
                self._models.append(Conference)
        return self._models or [Exposition, Conference]

    @property
    def lookup_kwargs(self):
        if self._lookup_kwargs is None:
            d = self.cleaned_data
            self._lookup_kwargs = {}
            self._related_fields = []
            self._local_fields = []
            if d.get('theme'):
                self._related_fields.append('theme')
                self._lookup_kwargs['theme__in'] = d.get('theme')
            if d.get('tag'):
                self._related_fields.append('tag')
                self._lookup_kwargs['tag__in'] = d.get('tag')
            if d.get('country'):
                self._related_fields.append('country')
                self._lookup_kwargs['country_id__in'] = d.get('country')
            if d.get('city'):
                self._related_fields.append('city')
                self._lookup_kwargs['city_id__in'] = d.get('city')
            if d.get('members'):
                self._local_fields.append('members')
                self._lookup_kwargs['members_choice__in'] = d.get('members')
            if d.get('visitors'):
                self._local_fields.append('visitors')
                self._lookup_kwargs['visitors_choice__in'] = d.get('visitors')
            if d.get('price'):
                self._local_fields.append('price')
                self._lookup_kwargs['price_choice__in'] = d.get('price')
        return self._lookup_kwargs

    def filter(self, qs=None):
        qs = qs or self.default_filter()
        # lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values()))
        qs = self.make_data_begin_filter(qs)
        return qs.filter(**self.lookup_kwargs).order_by('data_begin')

    def default_filter(self, load_all=True, _models=None):
        models = _models or self.models
        qs = RelatedSearchQuerySet().models(*models)
        if load_all:
            qs = qs.load_all()
            for model in models:
                qs = qs.load_all_queryset(model, model.enable.all())
        if not self.get_date_begin_periods() and not self.get_date_from_to_range():
            qs = qs.filter(data_begin__gte=datetime.now().date())
        return qs

    def make_data_begin_filter(self, qs):
        params = self.make_date_begin_sqs_params()
        if params is not None:
            qs = qs.filter(params)
        return qs

    def recalculate_choices(self):
        for field in ['theme', 'tag', 'city', 'country']:
            # field_qs = self.default_filter(load_all=False)
            # if not field_lookup_kwargs:
            #     continue
            # field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id)
            # if field == 'theme':
            #     self.fields[field].queryset = self.get_theme_choices(field_qs)
            values = ['pk', 'name']
            order_by = []
            select = self.make_count_select(field)
            qs = self.fields[field].queryset
            for key in select.iterkeys():
                values.append(key)
                if key == 'selected':
                    order_by.insert(0, '-' + key)
                else:
                    order_by.append('-' + key)

            qs = qs.extra(select=select)
            if 'count' in values:
                having = [''' `count` > 0 ''']
                if 'selected' in values:
                    having.append(''' `selected` = 1 ''')

                qs.query.having.add(ExtraWhere(having, [], OR), AND)
            qs = qs.values(*values).order_by(*order_by)
            self.fields[field].queryset = qs

            # для того чтобы взяло чойсы из новых результатов
            self.fields[field].cache_choices = False
            self.fields[field].choice_cache = None
            print(self.fields[field].queryset.query)

        for field in ['members', 'visitors', 'price']:
            self.fields[field].choices = self.make_local_field_count(field) or self.fields[field].choices

        self.fields['month'].choices = self.make_date_begin_counts()
        self.make_event_type_choices_count()
        # self.fields['month'].choices = self.month_choices()

        # for field in self.fields:
        #     field = self.fields[field]
        #     if hasattr(field, 'queryset'):
        #         field.queryset = field.queryset[:15]

    def make_event_type_choices_count(self):
        types = {1: Exposition, 2: Conference}
        choices = []
        for _type, label in TYPES:
            qs = self.default_filter(load_all=False, _models=[types.get(_type)])
            qs = self.make_data_begin_filter(qs)
            count = qs.filter(**self.lookup_kwargs).count()
            choices.append((_type, label + ' <i>({count})</i>'.format(count=count)))
        self.fields['event_type'].choices = choices

    def make_ids_in_sql_format(self, values):
        # stance(values, (list, tuple, set)):
        #     return tuple(values) if len(values) > 1 else '({})'.format(*values)
        # return
        return tuple(values) if len(values) > 1 else '({})'.format(*values)

    def make_joins_from_selected(self, field, model):
        joins = []
        where = []

        # joins and where lookups in relations
        for l_field in self._related_fields:
            if l_field == field:
                continue
            _l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field)
            values = self.cleaned_data[l_field]
            if _m2m and _direct and isinstance(_l_field, ManyToManyField):
                _format_kwargs = {
                    'm2m_db_table': _l_field.m2m_db_table(),
                    'm2m_column_name': _l_field.m2m_column_name(),
                    'db_table': model._meta.db_table,
                    'm2m_reverse_name': _l_field.m2m_reverse_name(),
                    'ids': self.make_ids_in_sql_format(values),
                }
                joins.append(
                    ''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\
                    .format(**_format_kwargs))
                where.append(
                    '''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\
                    .format(**_format_kwargs))
            elif not _m2m and _direct and isinstance(_l_field, ForeignKey):
                _format_kwargs = {
                    'db_table': model._meta.db_table,
                    'attname': _l_field.column,
                    'ids': self.make_ids_in_sql_format(values),
                }
                where.append(
                    '''`{db_table}`.`{attname}` IN {ids}'''\
                    .format(**_format_kwargs))

        # joins and where lookups in local fields
        for l_field in self._local_fields:
            if l_field == field:
                continue
            val = self.cleaned_data[l_field]
            _format_kwargs = {
                'db_table': model._meta.db_table,
                'attname': fields_mapping.get(l_field, l_field),
                'val': self.make_ids_in_sql_format(val),
            }
            where.append(
                '''`{db_table}`.`{attname}` IN {val}'''\
                .format(**_format_kwargs))

        return joins, where

    def make_local_field_count(self, field):
        sql = ''
        selects = []
        l_field = fields_mapping.get(field, field)
        _values_mapping = values_mapping.get(l_field).items()
        for model in self.models:
            joins = []
            where = []
            group_by = ''

            format_kwargs = {
                'db_table': model._meta.db_table,
                'l_field': l_field,
                'lang': self.lang,
            }

            cases = []
            for key, val in _values_mapping:
                cases.append(
                    ''' sum(case when (`{db_table}`.`{l_field}` = {val}) then 1 else 0 end) as '{key}' '''\
                    .format(key=key, val=val.get('value'), **format_kwargs)
                    )

            select = \
                ''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
                .format(cases=', '.join(cases), **format_kwargs)

            where.append(
                ''' `{db_table}_translation`.`language_code` = '{lang}' '''\
                .format(**format_kwargs)
                )

            _joins, _where = self.make_joins_from_selected(field, model)
            joins.extend(_joins)
            where.extend(_where)
            where.append(self.make_default_where(db_table=model._meta.db_table))
            self.make_date_begin_where(where, db_table=model._meta.db_table)

            selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)

        sql = ' union '.join(selects)
        # print(sql)
        choices = []
        if sql:
            c = connection.cursor()
            try:
                c.execute(sql)
                mapper = namedtuple('Result', [col[0] for col in c.description])
                data = [mapper(*raw) for raw in c.fetchall()]
                for key, val in _values_mapping:
                    count = sum([getattr(x, key, 0) or 0 for x in data])
                    choices.append((val.get('value'), val.get('label') + ' <i>({count})</i>'.format(count=count)))
            finally:
                c.close()
            # some bug with these! AttributeError: __exit__
            # with connection.cursor() as c:
        return choices

    def make_default_where(self, **kwargs):
        where = [''' (`{db_table}`.`is_published` = True) '''.format(**kwargs)]
        if not self.get_date_begin_periods() and not self.get_date_from_to_range():
            where.append(''' (`{db_table}`.`data_begin` >= '{date_today}') '''\
                .format(date_today=datetime.now().strftime('%Y-%m-%d'), **kwargs)
            )
        return ' AND '.join(where)

    def make_date_begin_counts(self):
        ranges = []
        cleaned_year = self.cleaned_data.get('year')
        for value, label in self.fields['month'].choices:
            match, _year, month = get_month_and_year(value)
            if not match and cleaned_year and cleaned_year == self.clean_year(value=value):
                _year = cleaned_year
            elif not match:
                _year = cleaned_year or datetime.today().year
                month = value
            ranges.append({
                'value': value,
                'label': label,
                'year': _year,
                'month': monthes_abr_to_num.get(month) if month else None
            })

        sql = ''
        selects = []
        for model in self.models:
            joins = []
            where = []
            group_by = ''

            format_kwargs = {
                'db_table': model._meta.db_table,
                'field': 'data_begin',
                'lang': self.lang,
            }

            cases = []
            for case in ranges:
                if case.get('month'):
                    cases.append(
                        ''' sum(case when (year(`{db_table}`.`{field}`) = {year} and month(`{db_table}`.`{field}`) = {month}) then 1 else 0 end) as '{key}' '''\
                        .format(key=case['value'], year=case['year'], month=case['month'], **format_kwargs)
                    )
                else:
                    cases.append(
                        ''' sum(case when (year(`{db_table}`.`{field}`) = {year}) then 1 else 0 end) as '{key}' '''\
                        .format(key=case['value'], year=case['year'], **format_kwargs)
                    )

            select = \
                ''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
                .format(cases=', '.join(cases), **format_kwargs)

            where.append(
                ''' `{db_table}_translation`.`language_code` = '{lang}' '''\
                .format(**format_kwargs)
                )

            _joins, _where = self.make_joins_from_selected('data_begin', model)
            joins.extend(_joins)
            where.extend(_where)
            where.append(self.make_default_where(db_table=model._meta.db_table))

            selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)

        sql = ' union '.join(selects)
        # print(sql)
        choices = []
        if sql:
            c = connection.cursor()
            try:
                c.execute(sql)
                mapper = namedtuple('Result', ['c' + col[0] for col in c.description])
                data = [mapper(*raw) for raw in c.fetchall()]
                for value, label in self.fields['month'].choices:
                    count = sum([getattr(x, 'c' + value, 0) or 0 for x in data])
                    choices.append((value, label + ' <i>({count})</i>'.format(count=count)))
            finally:
                c.close()
            # some bug with these! AttributeError: __exit__
            # with connection.cursor() as c:
        return choices

    def get_date_begin_periods(self):
        periods = getattr(self, '_periods', None)
        if periods is None:
            periods = {}
            cleaned_year = self.cleaned_data.get('year')
            cleaned_year_str = str(cleaned_year)
            year = cleaned_year or datetime.today().year
            current_month = datetime.today().month
            for value in self.cleaned_data['month']:
                match, _year, month = get_month_and_year(value)
                if not match and cleaned_year and cleaned_year == self.clean_year(value=value):
                    periods.setdefault(cleaned_year, [])
                    continue
                elif not match:
                    _year = year
                    month = value
                month_num = monthes_abr_to_num.get(month)
                periods.setdefault(_year, []).append(month_num)
            if cleaned_year\
                and not cleaned_year_str in self.cleaned_data.get('month', [])\
                and not periods.get(cleaned_year)\
                and not self.fields['month'].default_month\
                and not self.cleaned_data.get('default_year'):
                periods[cleaned_year] = []
                if self.fields['month'].valid_value(cleaned_year):
                    self.cleaned_data.setdefault('month', []).append(cleaned_year_str)
                    self.data.update({'month': cleaned_year_str})
                    self.data['default_year'] = cleaned_year_str
            elif cleaned_year\
                and not cleaned_year_str in self.cleaned_data.get('month', [])\
                and not periods.get(cleaned_year)\
                and not self.fields['month'].default_month\
                and self.cleaned_data.get('default_year'):
                self.fields['month'].choices.insert(0, self.fields['month'].make_year_choice(cleaned_year))
            elif cleaned_year and cleaned_year_str in self.cleaned_data.get('month', []) and periods.get(cleaned_year):
                periods[cleaned_year] = []
                cleaned = self.data.getlist('month', [])
                cleaned.remove(cleaned_year_str)
                self.cleaned_data.setdefault('month', []).remove(cleaned_year_str)
                self.data.setlist('month', cleaned)
            elif cleaned_year and periods.get(cleaned_year) and not self.fields['month'].default_month:
                self.fields['month'].valid_value(cleaned_year)
            elif cleaned_year and self.fields['month'].default_month\
                and not list(filter(lambda x: x[0] == cleaned_year_str + self.fields['month'].default_month, self.fields['month'].choices)):
                self.fields['month'].valid_value(self.fields['month'].default_month)
            self._periods = periods
        return periods

    def get_date_from_to_range(self):
        date_from = self.cleaned_data.get('date_from')
        date_to = self.cleaned_data.get('date_to')
        if date_to or date_from:
            if not date_from:
                date_from = date.today()
            return date_from, date_to
        return None

    def make_date_begin_where(self, where, db_table):
        key = '_where_date_begin_where_{}'.format(db_table)
        _where = getattr(self, key, None)
        if _where is None:
            _where = []
            # years = {}
            # current_month = datetime.today().month
            # year = datetime.today().year
            # for month in self.cleaned_data['month']:
            #     month_num = monthes_abr_to_num.get(month)
            #     _year = year
            #     if month_num < current_month:
            #         _year += 1
            #     years.setdefault(_year, []).append(month_num)
            # kwargs.update(years)
            date_range = self.get_date_from_to_range()
            if date_range is not None:
                date_from, date_to = date_range
                w = [''' `{db_table}`.`data_begin` >= '{date_from}' '''.format(date_from=date_from, db_table=db_table)]
                if date_to:
                    w.append(''' `{db_table}`.`data_begin` <= '{date_to}' '''.format(date_to=date_to, db_table=db_table))
                _where.append(' ({where}) '.format(where=' AND '.join(w)))
            else:
                add_years = set()
                for year, monthes in self.get_date_begin_periods().iteritems():
                    if monthes:
                        _where.append(
                        ''' (month(`{db_table}`.`data_begin`) in {monthes} and year(`{db_table}`.`data_begin`) = {year}) '''\
                            .format(year=year, monthes=self.make_ids_in_sql_format(monthes), db_table=db_table)
                        )
                    else:
                        add_years.add(year)
                if add_years:
                    _where.append(
                        ''' (year(`{db_table}`.`data_begin`) in {year}) '''\
                            .format(year=self.make_ids_in_sql_format(add_years), db_table=db_table)
                        )
            if len(_where) > 1:
                _where = [''' ({}) '''.format(' OR '.join(_where))]
            setattr(self, key, _where)
        # print(_where)
        where.extend(_where)
        return

    # def get_prev_month(self, date):
    #     year = date.year
    #     month = date.month
    #     day = date.day
    #     last_day = False
    #     if day == calendar.monthrange(year, month)[1]:
    #         last_day = True
    #     if date.month == 1:
    #         year -= 1
    #         month = 12
    #     if last_day:
    #         day = calendar.monthrange(year, month)[1]
    #     return date.replace(month=month, year=year, day=day)

    # def get_next_month(self, date):
    #     year = date.year
    #     month = date.month
    #     day = date.day
    #     last_day = False
    #     if day == calendar.monthrange(year, month)[1]:
    #         last_day = True
    #     if date.month == 12:
    #         year += 1
    #         month = 1
    #     if last_day:
    #         day = calendar.monthrange(year, month)[1]
    #     return date.replace(month=month, year=year, day=day)

    def get_start_of_period(self, year, periods, month=None):
        if month is None:
            return year, 1
        self.checked_monthes.setdefault(year, set()).add(month)
        if month == 12:
            prev_month = month - 1
            prev_year = year
        elif month == 1:
            prev_month = 12
            prev_year = year - 1
        else:
            prev_month = month - 1
            prev_year = year
        if prev_month in periods.get(prev_year, []):
            return self.get_start_of_period(prev_year, periods, prev_month)
        return year, month

    def get_end_of_period(self, year, periods, month=None):
        if month is None:
            return year, 12
        self.checked_monthes.setdefault(year, set()).add(month)
        if month == 12:
            next_month = 1
            next_year = year + 1
        elif month == 1:
            next_month = month + 1
            next_year = year
        else:
            next_month = month + 1
            next_year = year
        if next_month in periods.get(next_year, []):
            return self.get_end_of_period(next_year, periods, next_month)
        return year, month

    def make_date_begin_sqs_params(self):
        periods = []
        self.checked_monthes = {}
        cleaned_periods = self.get_date_begin_periods()
        # print(cleaned_periods)
        date_range = self.get_date_from_to_range()
        if date_range is not None:
            periods.append(date_range)
        else:
            for year, monthes in cleaned_periods.iteritems():
                for month in monthes:
                    if month in self.checked_monthes.get(year, []):
                        continue
                    start_year, start_month = self.get_start_of_period(year, cleaned_periods, month)
                    end_year, end_month = self.get_end_of_period(year, cleaned_periods, month)
                    _first_day, _last_day = calendar.monthrange(end_year, end_month)
                    periods.append((
                        date.today().replace(day=1, month=start_month, year=start_year),
                        date.today().replace(day=_last_day, month=end_month, year=end_year)
                    ))
                if not monthes:
                    start_year, start_month = self.get_start_of_period(year, cleaned_periods)
                    end_year, end_month = self.get_end_of_period(year, cleaned_periods)
                    _first_day, _last_day = calendar.monthrange(end_year, end_month)
                    periods.append((
                        date.today().replace(day=1, month=start_month, year=start_year),
                        date.today().replace(day=_last_day, month=end_month, year=end_year)
                    ))

        params = None
        for start, end in periods:
            lookup = {'data_begin__range': (start, end)} if end else {'data_begin': start}
            if params is None:
                params = SQ(**lookup)
            else:
                params |= SQ(**lookup)
        return params

    def make_count_select(self, field):
        selects = []
        case = None
        count = None
        # print('looking {} {}'.format(field, self.lookup_kwargs))
        for model in self.models:
            _field, _model, direct, m2m = model._meta.get_field_by_name(field)
            joins = []
            where = []
            group_by = ''
            # ManyToManyField
            if m2m and direct and isinstance(_field, ManyToManyField):
                _field
                format_kwargs = {
                    'm2m_db_table': _field.m2m_db_table(),
                    'm2m_column_name': _field.m2m_column_name(),
                    'db_table': model._meta.db_table,
                    'm2m_reverse_name': _field.m2m_reverse_name(),
                    'm2m_rel_to_table': _field.rel.to._meta.db_table,
                    'lang': self.lang,
                }

                select = \
                    '''SELECT COUNT(DISTINCT `{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\
                    .format(**format_kwargs)
                where.append(
                    ''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) AND `{m2m_rel_to_table}_translation`.`language_code` = '{lang}' '''.format(**format_kwargs)
                )

            # ForeignKey
            elif not m2m and direct and isinstance(_field, ForeignKey):
                _format_kwargs = {
                    'attname': _field.column,
                    'db_table': model._meta.db_table,
                    'rel_db_table': _field.rel.to._meta.db_table,
                    'lang': self.lang,
                    # 'attname': _field.related.field.get_attname(),
                }
                select = \
                    '''SELECT COUNT(DISTINCT `{db_table}`.`id`) FROM `{db_table}`'''\
                    .format(**_format_kwargs)
                    # '''SELECT COUNT(`{db_table}`.`id`) FROM `{rel_db_table}` LEFT OUTER JOIN `{db_table}` ON (`{rel_db_table}`.`id` = `{db_table}`.`{attname}`)'''\
                where.append(
                    ''' `{db_table}`.`{attname}` = `{rel_db_table}_translation`.`master_id` AND `{rel_db_table}_translation`.`language_code` = '{lang}' '''\
                    .format(**_format_kwargs))
                group_by = ''' GROUP BY `{rel_db_table}_translation`.`master_id` '''.format(**_format_kwargs)

            # mark selected items (for ordering)
            # only for M2M and ForeignKey
            if case is None and field in self._related_fields and direct\
                and (isinstance(_field, ForeignKey) or isinstance(_field, ManyToManyField)):
                values = self.cleaned_data[field]
                case = \
                ''' CASE WHEN `{table}`.`master_id` in {ids} THEN 1 ELSE 0 END '''.format(
                    table=self.fields[field].queryset.model._meta.db_table,
                    ids=self.make_ids_in_sql_format(values))

            # FILTER current by other values
            _joins, _where = self.make_joins_from_selected(field, model)
            joins.extend(_joins)
            where.extend(_where)
            where.append(self.make_default_where(db_table=model._meta.db_table))
            self.make_date_begin_where(where, db_table=model._meta.db_table)
            selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)

        if len(selects) == 2:
            count = ''' IFNULL(({0}), 0) + IFNULL(({1}), 0) '''.format(*selects)
        elif len(selects) == 1:
            count = selects[0]
        d = {}
        if case is not None:
            d['selected'] = case
        if count:
            d['count'] = count
        return d



### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый
## 1-й рабочий способ (без заполнения перевода)
# """
# SELECT
#     `theme_theme`.`id`,
#     (
#     SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
#     FROM `exposition_exposition_theme`
#     WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)
#     ) AS `e_count`,
#     (
#     SELECT COUNT(`conference_conference_theme`.`conference_id`)
#     FROM `conference_conference_theme`
#     WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`)
#     ) AS `c_count`,
#     `theme_theme_translation`.`name` as `name`
# FROM
#     `theme_theme`
# GROUP BY `theme_theme`.`id`
# ORDER BY NULL
# """

## 2-й рабочий способ (с заполнением перевода)
## аттрибут перевода 'name' присвоен на 'name_t', чтобы не ругалось на неправильно заполненый перевод
# theme_sql = \
# """
# SELECT
#     `theme_theme_translation`.`name` AS `name_t`,
#     `theme_theme_translation`.`master_id` AS `id`,
#     (
#     SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
#     FROM `exposition_exposition_theme`
#     WHERE (`theme_theme_translation`.`master_id` = `exposition_exposition_theme`.`theme_id`)
#     ) AS `e_count`,
#     (
#     SELECT COUNT(`conference_conference_theme`.`conference_id`)
#     FROM `conference_conference_theme`
#     WHERE (`theme_theme_translation`.`master_id` = `conference_conference_theme`.`theme_id`)
#     ) AS `c_count`
# FROM
#     `theme_theme_translation`
# WHERE (`theme_theme_translation`.`language_code` = 'ru')
# GROUP BY `theme_theme_translation`.`name`
# ORDER BY NULL
# """
# qs = Theme.objects.raw(theme_sql)
# _theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` '''
# theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) '''
# theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})'''
# _theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` '''
# theme_conf_count =  _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) '''
# theme_conf_count_selected =  _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})'''

# tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) '''
# tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) '''


# 3-й рабочий способ (с родным заполением перевода)
# в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8
# https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions
# from django.db.models.expressions import RawSQL

'''
SELECT
    `city_city_translation`.`master_id`,
    `<city_city_translation></city_city_translation>`.`name`,
    COUNT(`conference_conference`.`id`) AS `count`
FROM
    `city_city_translation`
        LEFT OUTER JOIN
    `city_city` ON (`city_city_translation`.`master_id` = `city_city`.`id`)
        LEFT OUTER JOIN
    `conference_conference` ON (`city_city`.`id` = `conference_conference`.`city_id`)
        LEFT OUTER JOIN
    `conference_conference_tag` ON (`conference_conference`.`id` = `conference_conference_tag`.`conference_id`)
WHERE
    `conference_conference_tag`.`tag_id` IN (469 , 832, 366, 922)
GROUP BY `city_city_translation`.`language_code` , `city_city_translation`.`master_id`
ORDER BY NULL
'''

'''
SELECT
    (SELECT
            COUNT(`exposition_exposition`.`country_id`)
        FROM
            `exposition_exposition`
                INNER JOIN
            `exposition_exposition_theme` ON (`exposition_exposition`.`id` = `exposition_exposition_theme`.`exposition_id`)
        where
            `exposition_exposition_theme`.`theme_id` IN (1, 2, 3)
            and `exposition_exposition`.`country_id` = `country_country_translation`.`master_id`
        GROUP BY `country_country_translation`.`master_id`) AS `count`,
    `country_country_translation`.`master_id`,
    `country_country_translation`.`name`
FROM
    `country_country_translation`
'''

'''
SELECT
    (SELECT
            COUNT(`exposition_exposition_theme`.`exposition_id`)
        FROM
            `exposition_exposition_theme`
                INNER JOIN
            `exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`)
                INNER JOIN
            `exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`)
        WHERE
            `exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922)
                and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`,
    CASE
        WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1
        ELSE 0
    END as `selected`,
    `theme_theme_translation`.`id`,
    `theme_theme_translation`.`name`,
    `theme_theme_translation`.`language_code`,
    `theme_theme_translation`.`master_id`
FROM
    `theme_theme_translation`
        INNER JOIN
    `theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`)
WHERE
    (`theme_theme`.`types` = (`theme_theme`.`types` | 1)
        OR `theme_theme`.`types` = (`theme_theme`.`types` | 2))
        AND (`theme_theme_translation`.`language_code` = 'ru')
order BY `selected` DESC , `count` DESC
'''