You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

1216 lines
53 KiB

# -*- coding: utf-8 -*-
import re
import operator
from itertools import chain, combinations
from collections import namedtuple
from datetime import datetime
from datetime import timedelta
from datetime import date
from datetime import MAXYEAR, MINYEAR
from dateutil import relativedelta
import calendar
# from calendar import TimeEncoding, month_name
try:
from collections import ChainMap
except ImportError:
from chainmap import ChainMap
from django import forms
from django.utils.translation import get_language, ugettext as _
from django.utils.translation import string_concat
from django.utils.encoding import smart_text, force_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.db.models import Count, Sum, Q, ForeignKey, ManyToManyField
from django.db import connection
from django.core.exceptions import ValidationError
from django.core.cache import cache
from django.conf import settings
from haystack.query import SearchQuerySet, RelatedSearchQuerySet, SQ
from functions.model_utils import EnumChoices
# from functions.model_mixin import get_dates
from exposition.models import Exposition
from conference.models import Conference
from theme.models import Theme, Tag
from country.models import Country
from city.models import City
from events.common import MEMBERS, VISITORS, PRICE, TYPES
from events.common import members_mapping, visitors_mapping, price_mapping
from events.common import ExtraWhere, OR, AND
class FilterCheckboxSelectMultiple(forms.CheckboxSelectMultiple):
def render(self, name, value, attrs=None, choices=()):
if value is None: value = []
has_id = attrs and 'id' in attrs
final_attrs = self.build_attrs(attrs, name=name)
output = ['<ul>']
# Normalize to strings
str_values = set([force_text(v) for v in value])
for i, (option_value, option_label) in enumerate(chain(self.choices, choices)):
# If an ID attribute was given, add a numeric index as a suffix,
# so that the checkboxes don't all have the same ID attribute.
if has_id:
final_attrs = dict(final_attrs, id='%s_%s' % (attrs['id'], i))
label_for = format_html(u' for="{0}"', final_attrs['id'])
else:
label_for = ''
cb = forms.CheckboxInput(final_attrs, check_test=lambda value: value in str_values)
option_value = force_text(option_value)
rendered_cb = cb.render(name, option_value)
option_label = mark_safe(force_text(option_label))
output.append(format_html(u'<li>{1}<label{0}><span>{2}</span></label></li>',
label_for, rendered_cb, option_label))
output.append(u'</ul>')
return mark_safe('\n'.join(output))
class WidgetDefaultMixin(object):
def widget_attrs(self, widget):
attrs = super(WidgetDefaultMixin, self).widget_attrs(widget)
attrs['class'] = 'default'
return attrs
class CountModelMultipleChoiceField(WidgetDefaultMixin, forms.ModelMultipleChoiceField):
widget = FilterCheckboxSelectMultiple
def label_from_instance(self, obj):
if obj.get('count', None) is None:
return smart_text(obj.get('name'))
return u'{label} <i>({count})</i>'.format(label=smart_text(obj.get('name')), count=obj.get('count'))
def prepare_value(self, value):
if isinstance(value, dict):
return value.get('pk')
return super(CountModelMultipleChoiceField, self).prepare_value(value)
def clean(self, value):
# return pks instead of QuerySet
if self.required and not value:
raise ValidationError(self.error_messages['required'])
elif not self.required and not value:
return self.queryset.none()
if not isinstance(value, (list, tuple)):
raise ValidationError(self.error_messages['list'])
key = self.to_field_name or 'pk'
for pk in value:
try:
self.queryset.filter(**{key: pk})
except ValueError:
raise ValidationError(self.error_messages['invalid_pk_value'] % pk)
pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True)))
for val in value:
if int(force_text(val)) not in pks:
raise ValidationError(self.error_messages['invalid_choice'] % val)
# Since this overrides the inherited ModelChoiceField.clean
# we run custom validators here
self.run_validators(value)
return pks
class FilterTypedMultipleChoiceField(WidgetDefaultMixin, forms.TypedMultipleChoiceField):
pass
class MonthChoiceField(FilterTypedMultipleChoiceField):
year = None
additional_choices = []
default_month = None
def make_year_choice(self, value):
return (str(value), _(u'{year} год').format(year=value))
def valid_value(self, value):
valid = super(MonthChoiceField, self).valid_value(value)
if not valid:
if self.year and (value == self.year or check_year(value) == self.year):
choice = self.make_year_choice(value)
self.additional_choices.append(choice)
self.choices.insert(0, choice)
return True
value = str(value)
year = self.year or datetime.today().year
match, _year, month = get_month_and_year(value)
if match:
year = _year
self.default_month = month
elif value in settings.MONTHES.keys():
month = value
self.default_month = month
if month in settings.MONTHES.keys():
choice = (value, string_concat(settings.MONTHES[month]['name'], ' ', year))
self.additional_choices.append(choice)
self.choices.insert(0, choice)
# print(choice, self.choices)
return super(MonthChoiceField, self).valid_value(value)
return valid
fields_mapping = {
'members': 'members_choice',
'visitors': 'visitors_choice',
'price': 'price_choice',
}
values_mapping = {
'members_choice': members_mapping,
'visitors_choice': visitors_mapping,
'price_choice': price_mapping,
}
RATING = (
('r1', _(u'Топовые (HIT)')),
('r2', _(u'Члены РСВЯ')),
('r3', _(u'Члены UFI')),
('r4', _(u'ExpoRating')),
)
rating_mapping = [
{'value': 'r1', 'db_value': 1, 'label': _(u'Топовые (HIT)'), 'field': 'expohit'},
{'value': 'r2', 'db_value': int(Exposition.quality_label.ufi), 'label': _(u'Члены РСВЯ'), 'field': 'quality_label'},
{'value': 'r3', 'db_value': int(Exposition.quality_label.rsva), 'label': _(u'Члены UFI'), 'field': 'quality_label'},
{'value': 'r4', 'db_value': int(Exposition.quality_label.exporating), 'label': _(u'ExpoRating'), 'field': 'quality_label'},
]
monthes_abr_to_num = {v.lower(): k for k, v in enumerate(calendar.month_abbr)}
monthes_num_to_abr = {v: k for k, v in monthes_abr_to_num.iteritems()}
year_month_regex = re.compile(r'^((?P<year>\d{4})(?P<month>\w{3}))|((?P<month_>\d{1,2})/(?P<year_>\d{4}))$')
def get_month_and_year(input_string):
year = None
month = None
match = year_month_regex.match(input_string)
if match:
month = match.group('month') or monthes_num_to_abr.get(int(match.group('month_')))
year = int(match.group('year') or match.group('year_'))
return bool(match), year, month
def check_year(value):
year = None
try:
year = int(value)
except (ValueError, ):
return
else:
if not MAXYEAR >= year >= MINYEAR:
return
return year
class FilterForm(forms.Form):
_month_choices = None
_month = None
event_type = FilterTypedMultipleChoiceField(
label=_(u'Тип события'), coerce=int,
choices=TYPES, required=False, widget=FilterCheckboxSelectMultiple())
theme = CountModelMultipleChoiceField(
label=_(u'Тематики'), required=False, cache_choices=True,
queryset=Theme.objects.language().values('pk', 'name'))
tag = CountModelMultipleChoiceField(
label=_(u'Теги'), required=False, cache_choices=True,
queryset=Tag.objects.language().values('pk', 'name'))
country = CountModelMultipleChoiceField(
label=_(u'Страны'), required=False, cache_choices=True,
queryset=Country.objects.language().values('pk', 'name'))
city = CountModelMultipleChoiceField(
label=_(u'Города'), required=False, cache_choices=True,
queryset=City.objects.language().values('pk', 'name'))
year = forms.RegexField(r'^(?P<year>\d{4})$', max_length=4, min_length=4,
required=False, widget=forms.HiddenInput())
default_month = forms.CharField(max_length=3, min_length=3,
required=False, widget=forms.HiddenInput())
default_year = forms.RegexField(r'^(?P<year>\d{4})$', max_length=4, min_length=4,
required=False, widget=forms.HiddenInput())
month = MonthChoiceField(
label=_(u'Период'), coerce=str,
choices=monthes_abr_to_num.iteritems(),
required=False, widget=FilterCheckboxSelectMultiple())
date_from = forms.DateField(required=False, input_formats=('%d.%m.%Y',),
widget=forms.DateInput(attrs={'class': 'date', 'id': 'dateFrom',
'placeholder': _(u'дд.мм.гггг')}))
date_to = forms.DateField(required=False, input_formats=('%d.%m.%Y',),
widget=forms.DateInput(attrs={'class': 'date', 'id': 'dateTo',
'placeholder': _(u'дд.мм.гггг')}))
price = FilterTypedMultipleChoiceField(
label=_(u'Стоимость'), coerce=int,
choices=PRICE,
required=False, widget=FilterCheckboxSelectMultiple(),
# help_text=_(u'За 1 м<sub>2</sub> необорудованной площади')
)
members = FilterTypedMultipleChoiceField(
label=_(u'Участники'), coerce=int,
choices=MEMBERS,
required=False, widget=FilterCheckboxSelectMultiple())
visitors = FilterTypedMultipleChoiceField(
label=_(u'Посетители'), coerce=int,
choices=VISITORS,
required=False, widget=FilterCheckboxSelectMultiple())
rating = FilterTypedMultipleChoiceField(
label=_(u'Рейтинги'), coerce=str,
choices=RATING,
required=False, widget=FilterCheckboxSelectMultiple())
def __init__(self, *args, **kwargs):
super(FilterForm, self).__init__(*args, **kwargs)
self._is_valid = False
self._models = None
self._lookup_kwargs = None
self._lookup_args = None
self._related_fields = []
self._local_fields = []
self.lang = get_language()
self.data = self.data.copy()
self.fill_default_choices_from_cache()
self.fields['month'].choices = self.month_choices()
def fill_default_choices_from_cache(self):
timeout = timedelta(hours=1).seconds
for field in ['theme', 'tag', 'country', 'city']:
key ='filter_{field}_{lang}'.format(field=field, lang=self.lang)
choices = cache.get(key)
if choices is None:
choices = list(self.fields[field].choices)
cache.set(key, choices, timeout)
else:
self.fields[field].choice_cache = choices
def get_dates(self):
if not (self.cleaned_data.get('date_from') or self.cleaned_data.get('date_to')):
return _(u'Указать диапазон')
result = ''
if self.cleaned_data.get('date_from'):
result += _(u'c {date_from} ').format(date_from=self.cleaned_data.get('date_from').strftime('%d.%m.%Y'))
if self.cleaned_data.get('date_to'):
result += _(u'по {date_to}').format(date_to=self.cleaned_data.get('date_to').strftime('%d.%m.%Y'))
return result
@classmethod
def month_choices(cls):
month = datetime.today().month
if cls._month != month or cls._month_choices is None:
year = datetime.today().year
depth = 7 # на сколько сколько месяцев вперед делать выборку (включая текущий)
monthes = dict([(v.get('value'), {'abr': k, 'name': v.get('name')}) for k, v in settings.MONTHES.iteritems()])
choices = []
for month_num in xrange(month, month + depth):
_year = year
if month_num > 12:
_year = year + 1
month_num -= 12
month_values = monthes.get(month_num)
# month_values['name'] = string_concat(month_values['name'], ' ', str(_year))
choices.append((str(_year) + month_values['abr'], string_concat(month_values['name'], ' ', str(_year))))
cls._month_choices = choices
cls._month = month
return cls._month_choices
def clean_year(self, value=None):
year = value or self.cleaned_data.get('year')
year = check_year(year)
self.fields['month'].year = year
return year
def clean_default_month(self):
v = self.cleaned_data.get('default_month')
if v and v in settings.MONTHES.keys():
self.fields['month'].default_month = v
return v
return
def clean_default_year(self):
y = self.cleaned_data.get('default_year')
y = check_year(y)
return y
def clean_month(self):
values = self.cleaned_data.get('month', [])
field = self.fields['month']
new_values = []
for val in values:
if len(val) == 3 and list(filter(lambda x: x[0] == str(field.year or datetime.today().year) + val, field.choices)):
new_values.append(str(field.year or datetime.today().year) + val)
field.choices = filter(lambda x: x[0] != val, field.choices)
elif '/' in val and list(filter(lambda x: x[0] == val, field.choices)):
try:
label = list(filter(lambda x: x[0] == val, field.choices))[0][1]
match, year, month = get_month_and_year(val)
if match:
new_val = str(year) + month
choices = list(filter(lambda x: x[0] != val, field.choices))
if not list(filter(lambda x: x[0] == new_val, field.choices)):
choices.insert(0, (new_val, label))
field.choices = choices
new_values.append(new_val)
except:
continue
else:
new_values.append(val)
if new_values != values:
self.data.setlist('month', new_values)
# print(new_values, self.data)
return new_values
def _post_clean(self):
# нужно для того, чтобы год зашел в поле month
# print(self.get_date_begin_periods())
if not self.cleaned_data.get('default_month') and self.fields['month'].default_month:
self.data['default_month'] = self.fields['month'].default_month
year = self.cleaned_data.get('year')
# if year:
# for val, name in self.fields['month'].additional_choices:
# self.
# additional_choices
def is_valid(self):
# if getattr(self, '_is_valid', None) is None:
self._is_valid = super(FilterForm, self).is_valid()
# нам нужно сбрасывать сохраненный выбор моделей,
# т.к. после валидации нужно вернуть только выбранные
self._models = None
self.cleaned_data = getattr(self, 'cleaned_data', {})
return self._is_valid
@property
def models(self):
if self._models is None and self._is_valid:
val = self.cleaned_data.get('event_type')
self._models = []
if TYPES.EXPO in val:
self._models.append(Exposition)
if TYPES.CONF in val:
self._models.append(Conference)
return self._models or [Exposition, Conference]
@property
def lookup_kwargs(self):
if self._lookup_kwargs is None:
d = self.cleaned_data
self._lookup_kwargs = {}
self._related_fields = []
self._local_fields = []
if d.get('theme'):
self._related_fields.append('theme')
self._lookup_kwargs['theme__in'] = d.get('theme')
if d.get('tag'):
self._related_fields.append('tag')
self._lookup_kwargs['tag__in'] = d.get('tag')
if d.get('country'):
self._related_fields.append('country')
self._lookup_kwargs['country_id__in'] = d.get('country')
if d.get('city'):
self._related_fields.append('city')
self._lookup_kwargs['city_id__in'] = d.get('city')
if d.get('members'):
self._local_fields.append('members')
self._lookup_kwargs['members_choice__in'] = d.get('members')
if d.get('visitors'):
self._local_fields.append('visitors')
self._lookup_kwargs['visitors_choice__in'] = d.get('visitors')
if d.get('price'):
self._local_fields.append('price')
self._lookup_kwargs['price_choice__in'] = d.get('price')
return self._lookup_kwargs
def filter(self, qs=None):
qs = qs or self.default_filter()
# lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values()))
qs = self.make_cleaned_sqs_filter(qs)
return qs.filter(**self.lookup_kwargs).order_by('data_begin')
def default_filter(self, load_all=True, _models=None):
models = _models or self.models
qs = RelatedSearchQuerySet().models(*models)
if load_all:
qs = qs.load_all()
for model in models:
qs = qs.load_all_queryset(model, model.enable.all())
if not self.get_date_begin_periods() and not self.get_date_from_to_range():
qs = qs.filter(data_begin__gte=datetime.now().date())
return qs
def make_cleaned_sqs_filter(self, qs):
date_params = self.make_date_begin_sqs_params()
if date_params is not None:
qs = qs.filter(date_params)
rating_params = self.make_rating_sqs_params()
if rating_params is not None:
qs = qs.filter(rating_params)
return qs
def recalculate_choices(self):
for field in ['theme', 'tag', 'city', 'country']:
# field_qs = self.default_filter(load_all=False)
# if not field_lookup_kwargs:
# continue
# field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id)
# if field == 'theme':
# self.fields[field].queryset = self.get_theme_choices(field_qs)
values = ['pk', 'name']
order_by = []
select = self.make_count_select(field)
qs = self.fields[field].queryset
for key in select.iterkeys():
values.append(key)
if key == 'selected':
order_by.insert(0, '-' + key)
else:
order_by.append('-' + key)
qs = qs.extra(select=select)
if 'count' in values:
having = [''' `count` > 0 ''']
if 'selected' in values:
having.append(''' `selected` = 1 ''')
qs.query.having.add(ExtraWhere(having, [], OR), AND)
qs = qs.values(*values).order_by(*order_by)
self.fields[field].queryset = qs
# для того чтобы взяло чойсы из новых результатов
self.fields[field].cache_choices = False
self.fields[field].choice_cache = None
print(self.fields[field].queryset.query)
for field in ['members', 'visitors', 'price']:
self.fields[field].choices = self.make_local_field_count(field) or self.fields[field].choices
self.fields['month'].choices = self.make_date_begin_counts()
self.fields['rating'].choices = self.make_rating_counts()
self.make_event_type_choices_count()
# self.fields['month'].choices = self.month_choices()
# for field in self.fields:
# field = self.fields[field]
# if hasattr(field, 'queryset'):
# field.queryset = field.queryset[:15]
def make_event_type_choices_count(self):
types = {1: Exposition, 2: Conference}
choices = []
for _type, label in TYPES:
qs = self.default_filter(load_all=False, _models=[types.get(_type)])
qs = self.make_cleaned_sqs_filter(qs)
count = qs.filter(**self.lookup_kwargs).count()
choices.append((_type, label + ' <i>({count})</i>'.format(count=count)))
self.fields['event_type'].choices = choices
def make_ids_in_sql_format(self, values):
# stance(values, (list, tuple, set)):
# return tuple(values) if len(values) > 1 else '({})'.format(*values)
# return
return tuple(values) if len(values) > 1 else '({})'.format(*values)
def make_joins_from_selected(self, field, model):
joins = []
where = []
# joins and where lookups in relations
for l_field in self._related_fields:
if l_field == field:
continue
_l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field)
values = self.cleaned_data[l_field]
if _m2m and _direct and isinstance(_l_field, ManyToManyField):
_format_kwargs = {
'm2m_db_table': _l_field.m2m_db_table(),
'm2m_column_name': _l_field.m2m_column_name(),
'db_table': model._meta.db_table,
'm2m_reverse_name': _l_field.m2m_reverse_name(),
'ids': self.make_ids_in_sql_format(values),
}
joins.append(
''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\
.format(**_format_kwargs))
where.append(
'''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\
.format(**_format_kwargs))
elif not _m2m and _direct and isinstance(_l_field, ForeignKey):
_format_kwargs = {
'db_table': model._meta.db_table,
'attname': _l_field.column,
'ids': self.make_ids_in_sql_format(values),
}
where.append(
'''`{db_table}`.`{attname}` IN {ids}'''\
.format(**_format_kwargs))
# joins and where lookups in local fields
for l_field in self._local_fields:
if l_field == field:
continue
val = self.cleaned_data[l_field]
_format_kwargs = {
'db_table': model._meta.db_table,
'attname': fields_mapping.get(l_field, l_field),
'val': self.make_ids_in_sql_format(val),
}
where.append(
'''`{db_table}`.`{attname}` IN {val}'''\
.format(**_format_kwargs))
return joins, where
def make_local_field_count(self, field):
sql = ''
selects = []
l_field = fields_mapping.get(field, field)
_values_mapping = values_mapping.get(l_field).items()
for model in self.models:
joins = []
where = []
group_by = ''
format_kwargs = {
'db_table': model._meta.db_table,
'l_field': l_field,
'lang': self.lang,
}
cases = []
for key, val in _values_mapping:
cases.append(
''' sum(case when (`{db_table}`.`{l_field}` = {val}) then 1 else 0 end) as '{key}' '''\
.format(key=key, val=val.get('value'), **format_kwargs)
)
select = \
''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
.format(cases=', '.join(cases), **format_kwargs)
where.append(
''' `{db_table}_translation`.`language_code` = '{lang}' '''\
.format(**format_kwargs)
)
_joins, _where = self.make_joins_from_selected(field, model)
joins.extend(_joins or [])
where.extend(_where or [])
where.append(self.make_default_where(db_table=model._meta.db_table))
self.make_date_begin_where(where, db_table=model._meta.db_table)
self.make_rating_where(where, db_table=model._meta.db_table)
selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
sql = ' union '.join(selects)
# print(sql)
choices = []
if sql:
c = connection.cursor()
try:
c.execute(sql)
mapper = namedtuple('Result', [col[0] for col in c.description])
data = [mapper(*raw) for raw in c.fetchall()]
for key, val in _values_mapping:
count = sum([getattr(x, key, 0) or 0 for x in data])
choices.append((val.get('value'), val.get('label') + ' <i>({count})</i>'.format(count=count)))
finally:
c.close()
# some bug with these! AttributeError: __exit__
# with connection.cursor() as c:
return choices
def make_default_where(self, **kwargs):
where = [''' (`{db_table}`.`is_published` = True) '''.format(**kwargs)]
if not self.get_date_begin_periods() and not self.get_date_from_to_range():
where.append(''' (`{db_table}`.`data_begin` >= '{date_today}') '''\
.format(date_today=datetime.now().strftime('%Y-%m-%d'), **kwargs)
)
return ' AND '.join(where)
def make_date_begin_counts(self):
ranges = []
cleaned_year = self.cleaned_data.get('year')
for value, label in self.fields['month'].choices:
match, _year, month = get_month_and_year(value)
if not match and cleaned_year and cleaned_year == self.clean_year(value=value):
_year = cleaned_year
elif not match:
_year = cleaned_year or datetime.today().year
month = value
ranges.append({
'value': value,
'label': label,
'year': _year,
'month': monthes_abr_to_num.get(month) if month else None
})
sql = ''
selects = []
for model in self.models:
joins = []
where = []
group_by = ''
format_kwargs = {
'db_table': model._meta.db_table,
'field': 'data_begin',
'lang': self.lang,
}
cases = []
for case in ranges:
if case.get('month'):
cases.append(
''' sum(case when (year(`{db_table}`.`{field}`) = {year} and month(`{db_table}`.`{field}`) = {month}) then 1 else 0 end) as '{key}' '''\
.format(key=case['value'], year=case['year'], month=case['month'], **format_kwargs)
)
else:
cases.append(
''' sum(case when (year(`{db_table}`.`{field}`) = {year}) then 1 else 0 end) as '{key}' '''\
.format(key=case['value'], year=case['year'], **format_kwargs)
)
select = \
''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
.format(cases=', '.join(cases), **format_kwargs)
where.append(
''' `{db_table}_translation`.`language_code` = '{lang}' '''\
.format(**format_kwargs)
)
_joins, _where = self.make_joins_from_selected('data_begin', model)
joins.extend(_joins or [])
where.extend(_where or [])
where.append(self.make_default_where(db_table=model._meta.db_table))
self.make_rating_where(where, db_table=model._meta.db_table)
selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
sql = ' union '.join(selects)
# print(sql)
choices = []
if sql:
c = connection.cursor()
try:
c.execute(sql)
mapper = namedtuple('Result', ['c' + col[0] for col in c.description])
data = [mapper(*raw) for raw in c.fetchall()]
for value, label in self.fields['month'].choices:
count = sum([getattr(x, 'c' + value, 0) or 0 for x in data])
choices.append((value, label + ' <i>({count})</i>'.format(count=count)))
finally:
c.close()
# some bug with these! AttributeError: __exit__
# with connection.cursor() as c:
return choices
def make_rating_counts(self):
choices = []
sql = ''
selects = []
for model in self.models:
joins = []
where = []
group_by = ''
format_kwargs = {
'db_table': model._meta.db_table,
'lang': self.lang,
}
cases = []
for case in filter(lambda x: x['field'] == 'expohit', rating_mapping):
cases.append(
''' sum(case when (`{db_table}`.`{field}` = {db_value}) then 1 else 0 end) as '{value}' '''\
.format(value=case['value'], field=case['field'], db_value=case['db_value'], **format_kwargs)
)
for case in filter(lambda x: x['field'] != 'expohit', rating_mapping):
cases.append(
''' sum(case when (`{db_table}`.`{field}` = `{db_table}`.`{field}` | {db_value}) then 1 else 0 end) as '{value}' '''\
.format(value=case['value'], field=case['field'], db_value=case['db_value'], **format_kwargs)
)
select = \
''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
.format(cases=', '.join(cases), **format_kwargs)
where.append(
''' `{db_table}_translation`.`language_code` = '{lang}' '''\
.format(**format_kwargs)
)
_joins, _where = self.make_joins_from_selected('rating', model)
joins.extend(_joins or [])
where.extend(_where or [])
where.append(self.make_default_where(db_table=model._meta.db_table))
self.make_date_begin_where(where, db_table=model._meta.db_table)
selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
sql = ' union '.join(selects)
choices = []
if sql:
c = connection.cursor()
try:
c.execute(sql)
mapper = namedtuple('Result', [col[0] for col in c.description])
data = [mapper(*raw) for raw in c.fetchall()]
for value, label in RATING:
count = sum([getattr(x, value, 0) or 0 for x in data])
choices.append((value, label + ' <i>({count})</i>'.format(count=count)))
finally:
c.close()
# some bug with these! AttributeError: __exit__
# with connection.cursor() as c:
return choices
def get_date_begin_periods(self):
periods = getattr(self, '_periods', None)
if periods is None:
periods = {}
cleaned_year = self.cleaned_data.get('year')
cleaned_year_str = str(cleaned_year)
year = cleaned_year or datetime.today().year
current_month = datetime.today().month
for value in self.cleaned_data['month']:
match, _year, month = get_month_and_year(value)
if not match and cleaned_year and cleaned_year == self.clean_year(value=value):
periods.setdefault(cleaned_year, [])
continue
elif not match:
_year = year
month = value
month_num = monthes_abr_to_num.get(month)
periods.setdefault(_year, []).append(month_num)
if cleaned_year\
and not cleaned_year_str in self.cleaned_data.get('month', [])\
and not periods.get(cleaned_year)\
and not self.fields['month'].default_month\
and not self.cleaned_data.get('default_year'):
periods[cleaned_year] = []
if self.fields['month'].valid_value(cleaned_year):
self.cleaned_data.setdefault('month', []).append(cleaned_year_str)
self.data.update({'month': cleaned_year_str})
self.data['default_year'] = cleaned_year_str
elif cleaned_year\
and not cleaned_year_str in self.cleaned_data.get('month', [])\
and not periods.get(cleaned_year)\
and not self.fields['month'].default_month\
and self.cleaned_data.get('default_year'):
self.fields['month'].choices.insert(0, self.fields['month'].make_year_choice(cleaned_year))
elif cleaned_year and cleaned_year_str in self.cleaned_data.get('month', []) and periods.get(cleaned_year):
periods[cleaned_year] = []
cleaned = self.data.getlist('month', [])
cleaned.remove(cleaned_year_str)
self.cleaned_data.setdefault('month', []).remove(cleaned_year_str)
self.data.setlist('month', cleaned)
elif cleaned_year and periods.get(cleaned_year) and not self.fields['month'].default_month:
self.fields['month'].valid_value(cleaned_year)
elif cleaned_year and self.fields['month'].default_month\
and not list(filter(lambda x: x[0] == cleaned_year_str + self.fields['month'].default_month, self.fields['month'].choices)):
self.fields['month'].valid_value(self.fields['month'].default_month)
self._periods = periods
return periods
def get_date_from_to_range(self):
date_from = self.cleaned_data.get('date_from')
date_to = self.cleaned_data.get('date_to')
if date_to or date_from:
if not date_from:
date_from = date.today()
return date_from, date_to
return None
def make_date_begin_where(self, where, db_table):
key = '_make_date_begin_where_{}'.format(db_table)
_where = getattr(self, key, None)
if _where is None:
_where = []
# years = {}
# current_month = datetime.today().month
# year = datetime.today().year
# for month in self.cleaned_data['month']:
# month_num = monthes_abr_to_num.get(month)
# _year = year
# if month_num < current_month:
# _year += 1
# years.setdefault(_year, []).append(month_num)
# kwargs.update(years)
date_range = self.get_date_from_to_range()
if date_range is not None:
date_from, date_to = date_range
w = [''' `{db_table}`.`data_begin` >= '{date_from}' '''.format(date_from=date_from, db_table=db_table)]
if date_to:
w.append(''' `{db_table}`.`data_begin` <= '{date_to}' '''.format(date_to=date_to, db_table=db_table))
_where.append(' ({where}) '.format(where=' AND '.join(w)))
else:
add_years = set()
for year, monthes in self.get_date_begin_periods().iteritems():
if monthes:
_where.append(
''' (month(`{db_table}`.`data_begin`) in {monthes} and year(`{db_table}`.`data_begin`) = {year}) '''\
.format(year=year, monthes=self.make_ids_in_sql_format(monthes), db_table=db_table)
)
else:
add_years.add(year)
if add_years:
_where.append(
''' (year(`{db_table}`.`data_begin`) in {year}) '''\
.format(year=self.make_ids_in_sql_format(add_years), db_table=db_table)
)
if len(_where) > 1:
_where = [''' ({}) '''.format(' OR '.join(_where))]
setattr(self, key, _where)
# print(_where)
where.extend(_where or [])
return
def make_rating_where(self, where, db_table):
key = '_make_rating_where_{}'.format(db_table)
_where = getattr(self, key, None)
if _where is None:
cleaned = self.cleaned_data.get('rating', [])
_where = []
# choices = filter(lambda x: x['value'] in cleaned and x['field'] == 'expohit', rating_mapping)
for choice in filter(lambda x: x['field'] == 'expohit' and x['value'] in cleaned, rating_mapping):
_where.append(
''' `{db_table}`.`{field}` = {db_value} '''\
.format(field=choice['field'], db_value=choice['db_value'], db_table=db_table)
)
quality_label = []
for choice in filter(lambda x: x['field'] == 'quality_label' and x['value'] in cleaned, rating_mapping):
quality_label.append(choice['db_value'])
if quality_label:
_where.append(
''' `{db_table}`.`{field}` = `{db_table}`.`{field}` | {computed} '''\
.format(field=choice['field'], computed=reduce(operator.or_, quality_label), db_table=db_table)
)
if len(_where) > 1:
_where = [''' ({}) '''.format(' OR '.join(_where))]
setattr(self, key, _where)
where.extend(_where or [])
return
# def get_prev_month(self, date):
# year = date.year
# month = date.month
# day = date.day
# last_day = False
# if day == calendar.monthrange(year, month)[1]:
# last_day = True
# if date.month == 1:
# year -= 1
# month = 12
# if last_day:
# day = calendar.monthrange(year, month)[1]
# return date.replace(month=month, year=year, day=day)
# def get_next_month(self, date):
# year = date.year
# month = date.month
# day = date.day
# last_day = False
# if day == calendar.monthrange(year, month)[1]:
# last_day = True
# if date.month == 12:
# year += 1
# month = 1
# if last_day:
# day = calendar.monthrange(year, month)[1]
# return date.replace(month=month, year=year, day=day)
def get_start_of_period(self, year, periods, month=None):
if month is None:
return year, 1
self.checked_monthes.setdefault(year, set()).add(month)
if month == 12:
prev_month = month - 1
prev_year = year
elif month == 1:
prev_month = 12
prev_year = year - 1
else:
prev_month = month - 1
prev_year = year
if prev_month in periods.get(prev_year, []):
return self.get_start_of_period(prev_year, periods, prev_month)
return year, month
def get_end_of_period(self, year, periods, month=None):
if month is None:
return year, 12
self.checked_monthes.setdefault(year, set()).add(month)
if month == 12:
next_month = 1
next_year = year + 1
elif month == 1:
next_month = month + 1
next_year = year
else:
next_month = month + 1
next_year = year
if next_month in periods.get(next_year, []):
return self.get_end_of_period(next_year, periods, next_month)
return year, month
def make_date_begin_sqs_params(self):
periods = []
self.checked_monthes = {}
cleaned_periods = self.get_date_begin_periods()
# print(cleaned_periods)
date_range = self.get_date_from_to_range()
if date_range is not None:
periods.append(date_range)
else:
for year, monthes in cleaned_periods.iteritems():
for month in monthes:
if month in self.checked_monthes.get(year, []):
continue
start_year, start_month = self.get_start_of_period(year, cleaned_periods, month)
end_year, end_month = self.get_end_of_period(year, cleaned_periods, month)
_first_day, _last_day = calendar.monthrange(end_year, end_month)
periods.append((
date.today().replace(day=1, month=start_month, year=start_year),
date.today().replace(day=_last_day, month=end_month, year=end_year)
))
if not monthes:
start_year, start_month = self.get_start_of_period(year, cleaned_periods)
end_year, end_month = self.get_end_of_period(year, cleaned_periods)
_first_day, _last_day = calendar.monthrange(end_year, end_month)
periods.append((
date.today().replace(day=1, month=start_month, year=start_year),
date.today().replace(day=_last_day, month=end_month, year=end_year)
))
params = None
for start, end in periods:
lookup = {'data_begin__range': (start, end)} if end else {'data_begin': start}
if params is None:
params = SQ(**lookup)
else:
params |= SQ(**lookup)
return params
def make_rating_sqs_params(self):
params = None
if self.cleaned_data.get('rating'):
for choice in filter(lambda x: x['field'] == 'expohit' and x['value'] in self.cleaned_data.get('rating'), rating_mapping):
params = SQ(**{choice['field']: choice['db_value']})
quality_label = [x['db_value'] for x in filter(lambda x: x['field'] != 'expohit' and x['value'] in self.cleaned_data.get('rating'), rating_mapping)]
if quality_label:
if params is None:
params = SQ(**{'quality_label__in': quality_label})
else:
params |= SQ(**{'quality_label__in': quality_label})
return params
def make_count_select(self, field):
selects = []
case = None
count = None
# print('looking {} {}'.format(field, self.lookup_kwargs))
for model in self.models:
_field, _model, direct, m2m = model._meta.get_field_by_name(field)
joins = []
where = []
group_by = ''
# ManyToManyField
if m2m and direct and isinstance(_field, ManyToManyField):
_field
format_kwargs = {
'm2m_db_table': _field.m2m_db_table(),
'm2m_column_name': _field.m2m_column_name(),
'db_table': model._meta.db_table,
'm2m_reverse_name': _field.m2m_reverse_name(),
'm2m_rel_to_table': _field.rel.to._meta.db_table,
'lang': self.lang,
}
select = \
'''SELECT COUNT(DISTINCT `{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\
.format(**format_kwargs)
where.append(
''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) AND `{m2m_rel_to_table}_translation`.`language_code` = '{lang}' '''.format(**format_kwargs)
)
# ForeignKey
elif not m2m and direct and isinstance(_field, ForeignKey):
_format_kwargs = {
'attname': _field.column,
'db_table': model._meta.db_table,
'rel_db_table': _field.rel.to._meta.db_table,
'lang': self.lang,
# 'attname': _field.related.field.get_attname(),
}
select = \
'''SELECT COUNT(DISTINCT `{db_table}`.`id`) FROM `{db_table}`'''\
.format(**_format_kwargs)
# '''SELECT COUNT(`{db_table}`.`id`) FROM `{rel_db_table}` LEFT OUTER JOIN `{db_table}` ON (`{rel_db_table}`.`id` = `{db_table}`.`{attname}`)'''\
where.append(
''' `{db_table}`.`{attname}` = `{rel_db_table}_translation`.`master_id` AND `{rel_db_table}_translation`.`language_code` = '{lang}' '''\
.format(**_format_kwargs))
group_by = ''' GROUP BY `{rel_db_table}_translation`.`master_id` '''.format(**_format_kwargs)
# mark selected items (for ordering)
# only for M2M and ForeignKey
if case is None and field in self._related_fields and direct\
and (isinstance(_field, ForeignKey) or isinstance(_field, ManyToManyField)):
values = self.cleaned_data[field]
case = \
''' CASE WHEN `{table}`.`master_id` in {ids} THEN 1 ELSE 0 END '''.format(
table=self.fields[field].queryset.model._meta.db_table,
ids=self.make_ids_in_sql_format(values))
# FILTER current by other values
_joins, _where = self.make_joins_from_selected(field, model)
joins.extend(_joins or [])
where.extend(_where or [])
where.append(self.make_default_where(db_table=model._meta.db_table))
self.make_date_begin_where(where, db_table=model._meta.db_table)
self.make_rating_where(where, db_table=model._meta.db_table)
selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
if len(selects) == 2:
count = ''' IFNULL(({0}), 0) + IFNULL(({1}), 0) '''.format(*selects)
elif len(selects) == 1:
count = selects[0]
d = {}
if case is not None:
d['selected'] = case
if count:
d['count'] = count
return d
### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый
## 1-й рабочий способ (без заполнения перевода)
# """
# SELECT
# `theme_theme`.`id`,
# (
# SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
# FROM `exposition_exposition_theme`
# WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)
# ) AS `e_count`,
# (
# SELECT COUNT(`conference_conference_theme`.`conference_id`)
# FROM `conference_conference_theme`
# WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`)
# ) AS `c_count`,
# `theme_theme_translation`.`name` as `name`
# FROM
# `theme_theme`
# GROUP BY `theme_theme`.`id`
# ORDER BY NULL
# """
## 2-й рабочий способ (с заполнением перевода)
## аттрибут перевода 'name' присвоен на 'name_t', чтобы не ругалось на неправильно заполненый перевод
# theme_sql = \
# """
# SELECT
# `theme_theme_translation`.`name` AS `name_t`,
# `theme_theme_translation`.`master_id` AS `id`,
# (
# SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
# FROM `exposition_exposition_theme`
# WHERE (`theme_theme_translation`.`master_id` = `exposition_exposition_theme`.`theme_id`)
# ) AS `e_count`,
# (
# SELECT COUNT(`conference_conference_theme`.`conference_id`)
# FROM `conference_conference_theme`
# WHERE (`theme_theme_translation`.`master_id` = `conference_conference_theme`.`theme_id`)
# ) AS `c_count`
# FROM
# `theme_theme_translation`
# WHERE (`theme_theme_translation`.`language_code` = 'ru')
# GROUP BY `theme_theme_translation`.`name`
# ORDER BY NULL
# """
# qs = Theme.objects.raw(theme_sql)
# _theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` '''
# theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) '''
# theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})'''
# _theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` '''
# theme_conf_count = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) '''
# theme_conf_count_selected = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})'''
# tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) '''
# tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) '''
# 3-й рабочий способ (с родным заполением перевода)
# в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8
# https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions
# from django.db.models.expressions import RawSQL
'''
SELECT
`city_city_translation`.`master_id`,
`<city_city_translation></city_city_translation>`.`name`,
COUNT(`conference_conference`.`id`) AS `count`
FROM
`city_city_translation`
LEFT OUTER JOIN
`city_city` ON (`city_city_translation`.`master_id` = `city_city`.`id`)
LEFT OUTER JOIN
`conference_conference` ON (`city_city`.`id` = `conference_conference`.`city_id`)
LEFT OUTER JOIN
`conference_conference_tag` ON (`conference_conference`.`id` = `conference_conference_tag`.`conference_id`)
WHERE
`conference_conference_tag`.`tag_id` IN (469 , 832, 366, 922)
GROUP BY `city_city_translation`.`language_code` , `city_city_translation`.`master_id`
ORDER BY NULL
'''
'''
SELECT
(SELECT
COUNT(`exposition_exposition`.`country_id`)
FROM
`exposition_exposition`
INNER JOIN
`exposition_exposition_theme` ON (`exposition_exposition`.`id` = `exposition_exposition_theme`.`exposition_id`)
where
`exposition_exposition_theme`.`theme_id` IN (1, 2, 3)
and `exposition_exposition`.`country_id` = `country_country_translation`.`master_id`
GROUP BY `country_country_translation`.`master_id`) AS `count`,
`country_country_translation`.`master_id`,
`country_country_translation`.`name`
FROM
`country_country_translation`
'''
'''
SELECT
(SELECT
COUNT(`exposition_exposition_theme`.`exposition_id`)
FROM
`exposition_exposition_theme`
INNER JOIN
`exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`)
INNER JOIN
`exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`)
WHERE
`exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922)
and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`,
CASE
WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1
ELSE 0
END as `selected`,
`theme_theme_translation`.`id`,
`theme_theme_translation`.`name`,
`theme_theme_translation`.`language_code`,
`theme_theme_translation`.`master_id`
FROM
`theme_theme_translation`
INNER JOIN
`theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`)
WHERE
(`theme_theme`.`types` = (`theme_theme`.`types` | 1)
OR `theme_theme`.`types` = (`theme_theme`.`types` | 2))
AND (`theme_theme_translation`.`language_code` = 'ru')
order BY `selected` DESC , `count` DESC
'''