# -*- coding: utf-8 -*-
from itertools import chain
from collections import namedtuple
try:
from collections import ChainMap
except ImportError:
from chainmap import ChainMap
from django import forms
from django.utils.translation import get_language, ugettext as _
from django.utils.encoding import smart_text, force_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.db.models import Count, Sum, Q, ForeignKey, ManyToManyField
from django.db import connection
from django.core.exceptions import ValidationError
from haystack.query import SearchQuerySet, RelatedSearchQuerySet, SQ
from functions.model_utils import EnumChoices
from exposition.models import Exposition
from conference.models import Conference
from theme.models import Theme, Tag
from country.models import Country
from city.models import City
from events.common import MEMBERS, VISITORS, PRICE
from events.common import members_mapping, visitors_mapping, price_mapping
class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField):
widget = forms.CheckboxSelectMultiple
def label_from_instance(self, obj):
if obj.get('count', None) is None:
return smart_text(obj.get('name'))
return u'{label} ({count})'.format(label=smart_text(obj.get('name')), count=obj.get('count'))
def prepare_value(self, value):
if isinstance(value, dict):
return value.get('pk')
return super(CountModelMultipleChoiceField, self).prepare_value(value)
def clean(self, value):
# return pks instead of QuerySet
if self.required and not value:
raise ValidationError(self.error_messages['required'])
elif not self.required and not value:
return self.queryset.none()
if not isinstance(value, (list, tuple)):
raise ValidationError(self.error_messages['list'])
key = self.to_field_name or 'pk'
for pk in value:
try:
self.queryset.filter(**{key: pk})
except ValueError:
raise ValidationError(self.error_messages['invalid_pk_value'] % pk)
pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True)))
for val in value:
if int(force_text(val)) not in pks:
raise ValidationError(self.error_messages['invalid_choice'] % val)
# Since this overrides the inherited ModelChoiceField.clean
# we run custom validators here
self.run_validators(value)
return pks
fields_mapping = {
'members': 'members_choice',
'visitors': 'visitors_choice',
'price': 'price_choice',
}
values_mapping = {
'members_choice': members_mapping,
'visitors_choice': visitors_mapping,
'price_choice': price_mapping,
}
class FilterForm(forms.Form):
TYPES = EnumChoices(
EXPO=(1, _(u'Выставки')),
CONF=(2, _(_(u'Конференции'))),
)
# MEMBERS = MEMBERS
# VISITORS = VISITORS
# PRICE = PRICE
model = forms.TypedMultipleChoiceField(
label=_(u'Тип события'), coerce=int,
choices=TYPES, required=False, widget=forms.CheckboxSelectMultiple())
theme = CountModelMultipleChoiceField(
label=_(u'Тематики'), required=False,
queryset=Theme.objects.language().values('pk', 'name'))
tag = CountModelMultipleChoiceField(
label=_(u'Теги'), required=False,
queryset=Tag.objects.language().values('pk', 'name'))
country = CountModelMultipleChoiceField(
label=_(u'Страны'), required=False,
queryset=Country.objects.language().values('pk', 'name'))
city = CountModelMultipleChoiceField(
label=_(u'Города'), required=False,
queryset=City.objects.language().values('pk', 'name'))
price = forms.TypedMultipleChoiceField(
label=_(u'Стоимость'), coerce=int,
choices=PRICE,
required=False, widget=forms.CheckboxSelectMultiple(),
help_text=_(u'За 1 м2 необорудованной площади'))
members = forms.TypedMultipleChoiceField(
label=_(u'Участники'), coerce=int,
choices=MEMBERS,
required=False, widget=forms.CheckboxSelectMultiple())
visitors = forms.TypedMultipleChoiceField(
label=_(u'Посетители'), coerce=int,
choices=VISITORS,
required=False, widget=forms.CheckboxSelectMultiple())
def __init__(self, *args, **kwargs):
super(FilterForm, self).__init__(*args, **kwargs)
self._is_valid = False
self._models = None
self._lookup_kwargs = None
self._lookup_args = None
self.lang = get_language()
# self.db_cursor = connection.cursor()
# self.selected_filter_data = {}
# self.fields['theme'].queryset = self.get_theme_choices()
# self.fields['tag'].queryset = self.get_tag_choices()
# self.fields['country'].queryset = self.get_country_choices()
# self.fields['city'].queryset = self.get_city_choices()
def is_valid(self):
# if getattr(self, '_is_valid', None) is None:
self._is_valid = super(FilterForm, self).is_valid()
# нам нужно сбрасывать сохраненный выбор моделей,
# т.к. после валидации нужно вернуть только выбранные
self._models = None
return self._is_valid
@property
def models(self):
if self._models is None and self._is_valid:
val = self.cleaned_data.get('model')
self._models = []
if self.TYPES.EXPO in val:
self._models.append(Exposition)
if self.TYPES.CONF in val:
self._models.append(Conference)
return self._models or [Exposition, Conference]
@property
def lookup_kwargs(self):
if self._lookup_kwargs is None:
d = self.cleaned_data
self._lookup_kwargs = {}
self._related_fields = []
self._local_fields = []
if d.get('theme'):
self._related_fields.append('theme')
self._lookup_kwargs['theme__in'] = d.get('theme')
if d.get('tag'):
self._related_fields.append('tag')
self._lookup_kwargs['tag__in'] = d.get('tag')
if d.get('country'):
self._lookup_kwargs['country_id__in'] = d.get('country')
if d.get('city'):
self._related_fields.append('city')
self._lookup_kwargs['city_id__in'] = d.get('city')
if d.get('members'):
self._local_fields.append('members')
self._lookup_kwargs['members__in'] = d.get('members')
if d.get('visitors'):
self._local_fields.append('visitors')
self._lookup_kwargs['visitors__in'] = d.get('visitors')
if d.get('price'):
self._local_fields.append('price')
self._lookup_kwargs['price__in'] = d.get('price')
return self._lookup_kwargs
# @property
# def lookup_args(self):
# if self._lookup_args is None:
# d = self.cleaned_data
# self._lookup_args = {}
# self._local_fields = []
# if d.get('members'):
# self._local_fields.append('city')
# return self._lookup_args
def filter(self, qs=None):
qs = qs or self.default_filter()
# lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values()))
return qs.filter(**self.lookup_kwargs)
def default_filter(self, load_all=True):
qs = RelatedSearchQuerySet().models(*self.models)
if load_all:
qs = qs.load_all()
for model in self.models:
qs = qs.load_all_queryset(model, model.enable.all())
return qs
def recalculate_choices(self):
print(self._is_valid)
if self._is_valid and self.lookup_kwargs:
for field in ['theme', 'tag', 'city', 'country']:
# field_qs = self.default_filter(load_all=False)
# if not field_lookup_kwargs:
# continue
# field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id)
# if field == 'theme':
# self.fields[field].queryset = self.get_theme_choices(field_qs)
values = ['pk', 'name']
order_by = []
select = self.make_count_select(field)
qs = self.fields[field].queryset
for key in select.iterkeys():
values.append(key)
if key == 'selected':
order_by.insert(0, '-' + key)
else:
order_by.append('-' + key)
qs = qs.extra(select=select)
self.fields[field].queryset = qs\
.values(*values)\
.order_by(*order_by)
print(self.fields[field].queryset.query)
for field in ['members', 'visitors', 'price']:
self.fields[field].choices = self.make_local_field_count(field) or self.fields[field].choices
for field in self.fields:
field = self.fields[field]
if hasattr(field, 'queryset'):
field.queryset = field.queryset[:15]
def make_ids_in_sql_format(self, values):
return tuple(values) if len(values) > 1 else '({})'.format(*values)
def make_joins_from_selected(self, field, model):
joins = []
where = []
# joins and where lookups in relations
for l_field in self._related_fields:
if l_field == field:
continue
_l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field)
values = self.cleaned_data[l_field]
if _m2m and _direct and isinstance(_l_field, ManyToManyField):
_format_kwargs = {
'm2m_db_table': _l_field.m2m_db_table(),
'm2m_column_name': _l_field.m2m_column_name(),
'db_table': model._meta.db_table,
'm2m_reverse_name': _l_field.m2m_reverse_name(),
'ids': self.make_ids_in_sql_format(values),
}
joins.append(
''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\
.format(**_format_kwargs))
where.append(
'''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\
.format(**_format_kwargs))
elif not _m2m and _direct and isinstance(_l_field, ForeignKey):
_format_kwargs = {
'db_table': model._meta.db_table,
'attname': _l_field.column,
'ids': self.make_ids_in_sql_format(values),
}
where.append(
'''`{db_table}`.`{attname}` IN {ids}'''\
.format(**_format_kwargs))
# joins and where lookups in local fields
for l_field in self._local_fields:
if l_field == field:
continue
val = self.cleaned_data[l_field]
_format_kwargs = {
'db_table': model._meta.db_table,
'attname': l_field,
'val': self.make_ids_in_sql_format(val),
}
where.append(
'''`{db_table}`.`{attname}` IN {val}'''\
.format(**_format_kwargs))
return joins, where
def make_local_field_count(self, field):
# if 'members' not in self.lookup_kwargs:
sql = ''
selects = []
l_field = fields_mapping.get(field, field)
_values_mapping = values_mapping.get(l_field).items()
for model in self.models:
joins = []
where = []
group_by = ''
format_kwargs = {
'db_table': model._meta.db_table,
'l_field': l_field,
'lang': self.lang,
}
cases = []
for key, val in _values_mapping:
cases.append(
''' sum(case when (`{db_table}`.`{l_field}` = {val}) then 1 else 0 end) as '{key}' '''\
.format(key=key, val=val.get('value'), **format_kwargs)
)
select = \
''' SELECT {cases} FROM `{db_table}_translation` INNER JOIN `{db_table}` ON (`{db_table}_translation`.`master_id` = `{db_table}`.`id`) '''\
.format(cases=', '.join(cases), **format_kwargs)
where.append(
''' `{db_table}_translation`.`language_code` = '{lang}' '''\
.format(**format_kwargs)
)
_joins, _where = self.make_joins_from_selected(field, model)
joins.extend(_joins)
where.extend(_where)
where.append(self.make_default_where(db_table=model._meta.db_table))
selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
sql = ' union '.join(selects)
choices = []
if sql:
with connection.cursor() as c:
c.execute(sql)
mapper = namedtuple('Result', [col[0] for col in c.description])
data = [mapper(*raw) for raw in c.fetchall()]
for key, val in _values_mapping:
count = sum([getattr(x, key, 0) for x in data])
choices.append((val.get('value'), val.get('label') + ' ({count})'.format(count=count)))
return choices
# cursor.execute()
#""" SELECT sum(case when (`exposition_exposition`.`members` < 200) then 1 else 0 end) as 'N200', sum(case when (`exposition_exposition`.`members` >= 200 AND `exposition_exposition`.`members` <= 500) then 2 else 0 end) as 'N200500', sum(case when (`exposition_exposition`.`members` >= 500 AND `exposition_exposition`.`members` <= 1000) then 3 else 0 end) as 'N5001000', sum(case when (`exposition_exposition`.`members` >= 1000 AND `exposition_exposition`.`members` <= 2000) then 4 else 0 end) as 'N10002000', sum(case when (`exposition_exposition`.`members` >= 2000) then 5 else 0 end) as 'N2000'FROM `exposition_exposition_translation` INNER JOIN `exposition_exposition` ON (`exposition_exposition_translation`.`master_id` = `exposition_exposition`.`id`) WHERE `exposition_exposition_translation`.`language_code` = 'ru'union SELECT sum(case when (`conference_conference`.`members` < 200) then 1 else 0 end) as 'N200', sum(case when (`conference_conference`.`members` >= 200 AND `conference_conference`.`members` <= 500) then 1 else 0 end) as 'N200500', sum(case when (`conference_conference`.`members` >= 500 AND `conference_conference`.`members` <= 1000) then 1 else 0 end) as 'N5001000', sum(case when (`conference_conference`.`members` >= 1000 AND `conference_conference`.`members` <= 2000) then 1 else 0 end) as 'N10002000', sum(case when (`conference_conference`.`members` >= 2000) then 1 else 0 end) as 'N2000'FROM `conference_conference_translation` INNER JOIN `conference_conference` ON (`conference_conference_translation`.`master_id` = `conference_conference`.`id`) WHERE `conference_conference_translation`.`language_code` = 'ru' """
def make_default_where(self, **kwargs):
return ''' `{db_table}`.`is_published` = True '''.format(**kwargs)
def make_count_select(self, field):
count_selects = []
case = None
count = None
print('looking {} {}'.format(field, self.lookup_kwargs))
for model in self.models:
_field, _model, direct, m2m = model._meta.get_field_by_name(field)
joins = []
where = []
group_by = ''
# ManyToManyField
if m2m and direct and isinstance(_field, ManyToManyField):
_field
format_kwargs = {
'm2m_db_table': _field.m2m_db_table(),
'm2m_column_name': _field.m2m_column_name(),
'db_table': model._meta.db_table,
'm2m_reverse_name': _field.m2m_reverse_name(),
'm2m_rel_to_table': _field.rel.to._meta.db_table,
'lang': self.lang,
}
select = \
'''SELECT COUNT(`{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\
.format(**format_kwargs)
where.append(
''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) AND `{m2m_rel_to_table}_translation`.`language_code` = '{lang}' '''.format(**format_kwargs)
)
# ForeignKey
elif not m2m and direct and isinstance(_field, ForeignKey):
_format_kwargs = {
'attname': _field.column,
'db_table': model._meta.db_table,
'rel_db_table': _field.rel.to._meta.db_table,
'lang': self.lang,
# 'attname': _field.related.field.get_attname(),
}
select = \
'''SELECT COUNT(`{db_table}`.`{attname}`) FROM `{db_table}`'''\
.format(**_format_kwargs)
# '''SELECT COUNT(`{db_table}`.`id`) FROM `{rel_db_table}` LEFT OUTER JOIN `{db_table}` ON (`{rel_db_table}`.`id` = `{db_table}`.`{attname}`)'''\
where.append(
''' `{db_table}`.`{attname}` = `{rel_db_table}_translation`.`master_id` AND `{rel_db_table}_translation`.`language_code` = '{lang}' '''\
.format(**_format_kwargs))
group_by = ''' GROUP BY `{rel_db_table}_translation`.`master_id` '''.format(**_format_kwargs)
# mark selected items (for ordering)
# only for M2M and ForeignKey
if case is None and field in self._related_fields and direct\
and (isinstance(_field, ForeignKey) or isinstance(_field, ManyToManyField)):
values = self.cleaned_data[field]
case = \
''' CASE WHEN `{table}`.`master_id` in {ids} THEN 1 ELSE 0 END '''.format(
table=self.fields[field].queryset.model._meta.db_table,
ids=self.make_ids_in_sql_format(values))
# FILTER current by other values
_joins, _where = self.make_joins_from_selected(field, model)
joins.extend(_joins)
where.extend(_where)
where.append(self.make_default_where(db_table=model._meta.db_table))
count_selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by)
if len(count_selects) == 2:
count = '({}) + ({})'.format(*count_selects)
elif len(count_selects) == 1:
count = count_selects[0]
d = {}
if case is not None:
d['selected'] = case
if count:
d['count'] = count
return d
### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый
## 1-й рабочий способ (без заполнения перевода)
# """
# SELECT
# `theme_theme`.`id`,
# (
# SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
# FROM `exposition_exposition_theme`
# WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)
# ) AS `e_count`,
# (
# SELECT COUNT(`conference_conference_theme`.`conference_id`)
# FROM `conference_conference_theme`
# WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`)
# ) AS `c_count`,
# `theme_theme_translation`.`name` as `name`
# FROM
# `theme_theme`
# GROUP BY `theme_theme`.`id`
# ORDER BY NULL
# """
## 2-й рабочий способ (с заполнением перевода)
## аттрибут перевода 'name' присвоен на 'name_t', чтобы не ругалось на неправильно заполненый перевод
# theme_sql = \
# """
# SELECT
# `theme_theme_translation`.`name` AS `name_t`,
# `theme_theme_translation`.`master_id` AS `id`,
# (
# SELECT COUNT(`exposition_exposition_theme`.`exposition_id`)
# FROM `exposition_exposition_theme`
# WHERE (`theme_theme_translation`.`master_id` = `exposition_exposition_theme`.`theme_id`)
# ) AS `e_count`,
# (
# SELECT COUNT(`conference_conference_theme`.`conference_id`)
# FROM `conference_conference_theme`
# WHERE (`theme_theme_translation`.`master_id` = `conference_conference_theme`.`theme_id`)
# ) AS `c_count`
# FROM
# `theme_theme_translation`
# WHERE (`theme_theme_translation`.`language_code` = 'ru')
# GROUP BY `theme_theme_translation`.`name`
# ORDER BY NULL
# """
# qs = Theme.objects.raw(theme_sql)
# _theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` '''
# theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) '''
# theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})'''
# _theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` '''
# theme_conf_count = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) '''
# theme_conf_count_selected = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})'''
# tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) '''
# tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) '''
# 3-й рабочий способ (с родным заполением перевода)
# в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8
# https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions
# from django.db.models.expressions import RawSQL
'''
SELECT
`city_city_translation`.`master_id`,
``.`name`,
COUNT(`conference_conference`.`id`) AS `count`
FROM
`city_city_translation`
LEFT OUTER JOIN
`city_city` ON (`city_city_translation`.`master_id` = `city_city`.`id`)
LEFT OUTER JOIN
`conference_conference` ON (`city_city`.`id` = `conference_conference`.`city_id`)
LEFT OUTER JOIN
`conference_conference_tag` ON (`conference_conference`.`id` = `conference_conference_tag`.`conference_id`)
WHERE
`conference_conference_tag`.`tag_id` IN (469 , 832, 366, 922)
GROUP BY `city_city_translation`.`language_code` , `city_city_translation`.`master_id`
ORDER BY NULL
'''
'''
SELECT
(SELECT
COUNT(`exposition_exposition`.`country_id`)
FROM
`exposition_exposition`
INNER JOIN
`exposition_exposition_theme` ON (`exposition_exposition`.`id` = `exposition_exposition_theme`.`exposition_id`)
where
`exposition_exposition_theme`.`theme_id` IN (1, 2, 3)
and `exposition_exposition`.`country_id` = `country_country_translation`.`master_id`
GROUP BY `country_country_translation`.`master_id`) AS `count`,
`country_country_translation`.`master_id`,
`country_country_translation`.`name`
FROM
`country_country_translation`
'''
'''
SELECT
(SELECT
COUNT(`exposition_exposition_theme`.`exposition_id`)
FROM
`exposition_exposition_theme`
INNER JOIN
`exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`)
INNER JOIN
`exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`)
WHERE
`exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922)
and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`,
CASE
WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1
ELSE 0
END as `selected`,
`theme_theme_translation`.`id`,
`theme_theme_translation`.`name`,
`theme_theme_translation`.`language_code`,
`theme_theme_translation`.`master_id`
FROM
`theme_theme_translation`
INNER JOIN
`theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`)
WHERE
(`theme_theme`.`types` = (`theme_theme`.`types` | 1)
OR `theme_theme`.`types` = (`theme_theme`.`types` | 2))
AND (`theme_theme_translation`.`language_code` = 'ru')
order BY `selected` DESC , `count` DESC
'''