# -*- coding: utf-8 -*- from itertools import chain try: from collections import ChainMap except ImportError: from chainmap import ChainMap from django import forms from django.utils.translation import get_language, ugettext as _ from django.utils.encoding import smart_text, force_text from django.utils.html import format_html from django.utils.safestring import mark_safe from django.db.models import Count, Sum, Q, ForeignKey, ManyToManyField from django.core.exceptions import ValidationError from haystack.query import SearchQuerySet, RelatedSearchQuerySet from functions.model_utils import EnumChoices from exposition.models import Exposition from conference.models import Conference from theme.models import Theme, Tag from country.models import Country from city.models import City # class CountModelChoiceField(forms.ModelChoiceField): # def prepare_value(self, value): # if hasattr(value, '_meta'): # if self.to_field_name: # return value.serializable_value(self.to_field_name) # else: # return value.pk # elif isinstance(value, dict): # return value.get('pk') # return super(CountModelChoiceField, self).prepare_value(value) class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField): widget = forms.CheckboxSelectMultiple def label_from_instance(self, obj): if obj.get('count', None) is None: return smart_text(obj.get('name')) return u'{label} ({count})'.format(label=smart_text(obj.get('name')), count=obj.get('count')) def prepare_value(self, value): if isinstance(value, dict): return value.get('pk') return super(CountModelMultipleChoiceField, self).prepare_value(value) def clean(self, value): # return pks instead of QuerySet if self.required and not value: raise ValidationError(self.error_messages['required']) elif not self.required and not value: return self.queryset.none() if not isinstance(value, (list, tuple)): raise ValidationError(self.error_messages['list']) key = self.to_field_name or 'pk' for pk in value: try: self.queryset.filter(**{key: pk}) except ValueError: raise ValidationError(self.error_messages['invalid_pk_value'] % pk) pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True))) for val in value: if int(force_text(val)) not in pks: raise ValidationError(self.error_messages['invalid_choice'] % val) # Since this overrides the inherited ModelChoiceField.clean # we run custom validators here self.run_validators(value) return pks ### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый ## 1-й рабочий способ (без заполнения перевода) # """ # SELECT # `theme_theme`.`id`, # ( # SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) # FROM `exposition_exposition_theme` # WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) # ) AS `e_count`, # ( # SELECT COUNT(`conference_conference_theme`.`conference_id`) # FROM `conference_conference_theme` # WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) # ) AS `c_count`, # `theme_theme_translation`.`name` as `name` # FROM # `theme_theme` # GROUP BY `theme_theme`.`id` # ORDER BY NULL # """ ## 2-й рабочий способ (с заполнением перевода) ## аттрибут перевода 'name' присвоен на 'name_t', чтобы не ругалось на неправильно заполненый перевод # theme_sql = \ # """ # SELECT # `theme_theme_translation`.`name` AS `name_t`, # `theme_theme_translation`.`master_id` AS `id`, # ( # SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) # FROM `exposition_exposition_theme` # WHERE (`theme_theme_translation`.`master_id` = `exposition_exposition_theme`.`theme_id`) # ) AS `e_count`, # ( # SELECT COUNT(`conference_conference_theme`.`conference_id`) # FROM `conference_conference_theme` # WHERE (`theme_theme_translation`.`master_id` = `conference_conference_theme`.`theme_id`) # ) AS `c_count` # FROM # `theme_theme_translation` # WHERE (`theme_theme_translation`.`language_code` = 'ru') # GROUP BY `theme_theme_translation`.`name` # ORDER BY NULL # """ # qs = Theme.objects.raw(theme_sql) _theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` ''' theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) ''' theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})''' _theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` ''' theme_conf_count = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) ''' theme_conf_count_selected = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})''' tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) ''' tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) ''' class FilterForm(forms.Form): TYPES = EnumChoices( EXPO=(1, _(u'Выставки')), CONF=(2, _(_(u'Конференции'))), ) model = forms.TypedMultipleChoiceField(label=_(u'Тип события'), coerce=int, choices=TYPES, required=False, widget=forms.CheckboxSelectMultiple()) theme = CountModelMultipleChoiceField( label=_(u'Тематики'), required=False, queryset=Theme.objects.language().values('pk', 'name')) tag = CountModelMultipleChoiceField( label=_(u'Теги'), required=False, queryset=Tag.objects.language().values('pk', 'name')) country = CountModelMultipleChoiceField( label=_(u'Страны'), required=False, queryset=Country.objects.language().values('pk', 'name')) city = CountModelMultipleChoiceField( label=_(u'Города'), required=False, queryset=City.objects.language().values('pk', 'name')) def __init__(self, *args, **kwargs): super(FilterForm, self).__init__(*args, **kwargs) self._is_valid = False self._models = None self._lookup_kwargs = None self.lang = get_language() # self.selected_filter_data = {} # self.fields['theme'].queryset = self.get_theme_choices() # self.fields['tag'].queryset = self.get_tag_choices() # self.fields['country'].queryset = self.get_country_choices() # self.fields['city'].queryset = self.get_city_choices() def is_valid(self): # if getattr(self, '_is_valid', None) is None: self._is_valid = super(FilterForm, self).is_valid() # нам нужно сбрасывать сохраненные модели, # т.к. после валидации нужно вернуть только выбранные self._models = None return self._is_valid @property def models(self): if self._models is None and self._is_valid: val = self.cleaned_data.get('model') self._models = [] if self.TYPES.EXPO in val: self._models.append(Exposition) if self.TYPES.CONF in val: self._models.append(Conference) return self._models or [Exposition, Conference] def get_theme_choices(self, ids): # 3-й рабочий способ (с родным заполением перевода) # в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8 # https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions # from django.db.models.expressions import RawSQL conf = [] expo = [] for _type, pk in ids: if _type == 'conference': conf.append(pk) else: expo.append(int(pk)) if getattr(self, '_theme_choices', None) is None: if Exposition in self.models and Conference in self.models: count_query = '({q1}) + ({q2})'.format( q1=theme_expo_count if not expo else theme_expo_count_selected.format(ids=tuple(expo)), q2=theme_conf_count if not conf else theme_conf_count_selected.format(ids=tuple(conf)), ) filter_types = Q(types=Theme.types.conference) | Q(types=Theme.types.exposition) elif Exposition in self.models: count_query = theme_expo_count filter_types = Q(types=Theme.types.exposition) elif Conference in self.models: count_query = theme_conf_count filter_types = Q(types=Theme.types.conference) self._theme_choices = Theme.objects.language()\ .filter(filter_types)\ .extra(select={'count': count_query})\ # .order_by('-count', '-name') return self._theme_choices def get_tag_choices(self): tag_select = {} if getattr(self, '_tag_choices', None) is None: if Exposition in self.models and Conference in self.models: count_query = '({q1}) + ({q2})'.format( q1=tag_expo_count, q2=tag_conf_count) elif Exposition in self.models: count_query = tag_expo_count elif Conference in self.models: count_query = tag_conf_count self._tag_choices = Tag.objects.language()\ .extra(select={'count': count_query})\ # .order_by('-count', '-name') return self._tag_choices @property def lookup_kwargs(self): if self._lookup_kwargs is None: d = self.cleaned_data self._lookup_kwargs = {} if d.get('theme'): self._lookup_kwargs['theme'] = { 'theme__in': d.get('theme') } if d.get('tag'): self._lookup_kwargs['tag'] = { 'tag__in': d.get('tag'), } if d.get('country'): self._lookup_kwargs['country'] = { 'country_id__in': d.get('country'), } if d.get('city'): self._lookup_kwargs['city'] = { 'city_id__in': d.get('city'), } return self._lookup_kwargs def filter(self, lookup_kwargs=None, qs=None): qs = qs or self.default_filter() lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values())) return qs.filter(**lookup_kwargs) def default_filter(self, load_all=True): qs = RelatedSearchQuerySet().models(*self.models) if load_all: qs = qs.load_all() for model in self.models: qs = qs.load_all_queryset(model, model.enable.all()) return qs def recalculate_choices(self): print(self._is_valid) if self._is_valid: for field in ['theme', 'tag', 'city', 'country']: field_qs = self.default_filter(load_all=False) field_lookup_kwargs = self.lookup_kwargs.copy() if field in field_lookup_kwargs.keys(): del field_lookup_kwargs[field] # if not field_lookup_kwargs: # continue # field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id) # if field == 'theme': # self.fields[field].queryset = self.get_theme_choices(field_qs) values = ['pk', 'name'] order_by = [] select = self.make_count_select(field, field_lookup_kwargs) qs = self.fields[field].queryset for key in select.iterkeys(): values.append(key) if key == 'selected': order_by.insert(0, '-' + key) else: order_by.append('-' + key) qs = qs.extra(select=select) self.fields[field].queryset = qs\ .values(*values)\ .order_by(*order_by) print(self.fields[field].queryset.query) for field in self.fields: field = self.fields[field] if hasattr(field, 'queryset'): field.queryset = field.queryset[:15] def make_ids_in_sql_format(self, values): return tuple(values) if len(values) > 1 else '({})'.format(*values) def make_count_select(self, field, lookup_kwargs): count_selects = [] case = None count = None print('looking {} {}'.format(field, lookup_kwargs)) for model in self.models: _field, _model, direct, m2m = model._meta.get_field_by_name(field) joins = [] where = [] group_by = '' # ManyToManyField if m2m and direct and isinstance(_field, ManyToManyField): _field format_kwargs = { 'm2m_db_table': _field.m2m_db_table(), 'm2m_column_name': _field.m2m_column_name(), 'db_table': model._meta.db_table, 'm2m_reverse_name': _field.m2m_reverse_name(), 'm2m_rel_to_table': _field.rel.to._meta.db_table, 'lang': self.lang, } select = \ '''SELECT COUNT(`{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\ .format(**format_kwargs) where.append( ''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) AND `{m2m_rel_to_table}_translation`.`language_code` = '{lang}' AND `{db_table}`.`is_published` = True '''.format(**format_kwargs) ) # ForeignKey elif not m2m and direct and isinstance(_field, ForeignKey): _format_kwargs = { 'attname': _field.column, 'db_table': model._meta.db_table, 'rel_db_table': _field.rel.to._meta.db_table, 'lang': self.lang, # 'attname': _field.related.field.get_attname(), } select = \ '''SELECT COUNT(`{db_table}`.`{attname}`) FROM `{db_table}`'''\ .format(**_format_kwargs) # '''SELECT COUNT(`{db_table}`.`id`) FROM `{rel_db_table}` LEFT OUTER JOIN `{db_table}` ON (`{rel_db_table}`.`id` = `{db_table}`.`{attname}`)'''\ where.append( ''' `{db_table}`.`{attname}` = `{rel_db_table}_translation`.`master_id` AND `{rel_db_table}_translation`.`language_code` = '{lang}' AND `{db_table}`.`is_published` = True '''\ .format(**_format_kwargs)) group_by = ''' GROUP BY `{rel_db_table}_translation`.`master_id` '''.format(**_format_kwargs) # mark selected items (for ordering) if case is None and field in self.lookup_kwargs.keys(): values = self.lookup_kwargs[field].values()[0] case = \ ''' CASE WHEN `{table}`.`master_id` in {ids} THEN 1 ELSE 0 END '''.format( table=self.fields[field].queryset.model._meta.db_table, ids=self.make_ids_in_sql_format(values)) # joins and where lookups for l_field, lookups in lookup_kwargs.iteritems(): _l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field) values = lookups.values()[0] if _m2m and _direct and isinstance(_l_field, ManyToManyField): _format_kwargs = { 'm2m_db_table': _l_field.m2m_db_table(), 'm2m_column_name': _l_field.m2m_column_name(), 'db_table': model._meta.db_table, 'm2m_reverse_name': _l_field.m2m_reverse_name(), 'ids': self.make_ids_in_sql_format(values), } joins.append( ''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\ .format(**_format_kwargs)) where.append( '''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\ .format(**_format_kwargs)) elif not m2m and _direct and isinstance(_l_field, ForeignKey): _format_kwargs = { 'db_table': model._meta.db_table, 'attname': _l_field.column, 'ids': self.make_ids_in_sql_format(values), } where.append( '''`{db_table}`.`{attname}` IN {ids}'''\ .format(**_format_kwargs)) count_selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where) + group_by) if len(count_selects) == 2: count = '({}) + ({})'.format(*count_selects) elif len(count_selects) == 1: count = count_selects[0] d = {} if case is not None: d['selected'] = case if count: d['count'] = count return d ''' SELECT `city_city_translation`.`master_id`, ``.`name`, COUNT(`conference_conference`.`id`) AS `count` FROM `city_city_translation` LEFT OUTER JOIN `city_city` ON (`city_city_translation`.`master_id` = `city_city`.`id`) LEFT OUTER JOIN `conference_conference` ON (`city_city`.`id` = `conference_conference`.`city_id`) LEFT OUTER JOIN `conference_conference_tag` ON (`conference_conference`.`id` = `conference_conference_tag`.`conference_id`) WHERE `conference_conference_tag`.`tag_id` IN (469 , 832, 366, 922) GROUP BY `city_city_translation`.`language_code` , `city_city_translation`.`master_id` ORDER BY NULL ''' ''' SELECT (SELECT COUNT(`exposition_exposition`.`country_id`) FROM `exposition_exposition` INNER JOIN `exposition_exposition_theme` ON (`exposition_exposition`.`id` = `exposition_exposition_theme`.`exposition_id`) where `exposition_exposition_theme`.`theme_id` IN (1, 2, 3) and `exposition_exposition`.`country_id` = `country_country_translation`.`master_id` GROUP BY `country_country_translation`.`master_id`) AS `count`, `country_country_translation`.`master_id`, `country_country_translation`.`name` FROM `country_country_translation` ''' ''' SELECT (SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` INNER JOIN `exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`) INNER JOIN `exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`) WHERE `exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922) and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`, CASE WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1 ELSE 0 END as `selected`, `theme_theme_translation`.`id`, `theme_theme_translation`.`name`, `theme_theme_translation`.`language_code`, `theme_theme_translation`.`master_id` FROM `theme_theme_translation` INNER JOIN `theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`) WHERE (`theme_theme`.`types` = (`theme_theme`.`types` | 1) OR `theme_theme`.`types` = (`theme_theme`.`types` | 2)) AND (`theme_theme_translation`.`language_code` = 'ru') order BY `selected` DESC , `count` DESC '''