From eb032d5257bd0f550bb604df36d586012465e434 Mon Sep 17 00:00:00 2001 From: Alexander Burdeiny Date: Mon, 1 Aug 2016 00:40:26 +0300 Subject: [PATCH] custom sql making progress --- events/forms.py | 278 ++++++++++++++++++++++++++++++++++++++++-------- events/views.py | 1 + 2 files changed, 234 insertions(+), 45 deletions(-) diff --git a/events/forms.py b/events/forms.py index 51e4d0cd..84a3f8bb 100644 --- a/events/forms.py +++ b/events/forms.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- from itertools import chain +try: + from collections import ChainMap +except ImportError: + from chainmap import ChainMap from django import forms from django.utils.translation import get_language, ugettext as _ @@ -8,6 +12,7 @@ from django.utils.encoding import smart_text, force_text from django.utils.html import format_html from django.utils.safestring import mark_safe from django.db.models import Count, Sum, Q +from django.core.exceptions import ValidationError from haystack.query import SearchQuerySet, RelatedSearchQuerySet @@ -15,12 +20,56 @@ from functions.model_utils import EnumChoices from exposition.models import Exposition from conference.models import Conference from theme.models import Theme, Tag +from country.models import Country +from city.models import City + + +# class CountModelChoiceField(forms.ModelChoiceField): +# def prepare_value(self, value): +# if hasattr(value, '_meta'): +# if self.to_field_name: +# return value.serializable_value(self.to_field_name) +# else: +# return value.pk +# elif isinstance(value, dict): +# return value.get('pk') +# return super(CountModelChoiceField, self).prepare_value(value) class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField): + widget = forms.CheckboxSelectMultiple def label_from_instance(self, obj): - return u'{label} ({count})'.format(label=smart_text(obj.name), count=obj.count) + if obj.get('count', None) is None: + return smart_text(obj.get('name')) + return u'{label} ({count})'.format(label=smart_text(obj.get('name')), count=obj.get('count')) + def prepare_value(self, value): + if isinstance(value, dict): + return value.get('pk') + return super(CountModelMultipleChoiceField, self).prepare_value(value) + + def clean(self, value): + # return pks instead of QuerySet + if self.required and not value: + raise ValidationError(self.error_messages['required']) + elif not self.required and not value: + return self.queryset.none() + if not isinstance(value, (list, tuple)): + raise ValidationError(self.error_messages['list']) + key = self.to_field_name or 'pk' + for pk in value: + try: + self.queryset.filter(**{key: pk}) + except ValueError: + raise ValidationError(self.error_messages['invalid_pk_value'] % pk) + pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True))) + for val in value: + if int(force_text(val)) not in pks: + raise ValidationError(self.error_messages['invalid_choice'] % val) + # Since this overrides the inherited ModelChoiceField.clean + # we run custom validators here + self.run_validators(value) + return pks ### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый @@ -69,11 +118,15 @@ class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField): # ORDER BY NULL # """ # qs = Theme.objects.raw(theme_sql) -extra_theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) ''' -extra_theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) ''' +_theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` ''' +theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) ''' +theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})''' +_theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` ''' +theme_conf_count = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) ''' +theme_conf_count_selected = _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})''' -extra_tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) ''' -extra_tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) ''' +tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) ''' +tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) ''' class FilterForm(forms.Form): TYPES = EnumChoices( @@ -82,19 +135,29 @@ class FilterForm(forms.Form): ) model = forms.TypedMultipleChoiceField(label=_(u'Тип события'), coerce=int, choices=TYPES, required=False, widget=forms.CheckboxSelectMultiple()) - theme = CountModelMultipleChoiceField(label=_(u'Тематики'), - queryset=Theme.objects.none(), - required=False, widget=forms.CheckboxSelectMultiple()) - tag = CountModelMultipleChoiceField(label=_(u'Теги'), - queryset=Tag.objects.none(), - required=False, widget=forms.CheckboxSelectMultiple()) + theme = CountModelMultipleChoiceField( + label=_(u'Тематики'), required=False, + queryset=Theme.objects.language().values('pk', 'name')) + tag = CountModelMultipleChoiceField( + label=_(u'Теги'), required=False, + queryset=Tag.objects.language().values('pk', 'name')) + country = CountModelMultipleChoiceField( + label=_(u'Страны'), required=False, + queryset=Country.objects.language().values('pk', 'name')) + city = CountModelMultipleChoiceField( + label=_(u'Города'), required=False, + queryset=City.objects.language().values('pk', 'name')) def __init__(self, *args, **kwargs): super(FilterForm, self).__init__(*args, **kwargs) self._is_valid = False self._models = None - self.fields['theme'].queryset = self.get_theme_choices() - self.fields['tag'].queryset = self.get_tag_choices() + self._lookup_kwargs = None + # self.selected_filter_data = {} + # self.fields['theme'].queryset = self.get_theme_choices() + # self.fields['tag'].queryset = self.get_tag_choices() + # self.fields['country'].queryset = self.get_country_choices() + # self.fields['city'].queryset = self.get_city_choices() def is_valid(self): # if getattr(self, '_is_valid', None) is None: @@ -116,67 +179,192 @@ class FilterForm(forms.Form): self._models.append(Conference) return self._models or [Exposition, Conference] - def get_theme_choices(self): + def get_theme_choices(self, ids): # 3-й рабочий способ (с родным заполением перевода) # в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8 # https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions # from django.db.models.expressions import RawSQL + conf = [] + expo = [] + for _type, pk in ids: + if _type == 'conference': + conf.append(pk) + else: + expo.append(int(pk)) if getattr(self, '_theme_choices', None) is None: if Exposition in self.models and Conference in self.models: count_query = '({q1}) + ({q2})'.format( - q1=extra_theme_expo_count, - q2=extra_theme_conf_count) + q1=theme_expo_count if not expo else theme_expo_count_selected.format(ids=tuple(expo)), + q2=theme_conf_count if not conf else theme_conf_count_selected.format(ids=tuple(conf)), + ) filter_types = Q(types=Theme.types.conference) | Q(types=Theme.types.exposition) elif Exposition in self.models: - count_query = extra_theme_expo_count + count_query = theme_expo_count filter_types = Q(types=Theme.types.exposition) elif Conference in self.models: - count_query = extra_theme_conf_count + count_query = theme_conf_count filter_types = Q(types=Theme.types.conference) self._theme_choices = Theme.objects.language()\ .filter(filter_types)\ .extra(select={'count': count_query})\ - .order_by('-count', '-name') + # .order_by('-count', '-name') return self._theme_choices def get_tag_choices(self): - extra_tag_select = {} + tag_select = {} if getattr(self, '_tag_choices', None) is None: if Exposition in self.models and Conference in self.models: count_query = '({q1}) + ({q2})'.format( - q1=extra_tag_expo_count, - q2=extra_tag_conf_count) + q1=tag_expo_count, + q2=tag_conf_count) elif Exposition in self.models: - count_query = extra_tag_expo_count + count_query = tag_expo_count elif Conference in self.models: - count_query = extra_tag_conf_count + count_query = tag_conf_count self._tag_choices = Tag.objects.language()\ .extra(select={'count': count_query})\ - .order_by('-count', '-name') + # .order_by('-count', '-name') return self._tag_choices - def filter(self): - qs = self.default_filter() - d = self.cleaned_data - if d.get('theme'): - qs = qs.filter(theme__in=d.get('theme')) - if d.get('tag'): - qs = qs.filter(tag__in=d.get('tag')) + @property + def lookup_kwargs(self): + if self._lookup_kwargs is None: + d = self.cleaned_data + self._lookup_kwargs = {} + if d.get('theme'): + self._lookup_kwargs['theme'] = { + 'theme__in': d.get('theme') + } + if d.get('tag'): + self._lookup_kwargs['tag'] = { + 'tag__in': d.get('tag'), + } + if d.get('country'): + self._lookup_kwargs['country'] = { + 'country_id__in': d.get('country'), + } + if d.get('city'): + self._lookup_kwargs['country'] = { + 'city_id__in': d.get('city'), + } + return self._lookup_kwargs + + def filter(self, lookup_kwargs=None, qs=None): + qs = qs or self.default_filter() + lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values())) + return qs.filter(**lookup_kwargs) + + def default_filter(self, load_all=True): + qs = RelatedSearchQuerySet().models(*self.models) + if load_all: + qs = qs.load_all() + for model in self.models: + qs = qs.load_all_queryset(model, model.enable.all()) return qs - def default_filter(self): - qs = RelatedSearchQuerySet().models(*self.models).load_all() + def recalculate_choices(self): + print(self._is_valid) + if self._is_valid: + for field, val in self.lookup_kwargs.iteritems(): + field_qs = self.default_filter(load_all=False) + field_lookup_kwargs = self.lookup_kwargs.copy() + del field_lookup_kwargs[field] + # if not field_lookup_kwargs: + # continue + # field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id) + # if field == 'theme': + # self.fields[field].queryset = self.get_theme_choices(field_qs) + self.fields[field].queryset = self.fields[field].queryset.extra( + select=self.make_count_select(field, field_lookup_kwargs)).values('pk', 'name', 'count') + print(self.fields[field].queryset.query) + # self.make_count_select(field, field_lookup_kwargs) + # print(field_qs) + + for field in self.fields: + field = self.fields[field] + if hasattr(field, 'queryset'): + field.queryset = field.queryset[:15] + + def make_count_select(self, field, lookup_kwargs): + count_selects = [] + print('looking {} {}'.format(field, lookup_kwargs)) for model in self.models: - qs = qs.load_all_queryset(model, - model.enable.all() - # не реализовано в hvad <_< - # .only( - # 'canceled', 'name', 'main_title', 'expohit', 'logo', - # 'quality_label', 'services', 'visitors', 'members', - # 'data_begin', 'data_end', 'country__url', 'country__name', - # 'city__url', 'place__name' - # ) - ) - return qs + _field, _model, direct, m2m = model._meta.get_field_by_name(field) + if m2m: + _field + format_kwargs = { + 'm2m_db_table': _field.m2m_db_table(), + 'm2m_column_name': _field.m2m_column_name(), + 'db_table': model._meta.db_table, + 'm2m_reverse_name': _field.m2m_reverse_name(), + 'm2m_rel_to_table': _field.rel.to._meta.db_table, + } + + select = \ + '''SELECT COUNT(`{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\ + .format(**format_kwargs) + joins = [] + where = [ + ''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) '''.format(**format_kwargs) + ] + + for l_field, lookups in lookup_kwargs.iteritems(): + _l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field) + values = lookups.values()[0] + _format_kwargs = { + 'm2m_db_table': _l_field.m2m_db_table(), + 'm2m_column_name': _l_field.m2m_column_name(), + 'db_table': model._meta.db_table, + 'm2m_reverse_name': _l_field.m2m_reverse_name(), + 'ids': tuple(values) if len(values) > 1 else '({})'.format(*values), + } + joins.append( + ''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\ + .format(**_format_kwargs)) + where.append( + '''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\ + .format(**_format_kwargs)) + count_selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where)) + + # case selected + # todo + if len(count_selects) == 2: + count = '({}) + ({})'.format(*count_selects) + elif len(count_selects) == 1: + count = count_selects[0] + return {'count': count} + +''' +SELECT + (SELECT + COUNT(`exposition_exposition_theme`.`exposition_id`) + FROM + `exposition_exposition_theme` + INNER JOIN + `exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`) + INNER JOIN + `exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`) + WHERE + `exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922) + and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`, + CASE + WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1 + ELSE 0 + END as `selected`, + `theme_theme_translation`.`id`, + `theme_theme_translation`.`name`, + `theme_theme_translation`.`language_code`, + `theme_theme_translation`.`master_id` +FROM + `theme_theme_translation` + INNER JOIN + `theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`) +WHERE + (`theme_theme`.`types` = (`theme_theme`.`types` | 1) + OR `theme_theme`.`types` = (`theme_theme`.`types` | 2)) + AND (`theme_theme_translation`.`language_code` = 'ru') +order BY `selected` DESC , `count` DESC + +''' diff --git a/events/views.py b/events/views.py index 7daff5ff..e5769f2f 100644 --- a/events/views.py +++ b/events/views.py @@ -28,6 +28,7 @@ class FilterListView(ContextMixin, FormMixin, ListView): qs = self.form.filter() else: qs = self.form.default_filter() + self.form.recalculate_choices() return qs def get(self, request, *args, **kwargs):