custom sql making progress

events/forms.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 
 from itertools import chain
+try:
+    from collections import ChainMap
+except ImportError:
+    from chainmap import ChainMap
 
 from django import forms
 from django.utils.translation import get_language, ugettext as _
@@ -8,6 +12,7 @@ from django.utils.encoding import smart_text, force_text
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.db.models import Count, Sum, Q
+from django.core.exceptions import ValidationError
 
 from haystack.query import SearchQuerySet, RelatedSearchQuerySet
 
@@ -15,12 +20,56 @@ from functions.model_utils import EnumChoices
 from exposition.models import Exposition
 from conference.models import Conference
 from theme.models import Theme, Tag
+from country.models import Country
+from city.models import City
+
+
+# class CountModelChoiceField(forms.ModelChoiceField):
+#     def prepare_value(self, value):
+#         if hasattr(value, '_meta'):
+#             if self.to_field_name:
+#                 return value.serializable_value(self.to_field_name)
+#             else:
+#                 return value.pk
+#         elif isinstance(value, dict):
+#             return value.get('pk')
+#         return super(CountModelChoiceField, self).prepare_value(value)
 
 
 class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField):
+    widget = forms.CheckboxSelectMultiple
     def label_from_instance(self, obj):
-        return u'{label} ({count})'.format(label=smart_text(obj.name), count=obj.count)
+        if obj.get('count', None) is None:
+            return smart_text(obj.get('name'))
+        return u'{label} ({count})'.format(label=smart_text(obj.get('name')), count=obj.get('count'))
 
+    def prepare_value(self, value):
+        if isinstance(value, dict):
+            return value.get('pk')
+        return super(CountModelMultipleChoiceField, self).prepare_value(value)
+
+    def clean(self, value):
+        # return pks instead of QuerySet
+        if self.required and not value:
+            raise ValidationError(self.error_messages['required'])
+        elif not self.required and not value:
+            return self.queryset.none()
+        if not isinstance(value, (list, tuple)):
+            raise ValidationError(self.error_messages['list'])
+        key = self.to_field_name or 'pk'
+        for pk in value:
+            try:
+                self.queryset.filter(**{key: pk})
+            except ValueError:
+                raise ValidationError(self.error_messages['invalid_pk_value'] % pk)
+        pks = set((int(x) for x in self.queryset.filter(**{'%s__in' % key: value}).values_list('pk', flat=True)))
+        for val in value:
+            if int(force_text(val)) not in pks:
+                raise ValidationError(self.error_messages['invalid_choice'] % val)
+        # Since this overrides the inherited ModelChoiceField.clean
+        # we run custom validators here
+        self.run_validators(value)
+        return pks
 
 
 ### Делаем выборку по темам, сразу заполняя перевод и кол-во событиый
@@ -69,11 +118,15 @@ class CountModelMultipleChoiceField(forms.ModelMultipleChoiceField):
 # ORDER BY NULL
 # """
 # qs = Theme.objects.raw(theme_sql)
-extra_theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) '''
+_theme_expo_count = '''SELECT COUNT(`exposition_exposition_theme`.`exposition_id`) FROM `exposition_exposition_theme` '''
-extra_theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) '''
+theme_expo_count = _theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`) '''
+theme_expo_count_selected = theme_expo_count + ''' WHERE (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id` AND `exposition_exposition_theme`.`exposition_id` in {ids})'''
+_theme_conf_count = '''SELECT COUNT(`conference_conference_theme`.`conference_id`) FROM `conference_conference_theme` '''
+theme_conf_count =  _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id`) '''
+theme_conf_count_selected =  _theme_conf_count + ''' WHERE (`theme_theme`.`id` = `conference_conference_theme`.`theme_id` AND `conference_conference_theme`.`conference_id` in {ids})'''
 
-extra_tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) '''
+tag_expo_count = '''SELECT COUNT(`exposition_exposition_tag`.`exposition_id`) FROM `exposition_exposition_tag` WHERE (`theme_tag`.`id` = `exposition_exposition_tag`.`tag_id`) '''
-extra_tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) '''
+tag_conf_count = '''SELECT COUNT(`conference_conference_tag`.`conference_id`) FROM `conference_conference_tag` WHERE (`theme_tag`.`id` = `conference_conference_tag`.`tag_id`) '''
 
 class FilterForm(forms.Form):
     TYPES = EnumChoices(
@@ -82,19 +135,29 @@ class FilterForm(forms.Form):
     )
 
     model = forms.TypedMultipleChoiceField(label=_(u'Тип события'), coerce=int, choices=TYPES, required=False, widget=forms.CheckboxSelectMultiple())
-    theme = CountModelMultipleChoiceField(label=_(u'Тематики'),
+    theme = CountModelMultipleChoiceField(
-        queryset=Theme.objects.none(),
+        label=_(u'Тематики'), required=False,
-        required=False, widget=forms.CheckboxSelectMultiple())
+        queryset=Theme.objects.language().values('pk', 'name'))
-    tag = CountModelMultipleChoiceField(label=_(u'Теги'),
+    tag = CountModelMultipleChoiceField(
-        queryset=Tag.objects.none(),
+        label=_(u'Теги'), required=False,
-        required=False, widget=forms.CheckboxSelectMultiple())
+        queryset=Tag.objects.language().values('pk', 'name'))
+    country = CountModelMultipleChoiceField(
+        label=_(u'Страны'), required=False,
+        queryset=Country.objects.language().values('pk', 'name'))
+    city = CountModelMultipleChoiceField(
+        label=_(u'Города'), required=False,
+        queryset=City.objects.language().values('pk', 'name'))
 
     def __init__(self, *args, **kwargs):
         super(FilterForm, self).__init__(*args, **kwargs)
         self._is_valid = False
         self._models = None
-        self.fields['theme'].queryset = self.get_theme_choices()
+        self._lookup_kwargs = None
-        self.fields['tag'].queryset = self.get_tag_choices()
+        # self.selected_filter_data = {}
+        # self.fields['theme'].queryset = self.get_theme_choices()
+        # self.fields['tag'].queryset = self.get_tag_choices()
+        # self.fields['country'].queryset = self.get_country_choices()
+        # self.fields['city'].queryset = self.get_city_choices()
 
     def is_valid(self):
         # if getattr(self, '_is_valid', None) is None:
@@ -116,67 +179,192 @@ class FilterForm(forms.Form):
                 self._models.append(Conference)
         return self._models or [Exposition, Conference]
 
-    def get_theme_choices(self):
+    def get_theme_choices(self, ids):
         # 3-й рабочий способ (с родным заполением перевода)
         # в ходе поиска решения, был найден и 4-й рабочий способ используя RawSQL, но он для Django >= 1.8
         # https://docs.djangoproject.com/en/1.9/ref/models/expressions/#raw-sql-expressions
         # from django.db.models.expressions import RawSQL
+        conf = []
+        expo = []
+        for _type, pk in ids:
+            if _type == 'conference':
+                conf.append(pk)
+            else:
+                expo.append(int(pk))
 
         if getattr(self, '_theme_choices', None) is None:
             if Exposition in self.models and Conference in self.models:
                 count_query = '({q1}) + ({q2})'.format(
-                    q1=extra_theme_expo_count,
+                    q1=theme_expo_count if not expo else theme_expo_count_selected.format(ids=tuple(expo)),
-                    q2=extra_theme_conf_count)
+                    q2=theme_conf_count if not conf else theme_conf_count_selected.format(ids=tuple(conf)),
+                    )
                 filter_types = Q(types=Theme.types.conference) | Q(types=Theme.types.exposition)
             elif Exposition in self.models:
-                count_query = extra_theme_expo_count
+                count_query = theme_expo_count
                 filter_types = Q(types=Theme.types.exposition)
             elif Conference in self.models:
-                count_query = extra_theme_conf_count
+                count_query = theme_conf_count
                 filter_types = Q(types=Theme.types.conference)
 
             self._theme_choices = Theme.objects.language()\
                 .filter(filter_types)\
                 .extra(select={'count': count_query})\
-                .order_by('-count', '-name')
+                # .order_by('-count', '-name')
         return self._theme_choices
 
     def get_tag_choices(self):
-        extra_tag_select = {}
+        tag_select = {}
         if getattr(self, '_tag_choices', None) is None:
             if Exposition in self.models and Conference in self.models:
                 count_query = '({q1}) + ({q2})'.format(
-                    q1=extra_tag_expo_count,
+                    q1=tag_expo_count,
-                    q2=extra_tag_conf_count)
+                    q2=tag_conf_count)
             elif Exposition in self.models:
-                count_query = extra_tag_expo_count
+                count_query = tag_expo_count
             elif Conference in self.models:
-                count_query = extra_tag_conf_count
+                count_query = tag_conf_count
             self._tag_choices = Tag.objects.language()\
                 .extra(select={'count': count_query})\
-                .order_by('-count', '-name')
+                # .order_by('-count', '-name')
         return self._tag_choices
 
-    def filter(self):
+    @property
-        qs = self.default_filter()
+    def lookup_kwargs(self):
-        d = self.cleaned_data
+        if self._lookup_kwargs is None:
-        if d.get('theme'):
+            d = self.cleaned_data
-            qs = qs.filter(theme__in=d.get('theme'))
+            self._lookup_kwargs = {}
-        if d.get('tag'):
+            if d.get('theme'):
-            qs = qs.filter(tag__in=d.get('tag'))
+                self._lookup_kwargs['theme'] = {
+                    'theme__in': d.get('theme')
+                }
+            if d.get('tag'):
+                self._lookup_kwargs['tag'] = {
+                    'tag__in': d.get('tag'),
+                }
+            if d.get('country'):
+                self._lookup_kwargs['country'] = {
+                    'country_id__in': d.get('country'),
+                }
+            if d.get('city'):
+                self._lookup_kwargs['country'] = {
+                    'city_id__in': d.get('city'),
+                }
+        return self._lookup_kwargs
+
+    def filter(self, lookup_kwargs=None, qs=None):
+        qs = qs or self.default_filter()
+        lookup_kwargs = dict(ChainMap({}, *(lookup_kwargs or self.lookup_kwargs).values()))
+        return qs.filter(**lookup_kwargs)
+
+    def default_filter(self, load_all=True):
+        qs = RelatedSearchQuerySet().models(*self.models)
+        if load_all:
+            qs = qs.load_all()
+            for model in self.models:
+                qs = qs.load_all_queryset(model, model.enable.all())
         return qs
 
-    def default_filter(self):
+    def recalculate_choices(self):
-        qs = RelatedSearchQuerySet().models(*self.models).load_all()
+        print(self._is_valid)
+        if self._is_valid:
+            for field, val in self.lookup_kwargs.iteritems():
+                field_qs = self.default_filter(load_all=False)
+                field_lookup_kwargs = self.lookup_kwargs.copy()
+                del field_lookup_kwargs[field]
+                # if not field_lookup_kwargs:
+                #     continue
+                # field_qs = (x.id.split('.')[1:] for x in self.filter(qs=field_qs, lookup_kwargs=field_lookup_kwargs) if x.id)
+                # if field == 'theme':
+                #     self.fields[field].queryset = self.get_theme_choices(field_qs)
+                self.fields[field].queryset = self.fields[field].queryset.extra(
+                    select=self.make_count_select(field, field_lookup_kwargs)).values('pk', 'name', 'count')
+                print(self.fields[field].queryset.query)
+                # self.make_count_select(field, field_lookup_kwargs)
+                    # print(field_qs)
+
+        for field in self.fields:
+            field = self.fields[field]
+            if hasattr(field, 'queryset'):
+                field.queryset = field.queryset[:15]
+
+    def make_count_select(self, field, lookup_kwargs):
+        count_selects = []
+        print('looking {} {}'.format(field, lookup_kwargs))
         for model in self.models:
-            qs = qs.load_all_queryset(model,
+            _field, _model, direct, m2m = model._meta.get_field_by_name(field)
-                model.enable.all()
+            if m2m:
-                # не реализовано в hvad <_<
+                _field
-                # .only(
+                format_kwargs = {
-                #     'canceled', 'name', 'main_title', 'expohit', 'logo',
+                    'm2m_db_table': _field.m2m_db_table(),
-                #     'quality_label', 'services', 'visitors', 'members',
+                    'm2m_column_name': _field.m2m_column_name(),
-                #     'data_begin', 'data_end', 'country__url', 'country__name',
+                    'db_table': model._meta.db_table,
-                #     'city__url', 'place__name'
+                    'm2m_reverse_name': _field.m2m_reverse_name(),
-                # )
+                    'm2m_rel_to_table': _field.rel.to._meta.db_table,
-            )
+                }
-        return qs
+
+                select = \
+                    '''SELECT COUNT(`{m2m_db_table}`.`{m2m_column_name}`) FROM `{m2m_db_table}` INNER JOIN `{db_table}` ON (`{m2m_db_table}`.`{m2m_column_name}` = `{db_table}`.`id`) '''\
+                    .format(**format_kwargs)
+                joins = []
+                where = [
+                ''' (`{m2m_rel_to_table}_translation`.`master_id` = `{m2m_db_table}`.`{m2m_reverse_name}`) '''.format(**format_kwargs)
+                ]
+
+                for l_field, lookups in lookup_kwargs.iteritems():
+                    _l_field, l_model, _direct, _m2m = model._meta.get_field_by_name(l_field)
+                    values = lookups.values()[0]
+                    _format_kwargs = {
+                        'm2m_db_table': _l_field.m2m_db_table(),
+                        'm2m_column_name': _l_field.m2m_column_name(),
+                        'db_table': model._meta.db_table,
+                        'm2m_reverse_name': _l_field.m2m_reverse_name(),
+                        'ids': tuple(values) if len(values) > 1 else '({})'.format(*values),
+                    }
+                    joins.append(
+                        ''' INNER JOIN `{m2m_db_table}` ON (`{db_table}`.`id` = `{m2m_db_table}`.`{m2m_column_name}`)'''\
+                        .format(**_format_kwargs))
+                    where.append(
+                        '''`{m2m_db_table}`.`{m2m_reverse_name}` IN {ids}'''\
+                        .format(**_format_kwargs))
+                count_selects.append(select + ''.join(joins) + ' where ' + ' and '.join(where))
+
+            # case selected
+            # todo
+        if len(count_selects) == 2:
+            count = '({}) + ({})'.format(*count_selects)
+        elif len(count_selects) == 1:
+            count = count_selects[0]
+        return {'count': count}
+
+'''
+SELECT
+    (SELECT
+            COUNT(`exposition_exposition_theme`.`exposition_id`)
+        FROM
+            `exposition_exposition_theme`
+                INNER JOIN
+            `exposition_exposition` ON (`exposition_exposition_theme`.`exposition_id` = `exposition_exposition`.`id`)
+                INNER JOIN
+            `exposition_exposition_tag` ON (`exposition_exposition`.`id` = `exposition_exposition_tag`.`exposition_id`)
+        WHERE
+            `exposition_exposition_tag`.`tag_id` IN (469 , 832, 366, 922)
+                and (`theme_theme`.`id` = `exposition_exposition_theme`.`theme_id`)) AS `count`,
+    CASE
+        WHEN `theme_theme_translation`.`master_id` in (45 , 50, 60, 70, 80) THEN 1
+        ELSE 0
+    END as `selected`,
+    `theme_theme_translation`.`id`,
+    `theme_theme_translation`.`name`,
+    `theme_theme_translation`.`language_code`,
+    `theme_theme_translation`.`master_id`
+FROM
+    `theme_theme_translation`
+        INNER JOIN
+    `theme_theme` ON (`theme_theme_translation`.`master_id` = `theme_theme`.`id`)
+WHERE
+    (`theme_theme`.`types` = (`theme_theme`.`types` | 1)
+        OR `theme_theme`.`types` = (`theme_theme`.`types` | 2))
+        AND (`theme_theme_translation`.`language_code` = 'ru')
+order BY `selected` DESC , `count` DESC
+
+'''

events/views.py

@@ -28,6 +28,7 @@ class FilterListView(ContextMixin, FormMixin, ListView):
             qs = self.form.filter()
         else:
             qs = self.form.default_filter()
+        self.form.recalculate_choices()
         return qs
 
     def get(self, request, *args, **kwargs):