Merge remote-tracking branch 'origin/key_field' into key_field

.gitlab-ci.yml

@@ -15,7 +15,7 @@ deploy_branch:
     - ssh-add <(echo "$SSH_PRIVATE_KEY")
     - mkdir -p ~/.ssh
     - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
-    - ~/.composer/vendor/bin/envoy run deploy --cleanup --branch="$CI_BUILD_REF_SLUG"
+    - ~/.composer/vendor/bin/envoy run deploydev --cleanup --branch="$CI_BUILD_REF_SLUG"
   environment:
     name: production
     url: https://go.skillbox.ru

Envoy.blade.php

@@ -1,4 +1,5 @@
 @servers(['localhost' => 'deployer@192.168.0.13'])
+@servers(['dev' => 'deployer@192.168.0.5'])
 
 @setup
     $repository = 'ssh://git@g.skillbox.ru:22004/kor_a_m/skill-back.git';
@@ -13,11 +14,22 @@
     create_symlinks
     install_req
     run_tests
+    run_migrate
     update_symlinks
     deployment_option_cleanup
 @endstory
 
-@task('clone_repository', ['on' => 'localhost'])
+@story('deploydev', ['on' => 'dev'])
+    clone_repository
+    create_symlinks
+    install_req
+    run_tests
+    run_migrate
+    update_symlinks
+    deployment_option_cleanup
+@endstory
+
+@task('clone_repository')
     echo '>> Клонируем репозиторий'
     
     echo '{{ $new_release_dir }}'
@@ -27,7 +39,7 @@ echo '{{ $new_release_dir }}'
     @endif
 @endtask
 
-@task('create_symlinks', ['on' => 'localhost'])
+@task('create_symlinks')
     echo '>> Создание симлинков'
     ln -s {{ $app_dir }}/configs/master/ssl {{ $new_release_dir }}/ssl
     @if ($branch == 'master')
@@ -41,7 +53,7 @@ ln -s {{ $app_dir }}/configs/master/ssl {{ $new_release_dir }}/ssl
     ln -nfs {{ $app_dir }}/logs/{{ $branch }} {{ $new_release_dir }}/logs
 @endtask
 
-@task('install_req', ['on' => 'localhost'])
+@task('install_req')
     echo '>> Подтягиваем зависимости'
     @if ($branch)
         cd {{ $new_release_dir }}
@@ -49,7 +61,7 @@ echo '>> Подтягиваем зависимости'
     @endif
 @endtask
 
-@task('run_tests', ['on' => 'localhost'])
+@task('run_tests')
     echo '>> Запускаем тесты'
     @if ($branch)
         cd {{ $new_release_dir }}
@@ -57,7 +69,7 @@ echo '>> Запускаем тесты'
     @endif
 @endtask
 
-@task('run_migrate', ['on' => 'localhost'])
+@task('run_migrate')
     echo '>> Запускаем миграции'
     @if ($branch)
         cd {{ $new_release_dir }}
@@ -65,7 +77,7 @@ echo '>> Запускаем миграции'
     @endif
 @endtask
 
-@task('update_symlinks', ['on' => 'localhost'])
+@task('update_symlinks')
     @if ($branch)
         ln -nfs {{ $app_dir }}/configs/{{ $branch }}/wsgi.py {{ $new_release_dir }}/lms/wsgi.py
         echo '>> Создаем симлинк на папку текущего релиза'

access/models/user.py

@@ -71,8 +71,8 @@ class CustomUserManager(BaseUserManager):
             body = {
                 "subject": 'Спасибо за регистрацию',
                 "body": '''
-                        Вы были успешны зарегистрированны на портале go.skillbox.ru
+                        Вы были успешно зарегистрированы на портале go.skillbox.ru
-                        ваш пароль (он будет дествителен после активации по ссылке) %s 
+                        ваш пароль (он будет действителен после активации по ссылке) %s 
                         для подтверждения регистрации перейдите по ссылке
                         %s/api/v1/users/registration/?hash=%s''' % (password, settings.DOMAIN, invite.hash),
                 "from_email": 'robo@skillbox.ru',

access/views.py

@@ -83,10 +83,10 @@ class ResetPasswordView(APIView):
                     subject="Сброс пароля",
                     message='''
                         Ваш новый пароль (может быть использован после перехода по ссылке) %s,
-                        (в последствии вы сможите сменить его в личном кабинете),
+                        (в последствии вы сможете сменить его в личном кабинете),
                         если вы не отправляли заявку на сброс пароля просто проигнорируйте это сообщение,
                         для подтверждения смены пароля перейдите по %s/api/v1/users/reset/?hash=%s
-                        (ссылке ссылка действительна в течении одного дня)''' % (invite.password, settings.DOMAIN, invite.hash),
+                        ссылке (ссылка действительна в течении одного дня)''' % (invite.password, settings.DOMAIN, invite.hash),
                     from_email='robo@skillbox.ru',
                     recipient_list=[user.email],
                 )
@@ -314,7 +314,7 @@ class ManagementPassword(generics.GenericAPIView):
             password = ''.join(random.choice(string.ascii_letters) for _x in range(8))
 
         try:
-            user = get_user_model().objects.get(email=email)
+            user = get_user_model().objects.get(email=email.lower())
         except get_user_model().DoesNotExist:
             return Response('user not found', status=404)
 
@@ -327,7 +327,7 @@ class ManagementPassword(generics.GenericAPIView):
         msg = EmailMessage(
             subject='Установлен новый пароль',
             body='''Ваш новый пароль %s 
-                (в последствии вы сможите сменить его в личном кабинете).''' % password,
+                (в последствии вы сможете сменить его в личном кабинете).''' % password,
             from_email='robo@skillbox.ru',
             to=[user.email],
             bcc=[request.user.email],

courses/api.py

@@ -1,4 +1,4 @@
-from courses.models import Course
+from courses.models import Course, Lesson
 
 
 class InApiTeacher:
@@ -53,7 +53,15 @@ class CourseProgressApi:
 class CourseParamsApi:
 
     def __init__(self, token):
-        self.course = Course.objects.get(token=token)
+        self.__course = Course.objects.get(token=token)
 
     def get_slug_and_title(self):
-        return {"title": self.course.title, "slug": self.course.slug}
+        return {"title": self.__course.title, "slug": self.__course.slug}
+
+    def get_length(self):
+        return self.__course.topic_set.count()
+
+    def get_topic_by_lesson(self, lesson_token):
+        for idx, topic in enumerate(self.__course.topic_set.all()):
+            if topic.lesson_set.filter(token=lesson_token).exists():
+                return {"title": topic.title, "idx": idx}

courses/models.py

@@ -77,6 +77,7 @@ class CourseManager(models.Manager):
                                 level=None, direction=None, **kwargs):
 
         slug = slug if slug else slugify(unidecode.unidecode(kwargs['title']))
+        old_slug = slug if old_slug is None else old_slug
 
         kwargs['teacher_tokens'] = teacher_tokens
 
@@ -109,7 +110,8 @@ class CourseManager(models.Manager):
             course.slug = slug
             course.save()
 
-        except ObjectDoesNotExist:
+        # TODO Костылище
+        except Exception:
             kwargs['slug'] = slug
             course = self.create(**kwargs)
 

finance/models.py

@@ -44,8 +44,8 @@ class Bill(models.Model):
                 })
 
             msg = EmailMessage(
-                'Вы откозались от оплаты по счёту',
+                'Вы отказались от оплаты по счёту',
-                """Вы откозались от оплаты по счёту.
+                """Вы отказались от оплаты по счёту.
                 Вы сможете продолжить оплату в личном кабинете""",
                 to=[self.user.email],
                 bcc=[self.opener.email],

finance/signals.py

@@ -58,7 +58,7 @@ def invoice_signal(instance, **kwargs):
     if instance.status == 'C':
         msg = EmailMessage(
             'Ошибка платежа!'
-            """Внимание не прошёл платёж пользавателю %s,
+            """Внимание не прошёл платёж пользавателя %s,
             по курсу "%s" ID платежа: %s. Если не получается
             решить проблему самостоятельно, ответьте на это письмо,
             постарайтесь подробно описать последовательность действий,

finance/tasks.py

@@ -1,4 +1,3 @@
-import logging
 import os
 
 import requests

finance/views.py

@@ -187,6 +187,7 @@ class InvoiceDetailView(APIView):
                     )
 
             if invoice.status == "F":
+                invoice.real_price = invoice.real_price if real_price is None else real_price
                 return Response(InvoiceSerializer(invoice).data, status=200)
 
             invoice.real_price = real_price
@@ -205,7 +206,8 @@ class InvoiceDetailView(APIView):
             if pay_count > 1:
                 invoice.create_child_pays(pay_count)
 
-            if invoice.method == 'Y' and invoice.yandex_pay is None:
+            if invoice.method == 'Y' and invoice.status == 'P':
+                if invoice.yandex_pay is None:
                     yandex_pay = Payment(
                         order_amount=invoice.price,
                         shop_amount=0,
@@ -218,6 +220,7 @@ class InvoiceDetailView(APIView):
                         yandex_pay.scid = settings.YANDEX_MONEY_REBILLING_SCID
                     yandex_pay.save()
                     invoice.yandex_pay = yandex_pay
+
                 invoice.send_link()
 
                 context = {

lms/celery.py

@@ -21,9 +21,12 @@ register_signal(client)
 
 app.conf.beat_schedule = {
     'periodic_billing': {
-        # 'schedule': crontab(day_of_month='1', hour='1', minute='1'),  # заглушка на время отладки
         'schedule': crontab(minute='0', hour='*/3',),
         'task': 'finance.tasks.periodic_billing'
+    },
+    'statistic_teachers': {
+        'schedule': crontab(hour='16', minute='0'),
+        'task': 'progress.tasks.statistic_teachers'
     }
 }
 

progress/migrations/0011_auto_20180427_1213.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-04-27 12:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('progress', '0010_progress_exp_date'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='progresslesson',
+            name='last_update',
+            field=models.DateTimeField(verbose_name='Дата последней проверки'),
+        ),
+    ]

progress/migrations/0012_auto_20180427_1220.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-04-27 12:20
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('progress', '0011_auto_20180427_1213'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='progresslesson',
+            name='last_update',
+            field=models.DateTimeField(auto_now_add=True, verbose_name='Дата последней здачи'),
+        ),
+    ]

progress/migrations/0013_progresslesson_dif_check_timestamps.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-04-27 12:26
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import lms.tools
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('progress', '0012_auto_20180427_1220'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='progresslesson',
+            name='dif_check_timestamps',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(verbose_name='Время проверки'), default=lms.tools.get_empty_list, size=None, verbose_name='Временные интервалы проверок'),
+        ),
+    ]

progress/models.py

@@ -3,6 +3,7 @@ from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from model_utils import Choices
 
+from django.utils import timezone
 from lms.tools import get_empty_list
 
 
@@ -55,8 +56,30 @@ class ProgressLesson(models.Model):
     finish_date = models.DateTimeField(verbose_name='Дата зачтения задания', blank=True, null=True)
     start_date = models.DateTimeField(verbose_name='Дата начала прохождения задания', auto_now_add=True)
     status = models.CharField(choices=STATUSES, default=STATUSES.start, max_length=20)
-    last_update = models.DateTimeField(verbose_name='Дата последнего изменения', auto_now=True)
+    last_update = models.DateTimeField(verbose_name='Дата последней сдачи', auto_now_add=True)
     comment_tokens = ArrayField(models.UUIDField(verbose_name="Токен комента", editable=False), default=get_empty_list)
+    dif_check_timestamps = ArrayField(
+        models.IntegerField(
+            verbose_name="Время проверки"
+        ),
+        verbose_name='Временные интервалы проверок',
+        default=get_empty_list,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super(ProgressLesson, self).__init__(*args, **kwargs)
+        self.old_status = self.status
+
+    def save(self, force_insert=False, force_update=False, **kwargs):
+        if self.status == 'wait' and (self.old_status == 'start' or self.old_status == 'fail'):
+            self.last_update = timezone.now()
+
+        if self.old_status == 'wait' and (self.status == 'fail' or self.status == 'done'):
+            dif = int(timezone.now().timestamp() - self.last_update.timestamp())
+            self.dif_check_timestamps.append(dif)
+
+        super(ProgressLesson, self).save(force_insert, force_update, **kwargs)
+        self.old_status = self.status
 
     def __str__(self):
         return self.progress.user.email

progress/tasks.py

@@ -1,6 +1,16 @@
-from courses.api import CourseProgressApi
+from dateutil.relativedelta import relativedelta
+
+from courses.api import CourseProgressApi, CourseParamsApi
 from progress.models import ProgressLesson
 
+from django.utils import timezone
+import os
+import csv
+
+from lms import celery_app
+from django.conf import settings
+from django.core.mail import EmailMessage
+
 
 def add_next_lesson(progress):
     lesson_list = CourseProgressApi.get_next(
@@ -25,3 +35,44 @@ def add_next_lesson(progress):
             )
 
     return pl
+
+
+@celery_app.task
+def statistic_teachers():
+    file_dir = "%s/analytics/teachers/" % (settings.MEDIA_ROOT,)
+    os.makedirs(os.path.dirname(file_dir), exist_ok=True)
+    # TODO Почему-то при выставлении расширения всё ломается
+    path = '%s%s' % (file_dir, timezone.now().strftime('%Y-%m-%d__%H-%M'))
+
+    progresses = ProgressLesson.objects.filter(status='wait')
+
+    with open(path, 'w') as csv_file:
+        csv_writer = csv.writer(csv_file)
+        csv_writer.writerow(
+            ['Курс', 'Препод', 'Ранее', '6 Дней', '5 Дней', "4 Дня", "3 Дня", "2 Дня", "Вчера", "Сегодня"])
+        for i in progresses.values('progress__teacher__email', 'progress__course_token').distinct():
+            row = []
+            course_token = i['progress__course_token']
+            teacher_email = i['progress__teacher__email']
+            progresses_sub = progresses.filter(progress__course_token=course_token, progress__teacher__email=teacher_email)
+
+            row.append(CourseParamsApi(course_token).get_slug_and_title()['title'])
+            row.append(teacher_email)
+            row.append(progresses_sub.filter(last_update__lt=(timezone.now() - relativedelta(days=7))).count())
+            for j in range(7):
+                from_d = timezone.now() - relativedelta(days=(7 - j))
+                to_d = timezone.now() - relativedelta(days=(6 - j))
+                row.append(progresses_sub.filter(last_update__gte=from_d, last_update__lt=to_d).count())
+
+            csv_writer.writerow(row)
+
+    message = EmailMessage(
+        'Выгрузка по преподам.',
+        'Файл приложен',
+        'robo@skillbox.ru',
+        [settings.SUPPORT_EMAIL,],
+        ['andrey.korolev@skillbox.ru',],
+        reply_to=['andrey.korolev@skillbox.ru'],
+    )
+    message.attach_file(path)
+    message.send()

progress/urls.py

@@ -4,6 +4,8 @@ from progress import views
 
 urlpatterns = [
     url(r'progress_dynamic/$', views.CourseProgressDynamicView.as_view()),
+    url(r'progress_table_detail/$', views.ProgressTableDetailView.as_view()),
+    url(r'progress_table/$', views.CourseProgressTableView.as_view()),
     url(r'students/(?P<teacher_token>[0-9A-Fa-f-]+)/$', views.StudentWorkView.as_view()),
     url(r'student/$', views.StudentUpdateProgress.as_view()),
     url(r'find/$', views.FindProgressView.as_view()),

progress/views.py

@@ -15,14 +15,16 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 from rest_framework.views import APIView
-from django.db.models import Q, Count
+from django.db.models import Q, Count, F
 import jwt
 from django.conf import settings
 
+from django.db.models.functions import Cast
+from django.db.models.fields import DateField
 from access.serializers import UserProgressSearchSerializer
 from courses.models import Course, Lesson
 from progress.models import ProgressLesson, Progress
-from progress.serializers import ProgressAnalyticSerializer, ProgressLessonSerializer, ProgressSerializer, \
+from progress.serializers import ProgressAnalyticSerializer, ProgressLessonSerializer, \
     SecureProgressSerializer
 from courses.api import CourseProgressApi, CourseParamsApi
 from progress.tasks import add_next_lesson
@@ -65,27 +67,134 @@ class CourseProgressDynamicView(APIView):
 
     @staticmethod
     def get(request):
-        course_token = request.GET.get('course_token', None)
+        course_token = request.GET.get('course_token', '')
+        teacher_email = request.GET.get('teacher_email', '').lower()
         from_date = int(request.GET.get('from', '7'))
         to_date = int(request.GET.get('to', '0'))
+        only_hw = request.GET.get('only_hw', 'yes') == 'yes'
 
         if request.user.is_authenticated() and request.user.is_staff:
-            progresses = ProgressLesson.objects.filter(status='done')
+            progresses = ProgressLesson.objects.all()
-            if not course_token is None:
+            waiters = ProgressLesson.objects.filter(status='wait')
-                try:
+
+            if only_hw:
+                progresses = progresses.filter(checker=F('progress__teacher'))
+
+            if not course_token is '':
                 progresses = progresses.filter(progress__course_token=course_token)
-                except ProgressLesson.DoesNotExist:
-                    return Response('incorrect course token', status=404)
 
-            progresses = progresses.filter(finish_date__gt=timezone.now() - relativedelta(days=from_date))
+            if not teacher_email is '':
-            progresses = progresses.filter(finish_date__lt=timezone.now() - relativedelta(days=to_date))
+                waiters = waiters.filter(checker__email=teacher_email)
-            progresses = progresses.order_by('finish_date')
+                progresses = progresses.filter(checker__email=teacher_email)
+
+            res = {}
+
+            progresses_finish = progresses.filter(finish_date__gt=timezone.now() - relativedelta(days=from_date))
+            progresses_finish = progresses_finish.filter(finish_date__lt=timezone.now() - relativedelta(days=to_date))
+
+            progresses_start = progresses.filter(start_date__gt=timezone.now() - relativedelta(days=from_date))
+            progresses_start = progresses_start.filter(start_date__lt=timezone.now() - relativedelta(days=to_date))
+
+            progresses_finish = progresses_finish.annotate(name=Cast('finish_date', DateField())) \
+                                                 .values('name') \
+                                                 .annotate(value=Count('id'))
+
+            progresses_start = progresses_start.annotate(name=Cast('start_date', DateField())) \
+                                               .values('name') \
+                                               .annotate(value=Count('id'))
+
+            progresses_finish = progresses_finish.order_by('name')
+            progresses_start = progresses_start.order_by('name')
+
+            res['finish'] = progresses_finish
+            res['start'] = progresses_start
+            res['in_progress'] = waiters.count()
+
+            return Response(res, status=200)
+        return Response('Доступно только персоналу', status=403)
+
+
+class CourseProgressTableView(APIView):
+    renderer_classes = (JSONRenderer,)
+
+    @staticmethod
+    def get(request):
+        from_date = request.GET.get('from', None)
+        to_date = request.GET.get('to', None)
+
+        if request.user.is_authenticated() and request.user.is_staff:
+            progresses = ProgressLesson.objects.filter(checker=F('progress__teacher'), progress__only_watch=False) \
+                                               .exclude(status='done')
+
+            if from_date is not None:
+                date = timezone.now() - relativedelta(days=int(from_date))
+                progresses = progresses.filter(progress__user__useractivity__last_request__gt=date)
+
+            if to_date is not None:
+                date = timezone.now() - relativedelta(days=int(to_date))
+                progresses = progresses.filter(progress__user__useractivity__last_request__lt=date)
 
             res = {}
-            progresses = progresses.extra({'finish_date_day': "date(finish_date)"})\
+            progresses = progresses.values('progress__course_token', 'lesson_token').annotate(count=Count('id'))
-                                   .values('progress__course_token', 'finish_date_day')\
+            course_tokens = progresses.values('progress__course_token').distinct()
-                                   .annotate(total=Count('progress'))
+            res['val'] = []
-            #TODO Доделать
+            res['max_length'] = 0
+            for course_token in course_tokens:
+                ct = course_token['progress__course_token']
+                api = CourseParamsApi(ct)
+                course_title = api.get_slug_and_title()['title']
+                course_length = api.get_length()
+                res['max_length'] = course_length if course_length > res['max_length'] else res['max_length']
+                f_progresses = progresses.filter(progress__course_token=ct)
+                res_elem = {
+                    "course_title": course_title,
+                    "course_token": course_token['progress__course_token'],
+                    'stat': [0 for _i in range(course_length)],
+                    "sum": 0
+                }
+                res['val'].append(res_elem)
+                for i in f_progresses:
+                    topic = api.get_topic_by_lesson(i['lesson_token'])
+                    if topic is not None:
+                        res_elem['stat'][topic['idx']] += i["count"]
+                        res_elem['sum'] += i["count"]
+
+            return Response(res, status=200)
+        return Response('Доступно только персоналу', status=403)
+
+
+class ProgressTableDetailView(APIView):
+    renderer_classes = (JSONRenderer,)
+
+    @staticmethod
+    def get(request):
+        course_token = request.GET.get('course_token', None)
+        topic_idx = request.GET.get('topic_idx', None)
+
+        if course_token is None:
+            return Response('Должен быть передан токен курса', 400)
+
+        if request.user.is_authenticated() and request.user.is_staff:
+            progresses = ProgressLesson.objects.filter(
+                                                   checker=F('progress__teacher'),
+                                                   progress__only_watch=False,
+                                                   progress__course_token=course_token,
+                                               ) \
+                                               .exclude(status='done')
+
+            api = CourseParamsApi(course_token)
+            res = []
+            for progress in progresses:
+                topic = api.get_topic_by_lesson(progress.lesson_token)
+                if topic_idx is not None and not int(topic_idx)-1 == topic['idx']:
+                    continue
+                res.append({
+                    'email': progress.progress.user.email,
+                    'name': progress.progress.user.get_full_name(),
+                    'status': progress.status,
+                    'last_request': progress.progress.user.useractivity.last_request,
+                    'topic_title': topic['title'],
+                })
 
             return Response(res, status=200)
         return Response('Доступно только персоналу', status=403)
@@ -337,7 +446,7 @@ class ChangeTeacherView(APIView):
             except get_user_model().DoesNotExist:
                 return Response("Нет пользователя c таким email", status=404)
             p.teacher = teacher
-            p.progresslesson_set.filter(status=ProgressLesson.STATUSES.wait).update(checker=teacher)
+            p.progresslesson_set.exclude(status=ProgressLesson.STATUSES.done).update(checker=teacher)
             p.save()
         except Progress.DoesNotExist:
             return Response("не найден прогресс по заданному id", status=404)